Refactoring for better performance of large auto-cli specs

* Fixed: [very slow execution of load_set_file #288](https://github.com/clicon/clixon/issues/288)
* New `clixon-lib@2021-11-11.yang` revision
  * Modified option: RPC stats extended with YANG stats
* Modified `clixon-config@2021-11-11.yang` revision
  * Added option:
    * CLICON_PLUGIN_CALLBACK_CHECK
    * Enable to make plugin context check before and after all callbacks.
* Added statistics for YANG: number of objects and memory used
* Use the treeref no-copy option of CLIgen to reduce memory
* Refactored cli-generation/autocli-start code
* Refactored cligen glue functions to use cligen_eval directly (remove clicon_eval,clixon_cligen_eval)

CHANGELOG.md

@@ -59,7 +59,11 @@ Users may have to change how they access the system
   * New behavior:
     * JSON: `{"s":"<![CDATA[  z > x  & x < y ]]>"}`  
   * To keep old behavior, set `JSON_CDATA_STRIP` in clixon_custom.h
+* New `clixon-lib@2021-11-11.yang` revision
+  * Modified option: RPC stats extended with YANG stats
 * New `clixon-config@2021-11-11.yang` revision
+  * Added option:
+    * CLICON_PLUGIN_CALLBACK_CHECK
   * Modified options:
     * CLICON_CLI_GENMODEL_TYPE: added OC_COMPRESS enum
     * CLICON_YANG_DIR: recursive search
@@ -92,6 +96,11 @@ Developers may need to change their code
 
 ### Minor features
 
+* Added statistics for YANG: number of objects and memory used
+  * See clixon-lib: stats rpc
+* Performance improvement
+  * Added ancestor config cache indicating wether the node or an ancestor is config false or true
+  * Improved yang cardinality lookup
 * Added sorting of YANG statements
   * Some openconfig specs seem to have use/when before a "config" which it depends on. This leads to XML encoding being in the "wrong order.
   * When parsing, clixon now sorts container/list statements so that sub-statements with WHEN are put last.
@@ -101,6 +110,7 @@ Developers may need to change their code
   * Check blocked signals and signal handlers
   * Check termios settings
   * Any changes to context are logged at loglevel WARNING
+  * New option: CLICON_PLUGIN_CALLBACK_CHECK: enable it to for checks (default false)
 * [OpenConfig path compression](https://github.com/clicon/clixon/pull/276)
 * C API: Added set/get pointer API to clixon_data:
    * Changed signature of `rpc_callback_call()`
@@ -114,6 +124,7 @@ Developers may need to change their code
 
 ### Corrected Bugs
 
+* [very slow execution of load_set_file #288](https://github.com/clicon/clixon/issues/288)
 * [RPC output not verified by yang](https://github.com/clicon/clixon/issues/283)
 * [Statements given in "load set" are order dependent](https://github.com/clicon/clixon/issues/287)
   * Modify ordering of XML encoding to put sub-elements with YANG WHEN statements last

apps/backend/backend_client.c

@@ -198,7 +198,7 @@ backend_client_rm(clicon_handle        h,
  * @retval       -1       Error
  */
 static int
-clixon_stats_get_db(clicon_handle h,
+clixon_stats_datastore_get(clicon_handle h,
 			   char         *dbname,
 			   cbuf         *cb)
 {
@@ -234,6 +234,37 @@ clixon_stats_get_db(clicon_handle h,
     return retval;
 }
 
+/*! Get clixon per datastore stats
+ * @param[in]     h       Clicon handle
+ * @param[in]     dbname  Datastore name
+ * @param[in,out] cb      Cligen buf
+ * @retval        0       OK
+ * @retval       -1       Error
+ */
+static int
+clixon_stats_module_get(clicon_handle h,
+			yang_stmt    *ys,
+			cbuf         *cb)
+{
+    int       retval = -1;
+    uint64_t  nr = 0;
+    size_t    sz = 0;
+    cxobj    *xn = NULL;
+    
+    if (ys == NULL)
+	return 0;
+    if (yang_stats(ys, &nr, &sz) < 0)
+	goto done;
+    cprintf(cb, "<module xmlns=\"%s\"><name>%s</name><nr>%" PRIu64 "</nr>"
+	    "<size>%zu</size></module>",
+	    CLIXON_LIB_NS, yang_argument_get(ys), nr, sz);
+    retval = 0;
+ done:
+    if (xn)
+	xml_free(xn);
+    return retval;
+}
+
 /*! Loads all or part of a specified configuration to target configuration
  * 
  * @param[in]  h       Clicon handle 
@@ -998,17 +1029,39 @@ from_client_stats(clicon_handle h,
 {
     int        retval = -1;
     uint64_t   nr;
+    yang_stmt *ym;
     
     cprintf(cbret, "<rpc-reply xmlns=\"%s\">", NETCONF_BASE_NAMESPACE);
+    xml_stats_global(&nr);
+    cprintf(cbret, "<global xmlns=\"%s\">", CLIXON_LIB_NS);
     nr=0;
     xml_stats_global(&nr);
-    cprintf(cbret, "<global xmlns=\"%s\"><xmlnr>%" PRIu64 "</xmlnr></global>", CLIXON_LIB_NS, nr);
+    cprintf(cbret, "<xmlnr>%" PRIu64 "</xmlnr>", nr);
-    if (clixon_stats_get_db(h, "running", cbret) < 0)
+    nr=0;
+    yang_stats_global(&nr);
+    cprintf(cbret, "<yangnr>%" PRIu64 "</yangnr>", nr);
+    cprintf(cbret, "</global>");
+    if (clixon_stats_datastore_get(h, "running", cbret) < 0)
 	goto done;
-    if (clixon_stats_get_db(h, "candidate", cbret) < 0)
+    if (clixon_stats_datastore_get(h, "candidate", cbret) < 0)
 	goto done;
-    if (clixon_stats_get_db(h, "startup", cbret) < 0)
+    if (clixon_stats_datastore_get(h, "startup", cbret) < 0)
 	goto done;
+    ym = NULL;
+    while ((ym = yn_each(clicon_config_yang(h), ym)) != NULL) {    
+	if (clixon_stats_module_get(h, ym, cbret) < 0)
+	    goto done;
+    }
+    ym = NULL;
+    while ((ym = yn_each(clicon_dbspec_yang(h), ym)) != NULL) {    
+	if (clixon_stats_module_get(h, ym, cbret) < 0)
+	    goto done;
+    }
+    ym = NULL;
+    while ((ym = yn_each(clicon_nacm_ext_yang(h), ym)) != NULL) {    
+	if (clixon_stats_module_get(h, ym, cbret) < 0)
+	    goto done;
+    }
     cprintf(cbret, "</rpc-reply>");
     retval = 0;
  done:

apps/backend/backend_commit.c

@@ -931,20 +931,21 @@ from_client_restart_one(clicon_handle h,
     yang_stmt          *yspec;
     int                 i;
     cxobj              *xn;
-    plugin_context_t   *pc = NULL;
+    void               *wh = NULL;
     
     yspec =  clicon_dbspec_yang(h);
     if (xmldb_db_reset(h, db) < 0)
 	goto done;
     /* Application may define extra xml in its reset function*/
     if ((resetfn = clixon_plugin_api_get(cp)->ca_reset) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if ((retval = resetfn(h, db)) < 0) {
 	    clicon_debug(1, "plugin_start() failed");
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     /* 1. Start transaction */
@@ -998,7 +999,6 @@ from_client_restart_one(clicon_handle h,
 	xml_flag_set(xn, XML_FLAG_CHANGE);
 	xml_apply_ancestor(xn, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
     }
-    
     /* Call plugin transaction start callbacks */
     if (plugin_transaction_begin_one(cp, h, td) < 0)
 	goto fail;
@@ -1026,8 +1026,6 @@ from_client_restart_one(clicon_handle h,
 	goto fail;
     retval = 1;
  done:
-    if (pc)
-	free(pc);
     if (td){
 	 xmldb_get0_free(h, &td->td_target);
 	 transaction_free(td);

apps/backend/backend_plugin.c

@@ -80,24 +80,25 @@ clixon_plugin_reset_one(clixon_plugin_t *cp,
 {
     int         retval = -1;
     plgreset_t *fn;       /* callback */
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
 
     if ((fn = clixon_plugin_api_get(cp)->ca_reset) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, db) < 0) {
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (clicon_errno < 0) 
 		clicon_log(LOG_WARNING, "%s: Internal error: Reset callback in plugin: %s returned -1 but did not make a clicon_err call",
 			   __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
     
@@ -138,26 +139,26 @@ clixon_plugin_pre_daemon_one(clixon_plugin_t *cp,
 {
     int          retval = -1;
     plgdaemon_t *fn;          /* Daemonize plugin callback function */
-    plugin_context_t   *pc = NULL;
+    void        *wh = NULL;
 
     if ((fn = clixon_plugin_api_get(cp)->ca_pre_daemon) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
-
         if (fn(h) < 0) {
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
             if (clicon_errno < 0)
                 clicon_log(LOG_WARNING, "%s: Internal error: Pre-daemon callback in plugin:\
  %s returned -1 but did not make a clicon_err call",
                            __FUNCTION__, clixon_plugin_name_get(cp));
             goto done;
         }
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -199,24 +200,25 @@ clixon_plugin_daemon_one(clixon_plugin_t *cp,
 {
     int          retval = -1;
     plgdaemon_t *fn;          /* Daemonize plugin callback function */    
-    plugin_context_t   *pc = NULL;
+    void        *wh = NULL;
 
     if ((fn = clixon_plugin_api_get(cp)->ca_daemon) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h) < 0) {
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (clicon_errno < 0) 
 		clicon_log(LOG_WARNING, "%s: Internal error: Daemon callback in plugin: %s returned -1 but did not make a clicon_err call",
 			   __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
     
@@ -283,28 +285,29 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
     int              retval = -1;
     plgstatedata_t  *fn;          /* Plugin statedata fn */
     cxobj           *x = NULL;
-    plugin_context_t *pc = NULL;
+    void            *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_statedata) != NULL){
 	if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
 	    goto done;
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, nsc, xpath, x) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (clicon_errno < 0) 
 		clicon_log(LOG_WARNING, "%s: Internal error: State callback in plugin: %s returned -1 but did not make a clicon_err call",
 			   __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto fail;  /* Dont quit here on user callbacks */
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     if (xp && x)
 	*xp = x;
     retval = 1;
  done:
-    if (pc)
-	free(pc);
     return retval;
  fail:
     retval = 0;
@@ -437,20 +440,19 @@ clixon_plugin_lockdb_one(clixon_plugin_t *cp,
 {
     int          retval = -1;
     plglockdb_t *fn;          /* Plugin statedata fn */
-    plugin_context_t *pc = NULL;
+    void        *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_lockdb) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, db, lock, id) < 0)
 	    goto done;  
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -613,24 +615,25 @@ plugin_transaction_begin_one(clixon_plugin_t    *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_begin) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -673,24 +676,26 @@ plugin_transaction_validate_one(clixon_plugin_t      *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
 
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_validate) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
+
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -731,24 +736,25 @@ plugin_transaction_complete_one(clixon_plugin_t      *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_complete) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -821,24 +827,25 @@ plugin_transaction_commit_one(clixon_plugin_t      *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_commit) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -872,7 +879,6 @@ plugin_transaction_commit_all(clicon_handle       h,
     return retval;
 }
 
-
 /*! Call single plugin transaction_commit_done() in a commit transaction
  * @param[in]  cp      Plugin handle
  * @param[in]  h       Clixon handle
@@ -887,24 +893,25 @@ plugin_transaction_commit_done_one(clixon_plugin_t      *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_commit_done) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -945,24 +952,25 @@ plugin_transaction_end_one(clixon_plugin_t      *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_end) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -996,24 +1004,25 @@ plugin_transaction_abort_one(clixon_plugin_t      *cp,
 {
     int         retval = -1;
     trans_cb_t *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
     
     if ((fn = clixon_plugin_api_get(cp)->ca_trans_abort) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, (transaction_data)td) < 0){
+	    if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+		goto done;
 	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
 		clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error", 
 		       __FUNCTION__, clixon_plugin_name_get(cp));
 	    goto done;
 	}
-	if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 

apps/cli/cli_auto.c

@@ -80,7 +80,7 @@
 static char *
 co2apipath(cg_obj *co)
 {
-    struct cg_callback *cb;
+    cg_callback *cb;
     cvec        *cvv;
     cg_var      *cv;
     
@@ -467,14 +467,23 @@ cli_auto_edit(clicon_handle h,
 	clicon_err(OE_PLUGIN, 0, "No such parsetree header: %s", treename);
 	goto done;
     }
-    /* Find the matching cligen object */
+    /* Find the matching cligen object 
-    if ((co = cligen_co_match(cli_cligen(h))) != NULL &&
+     * Note, is complictead: either an instantiated tree (co_treeref_orig)
-	(coorig = co->co_treeref_orig) != NULL)
+     * or actual tree (co_ref)
+     */
+    if ((co = cligen_co_match(cli_cligen(h))) != NULL){
+	if ((coorig = co->co_treeref_orig) != NULL ||
+	    (coorig = co->co_ref) != NULL)
 	    cligen_ph_workpoint_set(ph, coorig);
 	else {
 	    clicon_err(OE_YANG, EINVAL, "No workpoint found");
 	    goto done;
 	}
+    }
+    else{
+	clicon_err(OE_YANG, EINVAL, "No workpoint found");
+	goto done;
+    }
     if ((cvv2 = cvec_append(clicon_data_cvec_get(h, "cli-edit-cvv"), cvv1)) == NULL)
 	goto done;
     /*  API_path format */
@@ -500,7 +509,7 @@ cli_auto_edit(clicon_handle h,
 /*! CLI callback: Working point tree up to parent
  * @param[in]  h    CLICON handle
  * @param[in]  cvv  Vector of variables from CLIgen command-line
- * @param[in]  argv Vector oif user-supplied keywords
+ * @param[in]  argv Vector of user-supplied keywords
  * Format of argv:
  *   <treename>     Name of generated cligen parse-tree, eg "datamodel"
  */
@@ -991,7 +1000,7 @@ cli_auto_sub_enter(clicon_handle h,
     }
     /* Find the point in the generated clispec tree where workpoint should be set */
     fa.fa_str = api_path_fmt;
-    if (pt_apply(cligen_ph_parsetree_get(ph), cli_auto_findpt, &fa) < 0)
+    if (pt_apply(cligen_ph_parsetree_get(ph), cli_auto_findpt, INT32_MAX, &fa) < 0)
 	goto done;
     if (fa.fa_co == NULL){
 	clicon_err(OE_PLUGIN, ENOENT, "No such cligen object found %s", api_path);

apps/cli/cli_generate.c

@@ -1113,36 +1113,27 @@ yang2cli_stmt(clicon_handle h,
     return retval;
 }
 
-/*! Generate CLI code for Yang specification
- * @param[in]  h         Clixon handle
- * @param[in]  yn        Create parse-tree from this yang node
- * @param[in]  printgen  Log generated CLIgen syntax
- * @param[in]  state     Set to include state syntax
- * @param[in]  show_tree Is tree for show cli command
- * @param[out] pt        CLIgen parse-tree (must be created on input)
- * @retval     0         OK
- * @retval    -1         Error
- */
 int
-yang2cli(clicon_handle      h, 
+yang2cli_yspec(clicon_handle      h, 
 	       yang_stmt         *yn, 
+	       char              *name0,
 	       int                printgen,
 	       int                state,
-	 int 		    show_tree,
+	       int 		  show_tree)
-	 parse_tree        *pt)
 {
     int           retval = -1;
-    cbuf         *cb = NULL;
+    parse_tree   *pt0 = NULL;
-    yang_stmt    *yc;
+    cbuf         *cb0 = NULL;
-    cvec         *globals;       /* global variables from syntax */
     genmodel_type gt;
     char         *excludelist;
     char        **exvec = NULL;
     int           nexvec = 0;
     int           e;
+    yang_stmt    *ym;
+    pt_head      *ph;
     
-    if (pt == NULL){
+    if ((pt0 = pt_new()) == NULL){
-	clicon_err(OE_YANG, EINVAL, "pt is NULL");
+	clicon_err(OE_UNIX, errno, "pt_new");
 	goto done;
     }
     /* List of modules that should not generate autocli */
@@ -1151,48 +1142,49 @@ yang2cli(clicon_handle      h,
 	    goto done;
     }
     gt = clicon_cli_genmodel_type(h);
-    if ((cb = cbuf_new()) == NULL){
+    if ((cb0 = cbuf_new()) == NULL){
 	clicon_err(OE_XML, errno, "cbuf_new");
 	goto done;
     }
     /* Traverse YANG, loop through all modules and generate CLI */
-    yc = NULL;
+    ym = NULL;
-    while ((yc = yn_each(yn, yc)) != NULL){
+    while ((ym = yn_each(yn, ym)) != NULL){
 	/* Check if module is in exclude list */
 	for (e = 0; e < nexvec; e++){
-	    if (strcmp(yang_argument_get(yc), exvec[e]) == 0)
+	    if (strcmp(yang_argument_get(ym), exvec[e]) == 0)
 		break;
 	}
 	if (e < nexvec)
 	    continue;
-	if (yang2cli_stmt(h, yc, gt, 0, state, show_tree, cb) < 0)
+	if (yang2cli_stmt(h, ym, gt, 0, state, show_tree, cb0) < 0)
 	    goto done;
     }
     if (printgen)
-	clicon_log(LOG_NOTICE, "%s: Generated CLI spec:\n%s", __FUNCTION__, cbuf_get(cb));
+	clicon_log(LOG_NOTICE, "%s: Top-level api-spec %s:\n%s",
+		   __FUNCTION__, name0, cbuf_get(cb0));
     else
-	clicon_debug(2, "%s: buf\n%s\n", __FUNCTION__, cbuf_get(cb));
+	clicon_debug(2, "%s: Top-level api-spec %s:\n%s",
-    /* Parse the buffer using cligen parser. XXX why this?*/
+		   __FUNCTION__, name0, cbuf_get(cb0));
-    if ((globals = cvec_new(0)) == NULL)
+    /* load top-level yangspec cli syntax (that point to modules) */
+    if (cligen_parse_str(cli_cligen(h), cbuf_get(cb0), "yang2cli", pt0, NULL) < 0)
 	goto done;
-    /* load cli syntax */
-    if (cligen_parse_str(cli_cligen(h), cbuf_get(cb), "yang2cli", pt, globals) < 0)
-	goto done;
-    cvec_free(globals);
     /* Resolve the expand callback functions in the generated syntax.
      * This "should" only be GENERATE_EXPAND_XMLDB
      * handle=NULL for global namespace, this means expand callbacks must be in
      * CLICON namespace, not in a cli frontend plugin.
      */
-    if (cligen_expandv_str2fn(pt, (expandv_str2fn_t*)clixon_str2fn, NULL) < 0)     
+    if (cligen_expandv_str2fn(pt0, (expandv_str2fn_t*)clixon_str2fn, NULL) < 0)     
+	goto done;
+    /* Append cligen tree and name it */
+    if ((ph = cligen_ph_add(cli_cligen(h), name0)) == NULL)
+	goto done;
+    if (cligen_ph_parsetree_set(ph, pt0) < 0)
 	goto done;
-
     retval = 0;
   done:
     if (exvec)
 	free(exvec);
-    if (cb)
+    if (cb0)
-	cbuf_free(cb);
+	cbuf_free(cb0);
     return retval;
 }
-

apps/cli/cli_generate.h

@@ -52,7 +52,7 @@
 /*
  * Prototypes
  */
-int yang2cli(clicon_handle h, yang_stmt *yspec, 
+int yang2cli_yspec(clicon_handle h, yang_stmt *yspec, char *name0,
-	     int printgen, int state, int show_tree, parse_tree *ptnew);
+		   int printgen, int state, int show_tree);
 
 #endif  /* _CLI_GENERATE_H_ */

apps/cli/cli_handle.c

@@ -107,7 +107,12 @@ cli_handle_init(void)
 	goto done;
     }
     cligen_userhandle_set(clih, cl);
+    cligen_eval_wrap_fn_set(clih, plugin_context_check, cl);
+    /* To minimize memory, dont copy full treerefs on expand */
+    cligen_reftree_copy_set(clih, 0);
+
     cl->cl_cligen = clih;
+
     h = (clicon_handle)cl;
   done:
     return h;

apps/cli/cli_main.c

@@ -72,7 +72,7 @@
 #include "cli_handle.h"
 
 /* Command line options to be passed to getopt(3) */
-#define CLI_OPTS "hD:f:E:l:F:1a:u:d:m:qp:GLy:c:U:o:"
+#define CLI_OPTS "+hD:f:E:l:F:1a:u:d:m:qp:GLy:c:U:o:"
 
 /*! Check if there is a CLI history file and if so dump the CLI histiry to it
  * Just log if file does not exist or is not readable
@@ -242,57 +242,6 @@ cli_interactive(clicon_handle h)
     return retval;
 }
 
-/*! Generate one autocli clispec tree
- *
- * @param[in]  h        Clixon handle
- * @param[in]  name     Name of tree
- * @param[in]  gt       genmodel-type, ie HOW to generate the CLI
- * @param[in]  printgen Print CLI syntax to stderr
- * @param[in]  show_tree Is tree for show cli command (1 - yes. 0 - no)
- *
- * Generate clispec (datamodel) from YANG dataspec and add to the set of cligen trees 
- * (as a separate mode)
- * This tree is referenced from the main CLI spec (CLICON_CLISPEC_DIR) using the "tree reference"
- * syntax, ie @datamodel
- * @param[in]  h        Clixon handle
- * @param[in]  state     Set to include state syntax
- * @param[in]  printgen Print CLI syntax generated from dbspec
- * @param[in]  show_tree Is tree for show cli command
- * @retval     0        OK
- * @retval    -1        Error
- *
- * @note that yang2cli generates syntax for ALL modules under the loaded yangspec.
- */
-static int
-autocli_tree(clicon_handle      h,
-	     char              *name,
-	     int                state,
-	     int                printgen,
-	     int                show_tree)
-{
-    int           retval = -1;
-    parse_tree   *pt = NULL;  /* cli parse tree */
-    yang_stmt    *yspec;
-    pt_head      *ph;
-    
-    if ((pt = pt_new()) == NULL){
-	clicon_err(OE_UNIX, errno, "pt_new");
-	goto done;
-    }
-    yspec = clicon_dbspec_yang(h);
-    /* Generate tree (this is where the action is) */
-    if (yang2cli(h, yspec, printgen, state, show_tree, pt) < 0)
-	goto done;
-    /* Append cligen tree and name it */
-    if ((ph = cligen_ph_add(cli_cligen(h), name)) == NULL)
-	goto done;
-    if (cligen_ph_parsetree_set(ph, pt) < 0)
-	goto done;
-    retval = 0;
- done:
-    return retval;
-}
-
 /*! Generate autocli, ie if enabled, generate clispec from YANG and add to cligen parse-trees
  *
  * Generate clispec (datamodel) from YANG dataspec and add to the set of cligen trees 
@@ -312,11 +261,14 @@ autocli_start(clicon_handle h,
 {
     int           retval = -1;
     int           autocli_model = 0;
-    cbuf              *show_treename = NULL, *treename = NULL;
+    cbuf         *show_treename = NULL;
+    cbuf         *treename = NULL;
+    yang_stmt    *yspec;
 	
     /* If autocli disabled quit */
     if ((autocli_model = clicon_cli_genmodel(h)) == 0)
 	goto ok;
+    yspec = clicon_dbspec_yang(h);
     /* Get the autocli type, ie HOW the cli is generated (could be much more here) */
     /* Create show_treename cbuf */
     if ((show_treename = cbuf_new()) == NULL){
@@ -330,23 +282,22 @@ autocli_start(clicon_handle h,
     }
     /* The tree name is by default @datamodel but can be changed by option (why would one do that?) */
     cprintf(treename, "%s", clicon_cli_model_treename(h));
-    if (autocli_tree(h, cbuf_get(treename), 0, printgen, 0) < 0)
+    if (yang2cli_yspec(h, yspec, cbuf_get(treename), printgen, 0, 0) < 0)
 	goto done;
 
     /* The tree name is by default @datamodelshow but can be changed by option (why would one do that?) */
     cprintf(show_treename, "%s", clicon_cli_model_treename(h));
     cprintf(show_treename, "show");
-    if (autocli_tree(h, cbuf_get(show_treename), 0, printgen, 1) < 0)
+    if (yang2cli_yspec(h, yspec, cbuf_get(show_treename), printgen, 0, 1) < 0)
 	goto done;
 
     /* Create a tree for config+state. This tree's name has appended "state" to @datamodel
      */
     if (autocli_model > 1){
 	cprintf(treename, "state");
-	if (autocli_tree(h, cbuf_get(treename), 1, printgen, 1) < 0)
+	if (yang2cli_yspec(h, yspec, cbuf_get(treename), printgen, 1, 1) < 0)
 	    goto done;
     }
-
  ok:
     retval = 0;
  done:

apps/cli/cli_plugin.c

@@ -522,105 +522,6 @@ cli_handler_err(FILE *f)
     return 0;
 }
 
-/*! Variant of eval for context checking 
- * @see cligen_eval
- */
-static int
-clixon_cligen_eval(cligen_handle h, 
-		   cg_obj       *co, 
-		   cvec         *cvv)
-{
-    struct cg_callback *cc;
-    int                 retval = -1;
-    cvec               *argv;
-    cvec               *cvv1 = NULL; /* Modified */
-    plugin_context_t   *pc = NULL; /* Clixon-specific */
-    
-
-    /* Save matched object for plugin use */
-    if (h)
-	cligen_co_match_set(h, co);
-    /* Make a copy of var argument for modifications */
-    if ((cvv1 = cvec_dup(cvv)) == NULL)
-	goto done;
-    /* Make modifications to cvv */
-    if (cligen_expand_first_get(h) &&
-	cvec_expand_first(cvv1) < 0)
-	goto done;
-    if (cligen_exclude_keys_get(h) &&
-	cvec_exclude_keys(cvv1) < 0)
-	goto done;
-    /* Traverse callbacks */
-    for (cc = co->co_callbacks; cc; cc=cc->cc_next){
-	/* Vector cvec argument to callback */
-    	if (cc->cc_fn_vec){
-	    argv = cc->cc_cvec ? cvec_dup(cc->cc_cvec) : NULL;
-	    cligen_fn_str_set(h, cc->cc_fn_str);
-	    /* Clixon-specific */
-	    if ((pc = plugin_context_get()) == NULL) 
-		break;
-	    if ((retval = (*cc->cc_fn_vec)(
-					cligen_userhandle(h)?cligen_userhandle(h):h, 
-					cvv1,
-					argv)) < 0){
-		if (argv != NULL)
-		    cvec_free(argv);
-		cligen_fn_str_set(h, NULL);
-		goto done;
-	    }
-	    /* Clixon-specific */
-	    if (plugin_context_check(pc, "CLIgen", cc->cc_fn_str) < 0)
-		break;
-	    if (pc){
-		free(pc);
-		pc = NULL;
-	    }
-	    if (argv != NULL)
-		cvec_free(argv);
-	    cligen_fn_str_set(h, NULL);
-	}
-    }
-    retval = 0;
- done:
-    if (pc) /* Clixon-specific */
-	free(pc);
-    if (cvv1)
-	cvec_free(cvv1);
-    return retval;
-}
-
-/*! Evaluate a matched command
- * @param[in]     h         Clicon handle
- * @param[in]     cmd       The command string
- * @param[in]     match_obj
- * @param[in]     cvv
- * @retval        int  If there is a callback, the return value of the callback is returned,
- * @retval        0    otherwise
- */
-int
-clicon_eval(clicon_handle h,
-	    char         *cmd,
-	    cg_obj       *match_obj,
-	    cvec         *cvv)
-{
-    int retval = 0;
-
-    cli_output_reset();
-    if (!cligen_exiting(cli_cligen(h))) {	
-	clicon_err_reset();
-
-	if ((retval = clixon_cligen_eval(cli_cligen(h), match_obj, cvv)) < 0) {
-#if 0 /* This is removed since we get two error messages on failure.
-	 But maybe only sometime?
-	 Both a real log when clicon_err is called, and the  here again.
-	 (Before clicon_err was silent)  */
-	    cli_handler_err(stdout); 
-#endif
-	}
-    }
-    return retval;
-}
-
 /*! Given a command string, parse and if match single command, eval it.
  * Parse and evaluate the string according to
  * the syntax parse tree of the syntax mode specified by *mode.
@@ -646,16 +547,18 @@ clicon_parse(clicon_handle  h,
 {
     int               retval = -1;
     char             *modename;
-    char         *modename0;
+    int               ret;
-    int           r;
     cli_syntax_t     *stx = NULL;
     cli_syntaxmode_t *csm;
     parse_tree       *pt;     /* Orig */
     cg_obj           *match_obj = NULL;
     cvec             *cvv = NULL;
+    cg_callback      *callbacks = NULL;
     FILE             *f;
     char             *reason = NULL;
+    cligen_handle     ch;
     
+    ch = cli_cligen(h);
     if (clicon_get_logflags()&CLICON_LOG_STDOUT)
 	f = stdout;
     else
@@ -672,34 +575,18 @@ clicon_parse(clicon_handle  h,
 	}
     }
     if (csm != NULL){
-	modename0 = NULL;
+	if (cligen_ph_active_set_byname(ch, modename) < 0){
-	if ((pt = cligen_pt_active_get(cli_cligen(h))) != NULL)
-	    modename0 = pt_name_get(pt);
-	if (cligen_ph_active_set(cli_cligen(h), modename) < 0){
 	    fprintf(stderr, "No such parse-tree registered: %s\n", modename);
 	    goto done;
 	}
-	if ((pt = cligen_pt_active_get(cli_cligen(h))) == NULL){
+	if ((pt = cligen_pt_active_get(ch)) == NULL){
 	    fprintf(stderr, "No such parse-tree registered: %s\n", modename);
 	    goto done;
 	}
-#if 0 // switch after 5.4
+	if (cliread_parse(ch, cmd, pt, &match_obj, &cvv, &callbacks, result, &reason) < 0)
-	if (cliread_parse2(cli_cligen(h), cmd, pt, &match_obj, &cvv, result, &reason) < 0)
 	    goto done;
-#else
-	if ((cvv = cvec_new(0)) == NULL)
-	    goto done;;
-	if (cliread_parse(cli_cligen(h), cmd, pt, &match_obj, cvv, result, &reason) < 0)
-	    goto done;
-#endif
 	/* Debug command and result code */
 	clicon_debug(1, "%s result:%d command: \"%s\"", __FUNCTION__, *result, cmd);
-	if (*result != CG_MATCH)
-	    pt_expand_cleanup(pt); /* XXX change to pt_expand_treeref_cleanup */
-	if (modename0){
-	    cligen_ph_active_set(cli_cligen(h), modename0);
-	    modename0 = NULL;
-	}
 	switch (*result) {
 	case CG_EOF: /* eof */
 	case CG_ERROR:
@@ -713,12 +600,17 @@ clicon_parse(clicon_handle  h,
 		*modenamep = modename;
 		cli_set_syntax_mode(h, modename);
 	    }
-	    if ((r = clicon_eval(h, cmd, match_obj, cvv)) < 0)
+	    cli_output_reset();
+	    if (!cligen_exiting(ch)) {	
+		clicon_err_reset();
+		if ((ret = cligen_eval(ch, match_obj, cvv, callbacks)) < 0) 
 		    cli_handler_err(stdout);
-	    pt_expand_cleanup(pt);
+		
-	    pt_expand_treeref_cleanup(pt);
+	    }
+	    else
+		ret = 0;
 	    if (evalres)
-		*evalres = r;
+		*evalres = ret;
 	    break;
 	default:
 	    fprintf(f, "CLI syntax error: \"%s\" is ambiguous\n", cmd);
@@ -727,6 +619,8 @@ clicon_parse(clicon_handle  h,
     }
     retval = 0;
 done:
+    if (callbacks)
+	co_callbacks_free(&callbacks);
     if (reason)
 	free(reason);
     if (cvv)
@@ -850,7 +744,7 @@ clicon_cliread(clicon_handle h,
 	cli_prompt_set(h, promptstr);
 	free(promptstr);
     }
-    cligen_ph_active_set(cli_cligen(h), mode->csm_name);
+    cligen_ph_active_set_byname(cli_cligen(h), mode->csm_name);
 
     if (cliread(cli_cligen(h), stringp) < 0){
 	clicon_err(OE_FATAL, errno, "CLIgen");

apps/cli/cli_plugin.h

@@ -45,7 +45,10 @@
 /* clicon generic callback pointer */
 typedef void (clicon_callback_t)(clicon_handle h);
 
-/* List of syntax modes */
+/* List of syntax modes 
+ * XXX: syntax modes seem not needed, could be replaced by existing (new) cligen structures, such
+ * as pt_head and others. But code is arcane and difficult to modify.
+ */
 typedef struct {
     qelem_t     csm_qelem;     /* List header */
     char       *csm_name;      /* Syntax mode name */
@@ -54,7 +57,8 @@ typedef struct {
     parse_tree *csm_pt;        /* CLIgen parse tree */
 } cli_syntaxmode_t;
 
-/* Plugin group object. Just a single object, not list. part of cli_handle */
+/* Plugin group object. Just a single object, not list. part of cli_handle 
+ */
 typedef struct  {
     int stx_nmodes;                              /* Number of syntax modes */
     cli_syntaxmode_t *stx_active_mode;       /* Current active syntax mode */
@@ -64,8 +68,6 @@ typedef struct  {
 
 void *clixon_str2fn(char *name, void *handle, char **error);
 
-int clicon_eval(clicon_handle h, char *cmd, cg_obj *match_obj, cvec *vr);
-
 int clicon_parse(clicon_handle h, char *cmd, char **mode, cligen_result *result, int *evalres);
 
 int clicon_cliread(clicon_handle h, char **stringp);

lib/clixon/clixon_plugin.h

@@ -337,7 +337,6 @@ struct clixon_plugin_api{
 #define ca_trans_abort    u.cau_backend.cb_trans_abort
 #define ca_datastore_upgrade  u.cau_backend.cb_datastore_upgrade
 
-
 /*
  * Macros
  */
@@ -379,8 +378,7 @@ int clixon_plugins_load(clicon_handle h, const char *function, const char *dir,
 
 int clixon_pseudo_plugin(clicon_handle h, const char *name, clixon_plugin_t **cpp);
 
-plugin_context_t * plugin_context_get(void);
+int plugin_context_check(clicon_handle h, void **wh, const char *name, const char *fn);
-int plugin_context_check(plugin_context_t *pc, const char *name, const char *fn);
 
 int clixon_plugin_start_one(clixon_plugin_t *cp, clicon_handle h);
 int clixon_plugin_start_all(clicon_handle h);

lib/clixon/clixon_yang.h

@@ -221,6 +221,10 @@ int        yang_filename_set(yang_stmt *ys, const char *filename);
 int        yang_linenum_get(yang_stmt *ys);
 int        yang_linenum_set(yang_stmt *ys, int linenum);
 
+/* Stats */
+int       yang_stats_global(uint64_t *nr);
+int       yang_stats(yang_stmt *y, uint64_t *nrp, size_t *szp);
+
 /* Other functions */
 yang_stmt *yspec_new(void);
 yang_stmt *ys_new(enum rfc_6020 keyw);

lib/src/clixon_plugin.c

@@ -63,6 +63,7 @@
 #include "clixon_file.h"
 #include "clixon_handle.h"
 #include "clixon_yang.h"
+#include "clixon_options.h"
 #include "clixon_xml.h"
 #include "clixon_xml_nsctx.h"
 #include "clixon_yang_module.h"
@@ -342,7 +343,7 @@ plugin_load_one(clicon_handle   h,
     clixon_plugin_t   *cp = NULL;
     char              *name;
     char              *p;
-    plugin_context_t  *pc = NULL;
+    void              *wh = NULL;
 
     clicon_debug(1, "%s file:%s function:%s", __FUNCTION__, file, function);
     dlerror();    /* Clear any existing error */
@@ -361,9 +362,9 @@ plugin_load_one(clicon_handle   h,
 	goto done;
     }
     clicon_err_reset();
-    if ((pc = plugin_context_get()) < 0)
+    wh = NULL;
+    if (plugin_context_check(h, &wh, file, __FUNCTION__) < 0)
 	goto done;
-
     if ((api = initfn(h)) == NULL) {
 	if (!clicon_errno){ 	/* if clicon_err() is not called then log and continue */
 	    clicon_log(LOG_DEBUG, "Warning: failed to initiate %s", strrchr(file,'/')?strchr(file, '/'):file);
@@ -375,7 +376,7 @@ plugin_load_one(clicon_handle   h,
 	    goto done;
 	}
     }
-    if (plugin_context_check(pc, file, __FUNCTION__) < 0)
+    if (plugin_context_check(h, &wh, file, __FUNCTION__) < 0)
 	goto done;
 
     /* Note: sizeof clixon_plugin_api which is largest of clixon_plugin_api:s */
@@ -401,8 +402,8 @@ plugin_load_one(clicon_handle   h,
     retval = 1;
  done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
-    if (pc)
+    if (wh != NULL)
-	free(pc);
+	free(wh);
     if (retval != 1 && handle)
 	dlclose(handle);
     if (cp)
@@ -510,7 +511,7 @@ done:
  * @retval     NULL    Error
  * @see plugin_context_check
  * */
-plugin_context_t *
+static void *
 plugin_context_get(void)
 {
     int                    i;
@@ -521,7 +522,6 @@ plugin_context_get(void)
 	goto done;
     }
     memset(pc, 0, sizeof(*pc));
-
     if (sigprocmask(0, NULL, &pc->pc_sigset) < 0){
 	clicon_err(OE_UNIX, errno, "sigprocmask");
 	goto done;
@@ -553,30 +553,53 @@ plugin_context_get(void)
 
 /*! Given an existing, old plugin context, check if anything has changed
  *
- * Make a new check and compare with the old (procided as in-parameter).
+ * Called twice:
+ * 1) Make a check of resources
+ * 2) Make a new check and compare with the old check, return 1 on success, 0 on fail
  * Log if there is a difference at loglevel WARNING.
  * You can modify the code to also fail with assert if you want early fail.
+ * Controlled by option 
  *
- * @param[in,out] oldpc  Old plugin context
+ * @param[in]     h      Clixon handle
+ * @param[in,out] wh     Either: NULL for init, will be assigned, OR previous handle (will be freed)
  * @param[in]     name   Name of plugin for logging. Can be other name, context dependent
  * @param[in]     fn     Typically name of callback, or caller function
  * @retval       -1      Error
  * @retval        0      Fail, log on syslog using LOG_WARNING
  * @retval        1      OK
+ * @note Only logs error, does not generate error
  * @note name and fn are context dependent, since the env of callback calls are very different
  * @see plugin_context_get
+ * @see CLICON_PLUGIN_CALLBACK_CHECK  Enable to activate these checks
  */
 int
-plugin_context_check(plugin_context_t *oldpc0,
+plugin_context_check(clicon_handle h,
+		     void        **wh,
 		     const char   *name,
 		     const char   *fn)
 {
     int                    retval = -1;
     int                    failed = 0;
     int                    i;
-    struct plugin_context *oldpc = oldpc0;
+    struct plugin_context *oldpc;
     struct plugin_context *newpc = NULL;
 
+    if (h == NULL){
+	errno = EINVAL;
+	return -1;
+    }
+    /* Check if plugion checks are enabled */
+    if (!clicon_option_bool(h, "CLICON_PLUGIN_CALLBACK_CHECK"))
+	return 1;
+    if (wh == NULL){
+	errno = EINVAL;
+	return -1;
+    }
+    if (*wh == NULL){
+	*wh = plugin_context_get();
+	return 1;
+    }
+    oldpc = (struct plugin_context *)*wh;
     if ((newpc = plugin_context_get()) == NULL)
 	goto done;
     if (oldpc->pc_termios.c_iflag != newpc->pc_termios.c_iflag){
@@ -644,11 +667,14 @@ plugin_context_check(plugin_context_t *oldpc0,
     }
     if (failed)
 	goto fail;
-
     retval = 1; /* OK */
  done:
     if (newpc)
 	free(newpc);
+    if (oldpc)
+	free(oldpc);
+    if (wh && *wh)
+	*wh = NULL;
     return retval;
  fail:
     retval = 0;
@@ -667,10 +693,11 @@ clixon_plugin_start_one(clixon_plugin_t *cp,
 {
     int         retval = -1;
     plgstart_t *fn;          /* Plugin start */
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
 
     if ((fn = cp->cp_api.ca_start) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h) < 0) {
 	    if (clicon_errno < 0) 
@@ -678,13 +705,11 @@ clixon_plugin_start_one(clixon_plugin_t *cp,
 			   __FUNCTION__, cp->cp_name);
 	    goto done;
 	}
-	if (plugin_context_check(pc, cp->cp_name, __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -722,10 +747,11 @@ clixon_plugin_exit_one(clixon_plugin_t *cp,
     int         retval = -1;
     char       *error;
     plgexit_t  *fn;
-    plugin_context_t *pc = NULL;
+    void       *wh = NULL;
 	
     if ((fn = cp->cp_api.ca_exit) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h) < 0) {
 	    if (clicon_errno < 0) 
@@ -733,7 +759,7 @@ clixon_plugin_exit_one(clixon_plugin_t *cp,
 			   __FUNCTION__, cp->cp_name);
 	    goto done;
 	}
-	if (plugin_context_check(pc, cp->cp_name, __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
 	if (dlclose(cp->cp_handle) != 0) {
 	    error = (char*)dlerror();
@@ -742,8 +768,6 @@ clixon_plugin_exit_one(clixon_plugin_t *cp,
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -794,11 +818,12 @@ clixon_plugin_auth_one(clixon_plugin_t     *cp,
 {
     int        retval = -1; 
     plgauth_t *fn;          /* Plugin auth */
-    plugin_context_t *pc = NULL;
+    void      *wh = NULL;
 
     clicon_debug(1, "%s", __FUNCTION__);
     if ((fn = cp->cp_api.ca_auth) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
 	if ((retval = fn(h, req, auth_type, authp)) < 0) {
 	    if (clicon_errno < 0) 
@@ -806,14 +831,12 @@ clixon_plugin_auth_one(clixon_plugin_t     *cp,
 			   __FUNCTION__, cp->cp_name);
 	    goto done;
 	}
-	if (plugin_context_check(pc, cp->cp_name, __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
     }
     else
 	retval = 0; /* Ignored / no callback */
  done:
-    if (pc)
-	free(pc);
     clicon_debug(1, "%s retval:%d auth:%s", __FUNCTION__, retval, *authp);
     return retval;
 }
@@ -879,10 +902,11 @@ clixon_plugin_extension_one(clixon_plugin_t *cp,
 {
     int             retval = 1;
     plgextension_t *fn;          /* Plugin extension fn */
-    plugin_context_t *pc = NULL;
+    void           *wh = NULL;
     
     if ((fn = cp->cp_api.ca_extension) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, yext, ys) < 0) {
 	    if (clicon_errno < 0) 
@@ -890,13 +914,11 @@ clixon_plugin_extension_one(clixon_plugin_t *cp,
 			   __FUNCTION__, cp->cp_name);
 	    goto done;
 	}
-	if (plugin_context_check(pc, cp->cp_name, __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -950,10 +972,11 @@ clixon_plugin_datastore_upgrade_one(clixon_plugin_t   *cp,
 {
     int                  retval = -1;
     datastore_upgrade_t *fn;
-    plugin_context_t    *pc = NULL;
+    void                *wh = NULL;
     
     if ((fn = cp->cp_api.ca_datastore_upgrade) != NULL){
-	if ((pc = plugin_context_get()) == NULL)
+	wh = NULL;
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
 	if (fn(h, db, xt, msd) < 0) {
 	    if (clicon_errno < 0) 
@@ -961,13 +984,11 @@ clixon_plugin_datastore_upgrade_one(clixon_plugin_t   *cp,
 			   __FUNCTION__, cp->cp_name);
 	    goto done;
 	}
-	if (plugin_context_check(pc, cp->cp_name, __FUNCTION__) < 0)
+	if (plugin_context_check(h, &wh, cp->cp_name, __FUNCTION__) < 0)
 	    goto done;
     }
     retval = 0;
  done:
-    if (pc)
-	free(pc);
     return retval;
 }
 
@@ -1121,7 +1142,7 @@ rpc_callback_call(clicon_handle h,
     char                 *ns;
     int                   nr = 0; /* How many callbacks */
     plugin_module_struct *ms = plugin_module_struct_get(h);
-    plugin_context_t     *pc = NULL;
+    void                 *wh = NULL;
     int                   ret;
 
     if (ms == NULL){
@@ -1136,19 +1157,18 @@ rpc_callback_call(clicon_handle h,
 	    if (strcmp(rc->rc_name, name) == 0 &&
 		ns && rc->rc_namespace &&
 		strcmp(rc->rc_namespace, ns) == 0){
-		if ((pc = plugin_context_get()) == NULL)
+		wh = NULL;
+		if (plugin_context_check(h, &wh, rc->rc_name, __FUNCTION__) < 0)
 		    goto done;
 		if (rc->rc_callback(h, xe, cbret, arg, rc->rc_arg) < 0){
 		    clicon_debug(1, "%s Error in: %s", __FUNCTION__, rc->rc_name);
+		    if (plugin_context_check(h, &wh, rc->rc_name, __FUNCTION__) < 0)
+			goto done;
 		    goto done;
 		}
 		nr++;
-		if (plugin_context_check(pc, rc->rc_name, __FUNCTION__) < 0)
+		if (plugin_context_check(h, &wh, rc->rc_name, __FUNCTION__) < 0)
 		    goto done;
-		if (pc){
-		    free(pc);
-		    pc = NULL;
-		}
 	    }
 	    rc = NEXTQ(rpc_callback_t *, rc);
 	} while (rc != ms->ms_rpc_callbacks);
@@ -1162,8 +1182,6 @@ rpc_callback_call(clicon_handle h,
     retval = 1; /* 0: none found, >0 nr of handlers called */
  done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
-    if (pc)
-	free(pc);
     return retval;
  fail:
     retval = 0;

lib/src/clixon_validate.c

@@ -1505,6 +1505,7 @@ rpc_reply_check(clicon_handle h,
     if ((ret = xml_bind_yang_rpc_reply(x, rpcname, yspec, &xret)) < 0)
 	goto done;
     if (ret == 0){
+	clicon_debug(1, "%s failure when validating:%s", __FUNCTION__, cbuf_get(cbret));
 	cbuf_reset(cbret);
 	if (clicon_xml2cbuf(cbret, xret, 0, 0, -1) < 0)
 	    goto done;
@@ -1513,6 +1514,7 @@ rpc_reply_check(clicon_handle h,
     if ((ret = xml_yang_validate_rpc_reply(h, x, &xret)) < 0)
 	goto done;
     if (ret == 0){
+	clicon_debug(1, "%s failure when validating:%s", __FUNCTION__, cbuf_get(cbret));
 	cbuf_reset(cbret);
 	if (clicon_xml2cbuf(cbret, xret, 0, 0, -1) < 0)
 	    goto done;

lib/src/clixon_xml.c

@@ -232,19 +232,20 @@ xml_type2str(enum cxobj_type type)
 }
 
 /* Stats */
-uint64_t _stats_nr = 0;
+static uint64_t _stats_xml_nr = 0;
 
 /*! Get global statistics about XML objects
+ *
+ * @param[out]  nr  Number of existing XML objects (created - freed)
  */
 int
 xml_stats_global(uint64_t *nr)
 {
     if (nr)
-	*nr = _stats_nr;
+	*nr = _stats_xml_nr;
     return 0;
 }
 
-
 /*! Return the alloced memory of a single XML obj 
  * @param[in]   x    XML object
  * @param[out]  szp  Size of this XML obj
@@ -257,7 +258,6 @@ xml_stats_one(cxobj    *x,
 {
     size_t sz = 0;
 
-
     if (x->x_name)
 	sz += strlen(x->x_name) + 1;
     if (x->x_prefix)
@@ -292,51 +292,12 @@ xml_stats_one(cxobj    *x,
     }
     if (szp)
 	*szp = sz;
-    clicon_debug(1, "%s %zu", __FUNCTION__, sz);
     return 0;
 }
 
-#if 0
-/*! Print memory stats of a single object
- */
-static int
-xml_print_stats_one(FILE   *f,
-		    cxobj  *x)
-{
-    size_t sz = 0;
-
-    xml_stats_one(x, &sz);
-    fprintf(f, "%s:\n", xml_name(x));
-    fprintf(f, "  sum: \t\t%u\n", (unsigned int)sz);
-    if (xml_type(x) == CX_ELMNT)
-	fprintf(f, "  base struct: \t%u\n", (unsigned int)sizeof(struct xml));
-    else
-	fprintf(f, "  base struct: \t%u\n", (unsigned int)sizeof(struct xmlbody));
-    if (x->x_name)
-	fprintf(f, "  name: \t%u\n", (unsigned int)strlen(x->x_name) + 1);
-    if (x->x_prefix)
-	fprintf(f, "  prefix: \t%u\n", (unsigned int)strlen(x->x_prefix) + 1);
-    if (xml_type(x) == CX_ELMNT){
-	if (x->x_childvec_max)
-	    fprintf(f, "  childvec: \t%u\n", (unsigned int)(x->x_childvec_max*sizeof(struct xml*)));
-	if (x->x_ns_cache)
-	    fprintf(f, "  ns-cache: \t%u\n", (unsigned int)cvec_size(x->x_ns_cache));
-	if (x->x_cv)
-	    fprintf(f, "  value-cv: \t%u\n", (unsigned int)cv_size(x->x_cv));
-	if (x->x_search_index)
-	    fprintf(f, "  search-index: \t%u\n",
-		    (unsigned int)(strlen(x->x_search_index->si_name) + 1 + clixon_xvec_len(x->x_search_index->si_xvec)*sizeof(struct cxobj*)));
-    }
-    else{
-	if (x->x_value_cb)
-	    fprintf(f, "  value-cb: \t%u\n", cbuf_buflen(x->x_value_cb));
-    }
-    return 0;
-}
-#endif
-
 /*! Return statistics of an XML tree recursively
  * @param[in]   xt   XML object
+ * @param[out]  nrp  Number of XML obj recursively
  * @param[out]  szp  Size of this XML obj recursively
  * @retval      0    OK
  * @retval     -1    Error
@@ -354,7 +315,6 @@ xml_stats(cxobj    *xt,
 	clicon_err(OE_XML, EINVAL, "xml node is NULL");
 	goto done;
     }
-    //    xml_print_stats_one(stderr, xt);
     *nrp += 1;
     xml_stats_one(xt, &sz);
     if (szp)
@@ -366,7 +326,6 @@ xml_stats(cxobj    *xt,
 	if (szp)
 	    *szp += sz;
     }
-    clicon_debug(1, "%s %zu", __FUNCTION__, *szp);
     retval = 0;
  done:
     return retval;
@@ -1131,7 +1090,7 @@ xml_new(char           *name,
 	    return NULL;
 	x->_x_i = xml_child_nr(xp)-1;
     }
-    _stats_nr++;
+    _stats_xml_nr++;
     return x;
 }
 
@@ -1901,7 +1860,7 @@ xml_free(cxobj *x)
 	break;
     }
     free(x);
-    _stats_nr--;
+    _stats_xml_nr--;
     return 0;
 }
 

lib/src/clixon_yang.c

@@ -473,6 +473,100 @@ yang_linenum_set(yang_stmt *ys,
 
 /* End access functions */
 
+/* Stats */
+static uint64_t _stats_yang_nr = 0;
+
+/*! Get global statistics about YANG statements: created - freed
+ *
+ * @param[out]  nr  Number of existing YANG objects (created - freed)
+ */
+int
+yang_stats_global(uint64_t *nr)
+{
+    if (nr)
+	*nr = _stats_yang_nr;
+    return 0;
+}
+
+/*! Return the alloced memory of a single YANG obj 
+ * @param[in]   y    YANG object
+ * @param[out]  szp  Size of this YANG obj
+ * @retval      0    OK
+ * (baseline: )
+ */
+static int
+yang_stats_one(yang_stmt *y,
+	       size_t    *szp)
+{
+    size_t           sz = 0;
+    yang_type_cache *yc;
+    
+    sz += sizeof(struct yang_stmt);
+    sz += y->ys_len*sizeof(struct yang_stmt*);
+    if (y->ys_argument)
+	sz += strlen(y->ys_argument) + 1;
+    if (y->ys_cv)
+	sz += cv_size(y->ys_cv);
+    if (y->ys_cvec)
+	sz += cvec_size(y->ys_cvec);
+    if ((yc = y->ys_typecache) != NULL){
+    	sz += sizeof(struct yang_type_cache);
+	if (yc->yc_cvv)
+	    sz += cvec_size(yc->yc_cvv);
+	if (yc->yc_patterns)
+	    sz += cvec_size(yc->yc_patterns);
+	if (yc->yc_regexps)
+	    sz += cvec_size(yc->yc_regexps);
+    }
+    if (y->ys_when_xpath)
+	sz += strlen(y->ys_when_xpath) + 1;
+    if (y->ys_when_nsc)
+	sz += cvec_size(y->ys_when_nsc);
+    if (y->ys_filename)
+	sz += strlen(y->ys_filename) + 1;
+    if (szp)
+	*szp = sz;
+    return 0;
+}
+
+/*! Return statistics of an YANG-stmt tree recursively
+ * @param[in]   yt   YANG object
+ * @param[out]  nrp  Number of YANG objects recursively
+ * @param[out]  szp  Size of this YANG stmt recursively
+ * @retval      0    OK
+ * @retval     -1    Error
+ */
+int
+yang_stats(yang_stmt *yt,
+	   uint64_t  *nrp,
+	   size_t    *szp)
+{
+    int        retval = -1;
+    size_t     sz = 0;
+    yang_stmt *ys;
+
+    if (yt == NULL){
+	clicon_err(OE_XML, EINVAL, "yang spec is NULL");
+	goto done;
+    }
+    *nrp += 1;
+    yang_stats_one(yt, &sz);
+    if (szp)
+	*szp += sz;
+    ys = NULL;
+    while ((ys = yn_each(yt, ys)) != NULL) {
+	sz = 0;
+	yang_stats(ys, nrp, &sz);
+	if (szp)
+	    *szp += sz;
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
+/* stats end */
+
 /*! Create new yang specification
  * @retval  yspec    Free with ys_free() 
  * @retval  NULL     Error
@@ -488,6 +582,7 @@ yspec_new(void)
     }
     memset(yspec, 0, sizeof(*yspec));
     yspec->ys_keyword = Y_SPEC;
+    _stats_yang_nr++;
     return yspec;
 }
 
@@ -514,6 +609,7 @@ ys_new(enum rfc_6020 keyw)
 	return NULL;
     }
     yang_cvec_set(ys, cvv);
+    _stats_yang_nr++;
     return ys;
 }
 
@@ -555,8 +651,10 @@ ys_free1(yang_stmt *ys,
 	free(ys->ys_stmt);
     if (ys->ys_filename)
 	free(ys->ys_filename);
-    if (self)
+    if (self){
 	free(ys);
+	_stats_yang_nr++;
+    }
     return 0;
 }
 

test/config.sh.in

@@ -65,8 +65,8 @@ CLIXON_VERSION=@CLIXON_VERSION@
 DATASTORE_TOP="config"
 
 # clixon yang revisions occuring in tests
-CLIXON_LIB_REV="2021-03-08"
+CLIXON_LIB_REV="2021-11-11"
-CLIXON_CONFIG_REV="2021-05-20"
+CLIXON_CONFIG_REV="2021-11-11"
 CLIXON_RESTCONF_REV="2021-05-20"
 CLIXON_EXAMPLE_REV="2020-12-01"
 

test/lib.sh

@@ -270,6 +270,7 @@ fi
 # arg1: expected
 # arg2: errmsg[optional]
 # Assumes: $dir and $expect are set
+# see err1
 function err(){
   echo -e "\e[31m\nError in Test$testnr [$testname]:"
   if [ $# -gt 0 ]; then 

test/test_cli_apipath.sh

@@ -112,8 +112,6 @@ expectpart "$($clixon_cli -1 -f $cfg err x)" 255 "Config error: api-path syntax
 new "err x a"
 expectpart "$($clixon_cli -1 -f $cfg err x a 99)" 255 "Config error: api-path syntax error \"/example:x/m1=%s\": rpc malformed-message List key m1 length mismatch : Invalid argument"
 
-endtest
-
 new "Kill backend"
 # Check if premature kill
 pid=$(pgrep -u root -f clixon_backend)
@@ -123,4 +121,6 @@ fi
 # kill backend
 stop_backend -f $cfg
 
+endtest
+
 rm -rf $dir

test/test_openconfig_network_instance.sh

@@ -107,6 +107,7 @@ expectpart "$(cat $dir/config.cli)" 0 "set network-instances network-instance de
 
 new "load saved cli config"
 expectpart "$(cat $dir/config.cli | $clixon_cli -D $DBG -f $cfg 2>&1 > /dev/null)" 0 "^$"
+#time cat $dir/config.cli | $clixon_cli -D $DBG -f $cfg
 
 if [ $BE -ne 0 ]; then
     new "Kill backend"

test/test_perf_mem.sh

@@ -92,7 +92,7 @@ function testrun(){
 
     new "netconf get stats"
     res=$(echo "$DEFAULTHELLO<rpc $DEFAULTNS><stats $LIBNS/></rpc>]]>]]>" | $clixon_netconf -qf $cfg)
-    echo "res:$res"
+#    echo "res:$res"
     err0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/rpc-error")
     err=${err0#"nodeset:"}
     if [ -n "$err" ]; then
@@ -104,7 +104,7 @@ function testrun(){
     echo "   objects: $objects"
 
 #
-    if [ -f /proc/$pid/statm ]; then     # This ony works on Linux 
+    if [ -f /proc/$pid/statm ]; then     # This only works on Linux 
 #	cat /proc/$pid/statm
 	echo -n "   /proc/$pid/statm: "
 	cat /proc/$pid/statm|awk '{print $1*4/1000 "M"}'
@@ -121,7 +121,15 @@ function testrun(){
 	echo -n "   mem: "
 	echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
     done
-
+    for mod in clixon-config; do
+	echo "$mod"
+	resmod0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/module[name=\"$mod\"]")
+	resmod=${resmod0#"nodeset:0:"}
+	echo -n "   objects: "
+	echo $resmod | $clixon_util_xpath -p "module/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
+	echo -n "   mem: "
+	echo $resmod | $clixon_util_xpath -p "module/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
+    done
     if [ $BE -ne 0 ]; then
 	new "Kill backend"
 	# Check if premature kill

test/test_yang_default.sh

@@ -88,9 +88,19 @@ if [ $BE -ne 0 ]; then     # Bring your own backend
     start_backend -s $db -f $cfg
 fi
 
+new "wait backend"
+wait_backend
+
 # permission kludges
+new "chmod datastores"
 sudo chmod 666 $dir/running_db
+if [ $? -ne 0 ]; then
+    err1 "chmod $dir/running_db"
+fi
 sudo chmod 666 $dir/startup_db
+if [ $? -ne 0 ]; then
+    err1 "chmod $dir/startup_db"
+fi
 
 new "Checking startup unchanged"
 ret=$(diff $dir/startup_db <(echo "<${DATASTORE_TOP}>$XML</${DATASTORE_TOP}>"))

util/clixon_util_validate.c

@@ -134,6 +134,7 @@ main(int    argc,
 	    break;
 	}
     clicon_debug_init(dbg, NULL);
+    yang_init(h);
     /* Find and read configfile */
     if (clicon_options_main(h) < 0)
 	goto done;

util/clixon_util_xml.c

@@ -176,6 +176,7 @@ main(int    argc,
 	goto done;
     if (clicon_conf_xml_set(h, xcfg) < 0)
 	goto done;
+
     optind = 1;
     opterr = 0;
     while ((c = getopt(argc, argv, UTIL_XML_OPTS)) != -1)
@@ -241,6 +242,7 @@ main(int    argc,
     }
     clicon_log_init(__FILE__, dbg?LOG_DEBUG:LOG_INFO, logdst);
     clicon_debug_init(dbg, NULL);
+    yang_init(h);
     
     /* 1. Parse yang */
     if (yang_file_dir){

util/clixon_util_xpath.c

@@ -228,6 +228,7 @@ main(int    argc,
     clicon_log_init("xpath", dbg?LOG_DEBUG:LOG_INFO, logdst);
     
     clicon_debug_init(dbg, NULL);
+    yang_init(h);
     
     /* Parse yang */
     if (yang_file_dir){

yang/clixon/Makefile.in

@@ -43,7 +43,7 @@ YANG_INSTALLDIR   = @YANG_INSTALLDIR@
 
 YANGSPECS	 = clixon-config@2021-07-11.yang   # 5.3
 YANGSPECS	 = clixon-config@2021-11-11.yang   # 5.4
-YANGSPECS	+= clixon-lib@2021-03-08.yang      # 5.1
+YANGSPECS	+= clixon-lib@2021-11-11.yang      # 5.4
 YANGSPECS	+= clixon-rfc5277@2008-07-01.yang
 YANGSPECS	+= clixon-xml-changelog@2019-03-21.yang
 YANGSPECS	+= clixon-restconf@2021-05-20.yang # 5.2

yang/clixon/clixon-config@2021-11-11.yang

@@ -45,7 +45,9 @@ module clixon-config {
 
     revision 2021-11-11 {
 	description
-	    "Modified options:
+	    "Added option:
+                    CLICON_PLUGIN_CALLBACK_CHECK
+	     Modified options:
                     CLICON_CLI_GENMODEL_TYPE: added OC_COMPRESS enum
                     CLICON_YANG_DIR: recursive search
              Released in Clixon 5.4";
@@ -930,6 +932,24 @@ module clixon-config {
                  lists, therefore it is recommended to enable it during development and debugging
                  but disable it in production, until this has been resolved.";
 	}
+	leaf CLICON_PLUGIN_CALLBACK_CHECK {
+	    type boolean;
+	    default false;
+	    description
+		"If enabled, make a check of resources before and after each plugin callback code
+                 to check if the plugin violated resources.
+                 This is primarily intended for development and debugging but may also be enabled 
+                 in a running system.
+                 If enabled, errors will be logged to syslog as WARNINGs.
+                 In case you want early detection and crash, you can uncomment assert statements and
+                 recompile.
+                 The checks are currently made by plugin_context_check() and include:
+                 - termios settings
+                 - signal vectors
+                 The checks will be made for all callbacks as defined in struct clixon_plugin_api
+                 as well as the CLIgen callbacks.
+                 See https://clixon-docs.readthedocs.io/en/latest/backend.html#plugin-callback-guidelines";
+	}
 	leaf CLICON_NAMESPACE_NETCONF_DEFAULT {
 	    type boolean;
 	    default false;

yang/clixon/clixon-lib@2021-11-11.yang

@@ -0,0 +1,262 @@
+module clixon-lib {
+    yang-version 1.1;
+    namespace "http://clicon.org/lib";
+    prefix cl;
+
+    import ietf-yang-types {
+	prefix yang;
+    }    
+    organization
+	"Clicon / Clixon";
+
+    contact
+	"Olof Hagsand <olof@hagsand.se>";
+
+    description
+      "Clixon Netconf extensions for communication between clients and backend.
+      
+       ***** BEGIN LICENSE BLOCK *****
+       Copyright (C) 2009-2019 Olof Hagsand
+       Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+       
+       This file is part of CLIXON
+
+       Licensed under the Apache License, Version 2.0 (the \"License\");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+            http://www.apache.org/licenses/LICENSE-2.0
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an \"AS IS\" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
+
+       Alternatively, the contents of this file may be used under the terms of
+       the GNU General Public License Version 3 or later (the \"GPL\"),
+       in which case the provisions of the GPL are applicable instead
+       of those above. If you wish to allow use of your version of this file only
+       under the terms of the GPL, and not to allow others to
+       use your version of this file under the terms of Apache License version 2, 
+       indicate your decision by deleting the provisions above and replace them with
+       the notice and other provisions required by the GPL. If you do not delete
+       the provisions above, a recipient may use your version of this file under
+       the terms of any one of the Apache License version 2 or the GPL.
+
+       ***** END LICENSE BLOCK *****";
+
+    revision 2021-11-11 {
+	description
+	    "Changed: RPC stats extended with YANG stats";
+    }
+    revision 2021-03-08 {
+	description
+	    "Changed: RPC process-control output to choice dependent on operation";
+    }
+    revision 2020-12-30 {
+	description
+	    "Changed: RPC process-control output parameter status to pid";
+    }
+    revision 2020-12-08 {
+	description
+	    "Added: autocli-op extension.
+                    rpc process-control for process/daemon management
+             Released in clixon 4.9";
+    }
+    revision 2020-04-23 {
+	description
+	    "Added: stats RPC for clixon XML and memory statistics.
+             Added: restart-plugin RPC for restarting individual plugins without restarting backend.";
+    }
+    revision 2019-08-13 {
+	description
+	    "No changes (reverted change)";
+    }
+    revision 2019-06-05 {
+	description
+	    "ping rpc added for liveness";
+    }
+    revision 2019-01-02 {
+	description
+	    "Released in Clixon 3.9";
+    }
+    typedef service-operation {
+        type enumeration {
+            enum start {
+                description
+                    "Start if not already running";
+            }
+            enum stop {
+                description
+                    "Stop if running";
+            }
+            enum restart {
+                description
+                    "Stop if running, then start";
+            }
+            enum status {
+                description
+                    "Check status";
+            }
+        }
+        description
+            "Common operations that can be performed on a service";
+    }
+    extension autocli-op {
+      description 
+        "Takes an argument an operation defing how to modify the clispec at 
+         this point in the YANG tree for the automated generated CLI.
+         Note that this extension is only used in clixon_cli.
+         Operations is expected to be extended, but the following operations are defined:
+         - hide  		 				  This command is active but not shown by ? or TAB (meaning, it hides the auto-completion of commands)
+		 - hide-database 				  This command hides the database
+         - hide-database-auto-completion  This command hides the database and the auto completion (meaning, this command acts as both commands above)";
+      argument cliop;
+   }
+   rpc debug {
+	description "Set debug level of backend.";
+	input {
+	    leaf level {
+		type uint32;
+	    }
+	}
+    }
+    rpc ping {
+        description "Check aliveness of backend daemon.";
+    }
+    rpc stats {
+        description "Clixon XML statistics.";
+	output {
+	    container global{
+		description
+		    "Clixon global statistics. 
+                     These are global counters incremented by new() and decreased by free() calls.
+                     This number is higher than the sum of all datastore/module residing objects, since
+                     objects may be used for other purposes than datastore/modules";
+		leaf xmlnr{
+		    description
+			"Number of existing XML objects: number of residing xml/json objects
+                         in the internal 'cxobj' representation.";
+		    type uint64;
+		}
+		leaf yangnr{
+		    description
+			"Number of resident YANG objects. ";
+		    type uint64;
+		}
+	    }
+	    list datastore{
+		description "Per datastore statistics for cxobj";
+		key "name";
+		leaf name{
+		    description "Name of datastore (eg running).";
+		    type string;
+		}
+		leaf nr{
+		    description "Number of XML objects. That is number of residing xml/json objects
+                             in the internal 'cxobj' representation.";
+		    type uint64;
+		}
+		leaf size{
+		    description "Size in bytes of internal datastore cache of datastore tree.";
+		    type uint64;
+		}
+	    }
+	    list module{
+		description "Per YANG module statistics";
+		key "name";
+		leaf name{
+		    description "Name of YANG module.";
+		    type string;
+		}
+		leaf nr{
+		    description
+			"Number of YANG objects. That is number of residing YANG objects";
+		    type uint64;
+		}
+		leaf size{
+		    description
+			"Size in bytes of internal YANG object representation.";
+		    type uint64;
+		}
+	    }
+	}
+    }
+    rpc restart-plugin {
+	description "Restart specific backend plugins.";
+	input {
+	    leaf-list plugin {
+		description "Name of plugin to restart";
+		type string;
+	    }
+	}
+    }
+
+    rpc process-control {
+	description
+	    "Control a specific process or daemon: start/stop, etc.
+             This is for direct managing of a process by the backend. 
+             Alternatively one can manage a daemon via systemd, containerd, kubernetes, etc.";
+	input {
+	    leaf name {
+		description "Name of process";
+		type string;
+		mandatory true;
+	    }
+	    leaf operation {
+		type service-operation;
+		mandatory true;
+		description
+		    "One of the strings 'start', 'stop', 'restart', or 'status'.";
+	    }
+	}
+	output {
+	    choice result {
+		case status {
+		    description
+			"Output from status rpc";
+		    leaf active {
+			description
+			    "True if process is running, false if not. 
+                             More specifically, there is a process-id and it exists (in Linux: kill(pid,0).
+                             Note that this is actual state and status is administrative state,
+                             which means that changing the administrative state, eg stopped->running
+                             may not immediately switch active to true.";
+			type boolean;
+		    }
+		    leaf description {
+			type string;
+			description "Description of process. This is a static string";
+		    }
+		    leaf command {
+			type string;
+			description "Start command with arguments";
+		    }
+		    leaf status {
+			description
+			    "Administrative status (except on external kill where it enters stopped
+                             directly from running):
+                             stopped: pid=0,   No process running
+                             running: pid set, Process started and believed to be running
+                             exiting: pid set, Process is killed by parent but not waited for";
+			type string;
+		    }
+		    leaf starttime {
+			description "Time of starting process UTC";
+			type yang:date-and-time;
+		    }
+		    leaf pid {
+			description "Process-id of main running process (if active)";
+			type uint32;
+		    }
+		}
+		case other {
+		    description
+			"Output from start/stop/restart rpc";
+		    leaf ok {
+			type empty;
+		    }
+		}
+	    }
+	}
+    }
+}