Refactoring for better performance of large auto-cli specs

* Fixed: [very slow execution of load_set_file #288](https://github.com/clicon/clixon/issues/288)
* New `clixon-lib@2021-11-11.yang` revision
  * Modified option: RPC stats extended with YANG stats
* Modified `clixon-config@2021-11-11.yang` revision
  * Added option:
    * CLICON_PLUGIN_CALLBACK_CHECK
    * Enable to make plugin context check before and after all callbacks.
* Added statistics for YANG: number of objects and memory used
* Use the treeref no-copy option of CLIgen to reduce memory
* Refactored cli-generation/autocli-start code
* Refactored cligen glue functions to use cligen_eval directly (remove clicon_eval,clixon_cligen_eval)

test/config.sh.in

@@ -65,8 +65,8 @@ CLIXON_VERSION=@CLIXON_VERSION@
 DATASTORE_TOP="config"
 
 # clixon yang revisions occuring in tests
-CLIXON_LIB_REV="2021-03-08"
-CLIXON_CONFIG_REV="2021-05-20"
+CLIXON_LIB_REV="2021-11-11"
+CLIXON_CONFIG_REV="2021-11-11"
 CLIXON_RESTCONF_REV="2021-05-20"
 CLIXON_EXAMPLE_REV="2020-12-01"
 

test/lib.sh

@@ -270,6 +270,7 @@ fi
 # arg1: expected
 # arg2: errmsg[optional]
 # Assumes: $dir and $expect are set
+# see err1
 function err(){
   echo -e "\e[31m\nError in Test$testnr [$testname]:"
   if [ $# -gt 0 ]; then 

test/test_cli_apipath.sh

@@ -112,8 +112,6 @@ expectpart "$($clixon_cli -1 -f $cfg err x)" 255 "Config error: api-path syntax
 new "err x a"
 expectpart "$($clixon_cli -1 -f $cfg err x a 99)" 255 "Config error: api-path syntax error \"/example:x/m1=%s\": rpc malformed-message List key m1 length mismatch : Invalid argument"
 
-endtest
-
 new "Kill backend"
 # Check if premature kill
 pid=$(pgrep -u root -f clixon_backend)
@@ -123,4 +121,6 @@ fi
 # kill backend
 stop_backend -f $cfg
 
+endtest
+
 rm -rf $dir

test/test_openconfig_network_instance.sh

@@ -107,6 +107,7 @@ expectpart "$(cat $dir/config.cli)" 0 "set network-instances network-instance de
 
 new "load saved cli config"
 expectpart "$(cat $dir/config.cli | $clixon_cli -D $DBG -f $cfg 2>&1 > /dev/null)" 0 "^$"
+#time cat $dir/config.cli | $clixon_cli -D $DBG -f $cfg
 
 if [ $BE -ne 0 ]; then
     new "Kill backend"

test/test_perf_mem.sh

@@ -92,7 +92,7 @@ function testrun(){
 
     new "netconf get stats"
     res=$(echo "$DEFAULTHELLO<rpc $DEFAULTNS><stats $LIBNS/></rpc>]]>]]>" | $clixon_netconf -qf $cfg)
-    echo "res:$res"
+#    echo "res:$res"
     err0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/rpc-error")
     err=${err0#"nodeset:"}
     if [ -n "$err" ]; then
@@ -104,7 +104,7 @@ function testrun(){
     echo "   objects: $objects"
 
 #
-    if [ -f /proc/$pid/statm ]; then     # This ony works on Linux 
+    if [ -f /proc/$pid/statm ]; then     # This only works on Linux 
 #	cat /proc/$pid/statm
 	echo -n "   /proc/$pid/statm: "
 	cat /proc/$pid/statm|awk '{print $1*4/1000 "M"}'
@@ -121,7 +121,15 @@ function testrun(){
 	echo -n "   mem: "
 	echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
     done
-
+    for mod in clixon-config; do
+	echo "$mod"
+	resmod0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/module[name=\"$mod\"]")
+	resmod=${resmod0#"nodeset:0:"}
+	echo -n "   objects: "
+	echo $resmod | $clixon_util_xpath -p "module/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
+	echo -n "   mem: "
+	echo $resmod | $clixon_util_xpath -p "module/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
+    done
     if [ $BE -ne 0 ]; then
 	new "Kill backend"
 	# Check if premature kill

test/test_yang_default.sh

@@ -88,9 +88,19 @@ if [ $BE -ne 0 ]; then     # Bring your own backend
     start_backend -s $db -f $cfg
 fi
 
+new "wait backend"
+wait_backend
+
 # permission kludges
+new "chmod datastores"
 sudo chmod 666 $dir/running_db
+if [ $? -ne 0 ]; then
+    err1 "chmod $dir/running_db"
+fi
 sudo chmod 666 $dir/startup_db
+if [ $? -ne 0 ]; then
+    err1 "chmod $dir/startup_db"
+fi
 
 new "Checking startup unchanged"
 ret=$(diff $dir/startup_db <(echo "<${DATASTORE_TOP}>$XML</${DATASTORE_TOP}>"))