From 4c0b412e9f559b4257c8e2291f4fda4e6bf7acdf Mon Sep 17 00:00:00 2001
From: Olof hagsand <olof@hagsand.se>
Date: Tue, 19 Feb 2019 13:16:59 +0100
Subject: [PATCH] Added valgrind memory leak tests for backend. Fixed some
 minor leaks and file descriptor closes.

---
 CHANGELOG.md                   |   3 +-
 apps/backend/backend_main.c    |  18 ++++--
 apps/backend/backend_socket.c  |  27 +++++++-
 lib/src/clixon_file.c          |   4 +-
 lib/src/clixon_log.c           |   1 +
 lib/src/clixon_yang.c          |   2 +
 test/README.md                 |  20 +++---
 test/lib.sh                    |  54 +++++++++++-----
 test/mem.sh                    |  34 +++++-----
 test/test_choice.sh            |  13 ++--
 test/test_cli.sh               |  13 ++--
 test/test_compat.sh            |  25 ++++----
 test/test_feature.sh           |  14 ++---
 test/test_identity.sh          |  14 ++---
 test/test_leafref.sh           |  13 ++--
 test/test_list.sh              |  13 ++--
 test/test_nacm.sh              |  11 +---
 test/test_nacm_ext.sh          |  11 +---
 test/test_nacm_module_read.sh  |  11 +---
 test/test_nacm_module_write.sh |  11 +---
 test/test_nacm_protocol.sh     |  11 +---
 test/test_netconf.sh           |  14 ++---
 test/test_order.sh             |  13 ++--
 test/test_perf.sh              | 111 +++++++++++++++------------------
 test/test_restconf.sh          |  11 +---
 test/test_restconf2.sh         |  11 +---
 test/test_rpc.sh               |  11 +---
 test/test_startup.sh           |  30 ++++-----
 test/test_stream.sh            |  13 ++--
 test/test_type.sh              |  13 ++--
 test/test_union.sh             |  13 ++--
 test/test_when_must.sh         |  13 ++--
 test/test_yang.sh              |  13 ++--
 test/test_yang_load.sh         | 111 ++++++++++++---------------------
 34 files changed, 318 insertions(+), 382 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e95772ab..02df0c2d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -135,7 +135,8 @@
 * Replaced all calls to (obsolete) `cli_output` with `fprintf`
 * Added _experimental_ config option `CLICON_CLI_UTF8` default set to 0.
   * CLIgen UTF8 does not work with scrolling and control editing
-* Added valgrind memory leak tests in testmem.sh for cli and netconf
+* Added valgrind memory leak tests in testmem.sh for cli, netconf and backend
+  * remains: restconf
 * Added `make test` from top-level Makefile
 * Added `xml_rootchild_node()` lib function as variant of `xml_rootchild()`
 * Added -o "<option>=<value>" command-line option to all programs: backend, cli, netconf, restconf.
diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c
index 93e25bed..0bf2a7f8 100644
--- a/apps/backend/backend_main.c
+++ b/apps/backend/backend_main.c
@@ -83,8 +83,10 @@ backend_terminate(clicon_handle h)
 {
     yang_spec *yspec;
     char      *pidfile = clicon_backend_pidfile(h);
+    int       sockfamily = clicon_sock_family(h);
     char      *sockpath = clicon_sock(h);
     cxobj     *x;
+    struct stat st;
 
     clicon_debug(1, "%s", __FUNCTION__);
     if ((yspec = clicon_dbspec_yang(h)) != NULL)
@@ -99,10 +101,12 @@ backend_terminate(clicon_handle h)
     clixon_plugin_exit(h);
     /* Delete all backend plugin RPC callbacks */
     rpc_callback_delete_all(); 
+
     if (pidfile)
 	unlink(pidfile);   
-    if (sockpath)
-	unlink(sockpath);   
+    if (sockfamily==AF_UNIX && lstat(sockpath, &st) == 0)
+	unlink(sockpath);
+	
     xmldb_plugin_unload(h); /* unload storage plugin */
     backend_handle_exit(h); /* Also deletes streams. Cannot use h after this. */
     event_exit();
@@ -203,9 +207,12 @@ db_merge(clicon_handle h,
 }
 
 /*! Create backend server socket and register callback
+ * @param[in]  h    Clicon handle
+ * @retval     s    Server socket file descriptor (see socket(2))
+ * @retval    -1    Error
  */
 static int
-server_socket(clicon_handle h)
+backend_server_socket(clicon_handle h)
 {
     int ss;
 
@@ -571,6 +578,7 @@ main(int    argc,
     yang_spec    *yspec = NULL;
     yang_spec    *yspecfg = NULL; /* For config XXX clixon bug */
     char         *str;
+    int           ss = -1; /* server socket */
     
     /* In the startup, logs to stderr & syslog and debug flag set later */
     clicon_log_init(__PROGRAM__, LOG_INFO, logdst);
@@ -924,7 +932,7 @@ main(int    argc,
     }
 	
     /* Initialize server socket */
-    if (server_socket(h) < 0)
+    if ((ss = backend_server_socket(h)) < 0)
 	goto done;
 
     if (debug)
@@ -937,6 +945,8 @@ main(int    argc,
     retval = 0;
   done:
     clicon_log(LOG_NOTICE, "%s: %u Terminated retval:%d", __PROGRAM__, getpid(), retval);
+    if (ss != -1)
+	close(ss);
     backend_terminate(h); /* Cannot use h after this */
 
     return retval;
diff --git a/apps/backend/backend_socket.c b/apps/backend/backend_socket.c
index 158c56ea..df2fa0df 100644
--- a/apps/backend/backend_socket.c
+++ b/apps/backend/backend_socket.c
@@ -74,6 +74,13 @@
 #include "backend_client.h"
 #include "backend_handle.h"
 
+/*! Open an INET stream socket and bind it to a file descriptor
+ *
+o * @param[in]  h    Clicon handle
+ * @param[in]  dst  IPv4 address (see inet_pton(3))
+ * @retval     s    Socket file descriptor (see socket(2))
+ * @retval    -1    Error
+ */
 static int
 config_socket_init_ipv4(clicon_handle h,
 			char         *dst)
@@ -112,10 +119,14 @@ config_socket_init_ipv4(clicon_handle h,
     return -1;
 }
 
-/*! Open a socket and bind it to a file descriptor
+/*! Open a UNIX domain socket and bind it to a file descriptor
  *
  * The socket is accessed via CLICON_SOCK option, has 770 permissions
  * and group according to CLICON_SOCK_GROUP option.
+ * @param[in]  h    Clicon handle
+ * @param[in]  sock Unix file-system path
+ * @retval     s    Socket file descriptor (see socket(2))
+ * @retval    -1    Error
  */
 static int
 config_socket_init_unix(clicon_handle h,
@@ -175,10 +186,16 @@ config_socket_init_unix(clicon_handle h,
     return -1;
 }
 
+/*! Open backend socket, the one clients send requests to, either ip or unix
+ * 
+ * @param[in]  h    Clicon handle
+ * @retval     s    Socket file descriptor (see socket(2))
+ * @retval    -1    Error
+ */
 int
 backend_socket_init(clicon_handle h)
 {
-    char *sock;
+    char *sock; /* unix path or ip address string */
 
     if ((sock = clicon_sock(h)) == NULL){
 	clicon_err(OE_FATAL, 0, "CLICON_SOCK option not set");
@@ -191,8 +208,12 @@ backend_socket_init(clicon_handle h)
     case AF_INET:
 	return config_socket_init_ipv4(h, sock);
 	break;
+    default:
+	clicon_err(OE_UNIX, EINVAL, "No such address family: %d",
+		   clicon_sock_family(h));
+	break;
     }
-    return 0;
+    return -1;
 }
 
 /*! Accept new socket client
diff --git a/lib/src/clixon_file.c b/lib/src/clixon_file.c
index 73a2b894..38a0234a 100644
--- a/lib/src/clixon_file.c
+++ b/lib/src/clixon_file.c
@@ -160,9 +160,11 @@ clicon_file_dirent(const char     *dir,
 
    qsort((void *)new, nent, sizeof(*new), clicon_file_dirent_sort);
    *ent = new;
+   new = NULL;
    retval = nent;
-
 quit:
+   if (new)
+       free(new);
    if (dirp)
        closedir(dirp);
    if (regexp)
diff --git a/lib/src/clixon_log.c b/lib/src/clixon_log.c
index 4c8641c3..1008210c 100644
--- a/lib/src/clixon_log.c
+++ b/lib/src/clixon_log.c
@@ -103,6 +103,7 @@ clicon_log_exit(void)
 {
     if (_logfile)
 	fclose(_logfile);
+    closelog(); /* optional */
     return 0;
 }
 
diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c
index 05b9a861..4d977b12 100644
--- a/lib/src/clixon_yang.c
+++ b/lib/src/clixon_yang.c
@@ -2556,6 +2556,8 @@ yang_spec_load_dir(clicon_handle h,
 	goto done;
     retval = 0;
   done:
+    if (dp)
+	free(dp);
     if (base)
 	free(base);
     return retval;
diff --git a/test/README.md b/test/README.md
index 8c93def9..bb46052d 100644
--- a/test/README.md
+++ b/test/README.md
@@ -3,12 +3,13 @@
 ## Overview
 
 Tests called 'test_*.sh' and placed in this directory will be
-automatically run as part of the all.sh, sum.sh tests etc. The scripts need to follow some rules to work properly, such as add this magic line as the first command line in the script, which ensures it works well when started from `all.sh`:
-```
-  s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
-```
+automatically run as part of the all.sh, sum.sh tests etc. The scripts
+need to follow some rules to work properly, please look at one or two
+to get the idea.
 
-## Prerequisites
+See also the [site.sh](#site.sh) for example for skipping tests or setting some site-specific variables.
+
+## Getting started
 
 You need to build and install the clixon utility programs before running the tests as some of the tests rely on them:
 ```
@@ -20,6 +21,7 @@ You need to build and install the clixon utility programs before running the tes
 You need to start nginx for some of the text. There are instructions in 
 * If you run systemd: `sudo systemctl start nginx.service`
 * The [example](../example/README.md) has instructions
+* See also the [clixon test container](../docker/system) where all test are encapsulated.
 
 ## Prefix variable
 
@@ -28,7 +30,7 @@ You can prefix a test with `BE=0` if you want to run your own backend.
 To run with debug flags, use the `DBG=<number>` environment variable.
 
 Other variables include:
-* DEMSLEEP Number of seconds to sleep after daemons have started
+* RCWAIT Number of seconds to sleep after daemons have started
 
 ## Run all tests
 
@@ -47,6 +49,8 @@ These tests use valgrind to check for memory leaks:
 ```
   mem.sh cli
   mem.sh netconf
+  mem.sh backend
+#  mem.sh restconf # NYI
 ```
 
 ## Site.sh
@@ -63,7 +67,3 @@ You may add your site-specific modifications in a `site.sh` file. Example:
   IETFRFC=$YANGMODELS/standard/ietf/RFC
 ```
 
-## See also
-
-The [clixon test container](../docker/system) encapsulates the `all.sh` test in a test container. 
-
diff --git a/test/lib.sh b/test/lib.sh
index c6b2d7c6..8e3cd52f 100755
--- a/test/lib.sh
+++ b/test/lib.sh
@@ -43,8 +43,16 @@ fi
 # Single test. Set by "new"
 testname=
 
-# If valgrindtest use a file to log valgrind output on (checked by new)
+# Valgind memory leak check.
+# The values are:
+# 0: No valgrind check
+# 1: Start valgrind at every new testcase. Check result every next new
+# 2: Start valgrind every new backend start. Check when backend stops
+# 3: Start valgrind every new restconf start. Check when restconf stops
+# 
 : ${valgrindtest=0}
+
+# Valgrind log file. This is usually removed automatically
 : ${valgrindfile=$(mktemp)}
 
 # If set to 0, override starting of clixon_backend in test (you bring your own)
@@ -72,22 +80,14 @@ testname=
 : ${IETFRFC=../yang/standard}
 #: ${IETFRFC=$YANGMODELS/standard/ietf/RFC}
 
-# For memcheck
-#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"
+# Follow the binary programs that can be parametrized (eg with valgrind)
+
 : ${clixon_cli:=clixon_cli}
 
-# For memcheck / performance
-#clixon_netconf="valgrind --tool=callgrind clixon_netconf"
-# use kcachegrind to view
-#clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
 : ${clixon_netconf:=clixon_netconf}
 
-# How to run restconf stand-alone and using valgrind
-#clixon_restconf="valgrind --trace-children=no --child-silent-after-fork=yes --leak-check=full --show-leak-kinds=all /www-data/clixon_restconf"
 : ${clixon_restconf:=/www-data/clixon_restconf}
 
-# If you test w valgrind, you need to set -F & and sleep 10 when starting
-#clixon_backend="valgrind --leak-check=full --show-leak-kinds=all clixon_backend"
 : ${clixon_backend:=clixon_backend}
 
 dir=/var/tmp/$0
@@ -122,11 +122,37 @@ checkvalgrind(){
 	res=$(cat $valgrindfile | grep -e "reachable" -e "lost:"|awk '{print  $4}' | grep -v '^0$')
 	if [ -n "$res" ]; then
 	    >&2 cat $valgrindfile
-	    rm -f $valgrindfile
+	    sudo rm -f $valgrindfile
 	    exit -1	    
 	fi
-	rm -f $valgrindfile
-    fi    
+	sudo rm -f $valgrindfile
+    fi
+}
+
+# Start backend with all varargs.
+# If valgrindtest == 2, start valgrind
+start_backend(){
+    if [ $valgrindtest -eq 2 ]; then
+	# Start in background since daemon version creates two traces: parent,
+	# child. If background then only the single relevant.
+	sudo $clixon_backend -F $* -D $DBG &
+    else
+	sudo $clixon_backend $* -D $DBG
+    fi
+    if [ $? -ne 0 ]; then
+	err
+    fi
+}
+
+stop_backend(){
+    sudo clixon_backend -z $*
+    if [ $? -ne 0 ]; then
+	err "kill backend"
+    fi
+    if [ $valgrindtest -eq 2 ]; then 
+	sleep 1
+	checkvalgrind
+    fi
 }
 
 # Increment test number and print a nice string
diff --git a/test/mem.sh b/test/mem.sh
index 426130ba..4c2e45f8 100755
--- a/test/mem.sh
+++ b/test/mem.sh
@@ -14,32 +14,33 @@ echo "valgrindfile:$valgrindfile"
 case "$PROGRAM" in
     'cli')
 	valgrindtest=1
-	DEMSLEEP=1
+	RCWAIT=1
 	clixon_cli="/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --suppressions=./clixon.supp --trace-children=no --child-silent-after-fork=yes --log-file=$valgrindfile clixon_cli"
 ;;
     'netconf')
 	valgrindtest=1
-	DEMSLEEP=1
+	RCWAIT=1
 	clixon_netconf="/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --suppressions=./clixon.supp --trace-children=no --child-silent-after-fork=yes --log-file=$valgrindfile clixon_netconf"
 ;;
-#    'backend')
-#	valgrindtest=2
-#	DEMSLEEP=20
-#	clixon_backend="/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --suppressions=./clixon.supp --trace-children=yes --log-file=$valgrindfile clixon_backend"
-#;;
+    'backend')
+	valgrindtest=2 # This means backend valgrind test
+	RCWAIT=10 # valgrind backend needs some time to get up 
+	perfnr=100 # test_perf.sh restconf put more or less stops
+	perfreq=10
+
+	clixon_backend="/usr/bin/valgrind --leak-check=full --show-leak-kinds=all --suppressions=./clixon.supp --track-fds=yes --trace-children=yes --log-file=$valgrindfile clixon_backend"
+;;
 *)
     echo "usage: $0 cli|netconf|restconf|backend" # valgrind memleak checks
     exit -1
 ;;
 esac
 
-rm -f 
-
 err=0
 testnr=0
-for test in test*.sh; do
+for test in test_*.sh; do
     testfile=$test
-    DEMSLEEP=$DEMSLEEP . ./$test 
+    . ./$test 
     errcode=$?
     if [ $errcode -ne 0 ]; then
 	err=1
@@ -47,12 +48,9 @@ for test in test*.sh; do
 	echo -ne "\e[0m"
 	exit $errcode
     fi
-    if [ $valgrindtest -eq 2 ]; then
-#	sudo cat $valgrindfile
-	sudo checkvalgrind
-#	sudo rm -f $valgrindfile
-    fi
 done
-checkvalgrind
-rm -f $valgrindfile
+if [ $valgrindtest -eq 1 ]; then
+    checkvalgrind
+    sudo rm -f $valgrindfile
+fi
 
diff --git a/test/test_choice.sh b/test/test_choice.sh
index 71fe751f..37d05cc3 100755
--- a/test/test_choice.sh
+++ b/test/test_choice.sh
@@ -99,10 +99,7 @@ if [ $BE -ne 0 ]; then
     sudo pkill clixon_backend # to be sure
     
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
 fi
 
 new "kill old restconf daemon"
@@ -111,6 +108,9 @@ sudo pkill -u www-data clixon_restconf
 new "start restconf daemon"
 sudo su -c "$clixon_restconf -f $RCLOG $cfg -y $fyang -D $DBG" -s /bin/sh www-data &
 
+new "waiting"
+sleep $RCWAIT
+
 # First vanilla (protocol) case
 new "netconf validate empty"
 expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
@@ -207,9 +207,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_cli.sh b/test/test_cli.sh
index bfe9e6fb..363a100f 100755
--- a/test/test_cli.sh
+++ b/test/test_cli.sh
@@ -44,10 +44,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "cli configure top"
@@ -131,9 +131,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_compat.sh b/test/test_compat.sh
index 887de956..8232581f 100755
--- a/test/test_compat.sh
+++ b/test/test_compat.sh
@@ -35,8 +35,7 @@ cat <<EOF > $cfg
 
 EOF
 
-
-run(){
+testrun(){
     mode=$1
     expect=$2
 
@@ -86,17 +85,18 @@ EOF
 	fi
     
 	new "start backend  -f $cfg -s $mode -c $dir/config"
-	sudo $clixon_backend -f $cfg -s $mode -c $dir/config
-	if [ $? -ne 0 ]; then
-	    err
-	fi
+	start_backend -f $cfg -s $mode -c $dir/config
+
+	new "waiting"
+	sleep $RCWAIT
+
     fi
 
     new "Check $mode"
     expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$expect</rpc-reply>]]>]]>$"
 
     if [ $BE -eq 0 ]; then
-	exit # BE
+	return # BE
     fi
 
     new "Kill backend"
@@ -106,14 +106,11 @@ EOF
 	err "backend already dead"
     fi
     # kill backend
-    sudo clixon_backend -z -f $cfg
-    if [ $? -ne 0 ]; then
-	err "kill backend"
-    fi
+    stop_backend -f $cfg
+} # testrun 
 
-} # run 
+testrun running '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
 
-run running '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
-run startup '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
+testrun startup '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
 
 rm -rf $dir
diff --git a/test/test_feature.sh b/test/test_feature.sh
index 4725ff68..7653f153 100755
--- a/test/test_feature.sh
+++ b/test/test_feature.sh
@@ -70,11 +70,10 @@ if [ $BE -ne 0 ]; then
     fi
 
     new "start backend -s init -f $cfg -y $fyang"
-    # start new backend
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "cli enabled feature"
@@ -183,9 +182,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_identity.sh b/test/test_identity.sh
index 7bfdb6da..ea92d00a 100755
--- a/test/test_identity.sh
+++ b/test/test_identity.sh
@@ -117,11 +117,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    # start new backend
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "Set crypto to aes"
@@ -196,9 +195,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_leafref.sh b/test/test_leafref.sh
index 7bbefbcd..7e53f71d 100755
--- a/test/test_leafref.sh
+++ b/test/test_leafref.sh
@@ -85,10 +85,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend  -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "leafref base config"
@@ -155,9 +155,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_list.sh b/test/test_list.sh
index cc9a8f89..ceaf7cb9 100755
--- a/test/test_list.sh
+++ b/test/test_list.sh
@@ -75,10 +75,10 @@ if [ $BE -ne 0 ]; then
     fi
     new "start backend -s init -f $cfg"
     # start new backend
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "minmax: minimal"
@@ -137,9 +137,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_nacm.sh b/test/test_nacm.sh
index e513df63..37a971cf 100755
--- a/test/test_nacm.sh
+++ b/test/test_nacm.sh
@@ -119,10 +119,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -132,6 +129,7 @@ sleep 1
 new "start restconf daemon (-a is enable basic authentication)"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG -- -a" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "auth get"
@@ -193,9 +191,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_nacm_ext.sh b/test/test_nacm_ext.sh
index d5d06e07..ef3742f0 100755
--- a/test/test_nacm_ext.sh
+++ b/test/test_nacm_ext.sh
@@ -139,10 +139,7 @@ if [ $BE -ne 0 ]; then
     sleep 1
     new "start backend -s init -f $cfg"
     # start new backend
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -151,6 +148,7 @@ sudo pkill -u www-data -f "/www-data/clixon_restconf"
 new "start restconf daemon (-a is enable http basic auth)"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG -- -a" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "auth get"
@@ -222,9 +220,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_nacm_module_read.sh b/test/test_nacm_module_read.sh
index 083db895..5624cb44 100755
--- a/test/test_nacm_module_read.sh
+++ b/test/test_nacm_module_read.sh
@@ -134,10 +134,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -147,6 +144,7 @@ sleep 1
 new "start restconf daemon (-a is enable basic authentication)"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG -- -a" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "auth set authentication config"
@@ -287,9 +285,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_nacm_module_write.sh b/test/test_nacm_module_write.sh
index b2e74170..3462fd3f 100755
--- a/test/test_nacm_module_write.sh
+++ b/test/test_nacm_module_write.sh
@@ -142,10 +142,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -155,6 +152,7 @@ sleep 1
 new "start restconf daemon (-a is enable basic authentication)"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG -- -a" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 # Set nacm from scratch
@@ -269,9 +267,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_nacm_protocol.sh b/test/test_nacm_protocol.sh
index 267671af..a918511b 100755
--- a/test/test_nacm_protocol.sh
+++ b/test/test_nacm_protocol.sh
@@ -142,10 +142,7 @@ if [ $BE -ne 0 ]; then
     fi
 
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -155,6 +152,7 @@ sleep 1
 new "start restconf daemon (-a is enable basic authentication)"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG -- -a" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "auth set authentication config"
@@ -231,9 +229,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir 
diff --git a/test/test_netconf.sh b/test/test_netconf.sh
index b28d86c0..c332fe89 100755
--- a/test/test_netconf.sh
+++ b/test/test_netconf.sh
@@ -43,11 +43,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend  -s init -f $cfg"
-    # start new backend
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "netconf hello"
@@ -216,9 +215,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_order.sh b/test/test_order.sh
index 2a7cc198..0711257e 100755
--- a/test/test_order.sh
+++ b/test/test_order.sh
@@ -140,10 +140,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend"
-    sudo $clixon_backend -s running -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s running -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 # STATE (should not be ordered)
@@ -227,9 +227,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_perf.sh b/test/test_perf.sh
index c545e9b9..1bb88806 100755
--- a/test/test_perf.sh
+++ b/test/test_perf.sh
@@ -4,24 +4,17 @@
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
-number=5000
-req=100
-if [ $# = 0 ]; then
-    number=1000
-elif [ $# = 1 ]; then
-    number=$1
-elif [ $# = 2 ]; then
-    number=$1
-    req=$2
-else
-    echo "Usage: $0 [<number> [<requests>]]"
-    exit 1 # Scaling
-fi
+# Number of list/leaf-list entries in file
+: ${perfnr:=1000}
+
+# Number of requests made get/put
+: ${perfreq:=100}
+
 APPNAME=example
 
 cfg=$dir/scaling-conf.xml
 fyang=$dir/scaling.yang
-fconfig=$dir/config
+fconfig=$dir/large.xml
 
 cat <<EOF > $fyang
 module scaling{
@@ -57,10 +50,11 @@ cat <<EOF > $cfg
   <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
   <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+  <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
 </config>
 EOF
 
-new "test params: -f $cfg" -y $fyang
+new "test params: -f $cfg -y $fyang"
 if [ $BE -ne 0 ]; then
     new "kill old backend"
     sudo clixon_backend -zf $cfg -y $fyang
@@ -68,12 +62,8 @@ if [ $BE -ne 0 ]; then
 	err
     fi
 
-    new "start backend  -s init -f $cfg -y $fyang"
-    # start new backend
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    new "start backend -s init -f $cfg -y $fyang"
+    start_backend -s init -f $cfg -y $fyang
 fi
 
 new "kill old restconf daemon"
@@ -82,69 +72,75 @@ sudo pkill -u www-data -f "/www-data/clixon_restconf"
 new "start restconf daemon"
 sudo su -c "$clixon_restconf -f $cfg -y $fyang $RCLOG -D $DBG" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
-new "generate 'large' config with $number list entries"
+new "generate 'large' config with $perfnr list entries"
 echo -n "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\">" > $fconfig
-for (( i=0; i<$number; i++ )); do  
+for (( i=0; i<$perfnr; i++ )); do  
     echo -n "<y><a>$i</a><b>$i</b></y>" >> $fconfig
 done
 echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
 
-# Just for manual dbg
-echo "$clixon_netconf -qf $cfg  -y $fyang"
-
+# Now take large config file and write it via netconf to candidate
 new "netconf write large config"
 expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
-#echo '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' | $clixon_netconf -qf $cfg  -y $fyang 
+# Here, there are $perfnr entries in candidate
 
 new "netconf write large config again"
 expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
-#echo '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' | $clixon_netconf -qf $cfg  -y $fyang 
-
+# Remove the file, its used for different purposes further down
 rm $fconfig
 
+# Now commit it from candidate to running 
 new "netconf commit large config"
 expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
+# Now commit it again from candidate (validation takes time when
+# comparing to existing)
 new "netconf commit large config again"
 expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
-new "netconf add small (1 entry) config"
-expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 '<rpc><edit-config><target><candidate/></target><config><x xmlns="urn:example:clixon"><y><a>x</a><b>y</b></y></x></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
-
-new "netconf get $req small config"
-time -p for (( i=0; i<$req; i++ )); do
-    rnd=$(( ( RANDOM % $number ) ))
-    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
-done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
-
-new "netconf get $req restconf small config"
-time -p for (( i=0; i<$req; i++ )); do
-    rnd=$(( ( RANDOM % $number ) ))
-#XXX    curl -sX PUT -d {"y":{"a":"$rnd","b":"$rnd"}} http://localhost/restconf/data/x/y=$rnd,$rnd 
-done 
-
-new "netconf add $req small config"
-time -p for (( i=0; i<$req; i++ )); do
-    rnd=$(( ( RANDOM % $number ) ))
+# Having a large db, get and put single entries many times
+# Note same entries in the range alreayd there, db has same size
+new "netconf add $perfreq small config"
+time -p for (( i=0; i<$perfreq; i++ )); do
+    rnd=$(( ( RANDOM % $perfnr ) ))
     echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
 done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 
-new "netconf add $req restconf small config"
-time -p for (( i=0; i<$req; i++ )); do
-    rnd=$(( ( RANDOM % $number ) ))
-    curl -sG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null
+new "netconf get $perfreq small config"
+time -p for (( i=0; i<$perfreq; i++ )); do
+    rnd=$(( ( RANDOM % $perfnr ) ))
+    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
+
+new "restconf get $perfreq small config"
+time -p for (( i=0; i<$perfreq; i++ )); do
+    rnd=$(( ( RANDOM % $perfnr ) ))
+    curl -sG http://localhost/restconf/data/scaling:x/y=$rnd,$rnd > /dev/null
+done
+
+new "restconf add $perfreq small config"
+time -p for (( i=0; i<$perfreq; i++ )); do
+    rnd=$(( ( RANDOM % $perfnr ) ))
+    curl -s -X PUT http://localhost/restconf/data/scaling:x/y=$rnd  -d '{"scaling:y":{"a":"'$rnd'","b":"'$rnd'"}}'
 done 
 
+# Instead of many small entries, get one large in netconf and restconf
 new "netconf get large config"
 expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg  -y $fyang" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" '^<rpc-reply><data><x xmlns="urn:example:clixon"><y><a>0</a><b>0</b></y><y><a>1</a><b>1</b>'
 
+new "restconf get large config"
+expecteof "/usr/bin/time -f %e curl -sG http://localhost/restconf/data" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" '^{"data": {"scaling:x": {"y": \[{"a": "0","b": "0"},{ "a": "1","b": "1"},'
+
+# Now do leaf-lists istead of leafs
+
 new "generate large leaf-list config"
 echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fconfig
-for (( i=0; i<$number; i++ )); do  
+for (( i=0; i<$perfnr; i++ )); do  
     echo -n "<c>$i</c>" >> $fconfig
 done
 echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
@@ -157,9 +153,9 @@ rm $fconfig
 new "netconf commit large leaf-list config"
 expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
-new "netconf add $req small leaf-list config"
-time -p for (( i=0; i<$req; i++ )); do
-    rnd=$(( ( RANDOM % $number ) ))
+new "netconf add $perfreq small leaf-list config"
+time -p for (( i=0; i<$perfreq; i++ )); do
+    rnd=$(( ( RANDOM % $perfnr ) ))
     echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><c>$rnd</c></x></config></edit-config></rpc>]]>]]>"
 done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 
@@ -186,9 +182,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_restconf.sh b/test/test_restconf.sh
index 71797f1e..43c5c96b 100755
--- a/test/test_restconf.sh
+++ b/test/test_restconf.sh
@@ -44,10 +44,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -56,6 +53,7 @@ sudo pkill -u www-data clixon_restconf
 new "start restconf daemon"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "restconf tests"
@@ -280,9 +278,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_restconf2.sh b/test/test_restconf2.sh
index 446a8caa..54d1ae3a 100755
--- a/test/test_restconf2.sh
+++ b/test/test_restconf2.sh
@@ -76,10 +76,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
 fi
 
 new "kill old restconf daemon"
@@ -88,6 +85,7 @@ sudo pkill -u www-data -f "/www-data/clixon_restconf"
 new "start restconf daemon"
 sudo su -c "$clixon_restconf -f $cfg -y $fyang $RCLOG -D $DBG" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "restconf tests"
@@ -197,9 +195,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_rpc.sh b/test/test_rpc.sh
index df0de28e..af543edc 100755
--- a/test/test_rpc.sh
+++ b/test/test_rpc.sh
@@ -41,10 +41,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
 fi
 
 new "kill old restconf daemon"
@@ -53,6 +50,7 @@ sudo pkill -u www-data clixon_restconf
 new "start restconf daemon"
 sudo su -c "$clixon_restconf -f $cfg $RCLOG -D $DBG" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 new "rpc tests"
@@ -164,9 +162,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_startup.sh b/test/test_startup.sh
index 2eaac38a..f62fbc81 100755
--- a/test/test_startup.sh
+++ b/test/test_startup.sh
@@ -36,7 +36,7 @@ cat <<EOF > $cfg
 
 EOF
 
-run(){
+testrun(){
     mode=$1
     expect=$2
 
@@ -83,11 +83,11 @@ EOF
 	err
     fi
 
-    new "start backend  -f $cfg -s $mode -c $dir/config"
-    sudo $clixon_backend -f $cfg -s $mode -c $dir/config
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    new "start backend -f $cfg -s $mode -c $dir/config"
+    start_backend -s $mode -f $cfg -c $dir/config
+
+    new "waiting"
+    sleep $RCWAIT
 
     new "Check $mode"
     expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$expect</rpc-reply>]]>]]>$"
@@ -99,15 +99,15 @@ EOF
 	err "backend already dead"
     fi
     # kill backend
-    sudo clixon_backend -z -f $cfg
-    if [ $? -ne 0 ]; then
-	err "kill backend"
-    fi
-}
+    stop_backend -f $cfg
+} # testrun
 
-run init    '<data/>'
-run none    '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
-run running '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
-run startup '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
+testrun init    '<data/>'
+
+testrun none    '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
+
+testrun running '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
+
+testrun startup '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>ex:loopback</type><enabled>true</enabled></interface><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
 
 rm -rf $dir
diff --git a/test/test_stream.sh b/test/test_stream.sh
index c50affef..e9613887 100755
--- a/test/test_stream.sh
+++ b/test/test_stream.sh
@@ -111,10 +111,7 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
 fi
 
 new "kill old restconf daemon"
@@ -123,6 +120,7 @@ sudo pkill -u www-data -f "/www-data/clixon_restconf"
 new "start restconf daemon"
 sudo su -c "$clixon_restconf -f $cfg -y $fyang $RCLOG -D $DBG" -s /bin/sh www-data &
 
+new "waiting"
 sleep $RCWAIT
 
 #
@@ -289,9 +287,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
-#rm -rf $dir
+rm -rf $dir
diff --git a/test/test_type.sh b/test/test_type.sh
index e8efe848..50504379 100755
--- a/test/test_type.sh
+++ b/test/test_type.sh
@@ -203,10 +203,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "cli set transitive string"
@@ -493,9 +493,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_union.sh b/test/test_union.sh
index 57cf1eaa..a8805d2a 100755
--- a/test/test_union.sh
+++ b/test/test_union.sh
@@ -86,10 +86,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "cli set transitive string"
@@ -112,9 +112,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_when_must.sh b/test/test_when_must.sh
index ee48314c..6b288373 100755
--- a/test/test_when_must.sh
+++ b/test/test_when_must.sh
@@ -99,10 +99,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "when: add static route"
@@ -152,9 +152,6 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 
 rm -rf $dir
diff --git a/test/test_yang.sh b/test/test_yang.sh
index e12c766c..764d7589 100755
--- a/test/test_yang.sh
+++ b/test/test_yang.sh
@@ -151,10 +151,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend -s init -f $cfg -y $fyang"
-    sudo $clixon_backend -s init -f $cfg -y $fyang -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "cli defined extension"
@@ -292,10 +292,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 rm -rf $dir
diff --git a/test/test_yang_load.sh b/test/test_yang_load.sh
index 1f6468ad..cbf14965 100755
--- a/test/test_yang_load.sh
+++ b/test/test_yang_load.sh
@@ -79,10 +79,10 @@ if [ $BE -ne 0 ]; then
 	err
     fi
     new "start backend  -s init -f $cfg"
-    sudo $clixon_backend -s init -f $cfg -D $DBG
-    if [ $? -ne 0 ]; then
-	err
-    fi
+    start_backend -s init -f $cfg
+
+    new "waiting"
+    sleep $RCWAIT
 fi
 
 new "1. Set newex"
@@ -105,10 +105,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 #--------------------------------------
@@ -133,10 +130,10 @@ EOF
 
 new "start backend  -s init -f $cfg"
 # start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set oldex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><oldex xmlns="urn:example:clixon">str</oldex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -154,10 +151,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 #--------------------------------------
@@ -177,11 +171,10 @@ cat <<EOF > $cfg
 EOF
 
 new "start backend  -s init -f $cfg"
-# start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set newex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><newex xmlns="urn:example:clixon">str</newex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -198,11 +191,7 @@ pid=`pgrep -u root -f clixon_backend`
 if [ -z "$pid" ]; then
     err "backend already dead"
 fi
-# kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 #--------------------------------------
@@ -223,11 +212,10 @@ cat <<EOF > $cfg
 EOF
 
 new "start backend  -s init -f $cfg"
-# start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set oldex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><oldex xmlns="urn:example:clixon">str</oldex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -245,10 +233,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 #--------------------------------------
@@ -268,11 +253,10 @@ cat <<EOF > $cfg
 EOF
 
 new "start backend  -s init -f $cfg"
-# start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set newex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><newex xmlns="urn:example:clixon">str</newex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -290,10 +274,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 #--------------------------------------
@@ -314,11 +295,10 @@ cat <<EOF > $cfg
 EOF
 
 new "start backend  -s init -f $cfg"
-# start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set oldex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><oldex xmlns="urn:example:clixon">str</oldex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -336,10 +316,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 
@@ -362,11 +339,10 @@ cat <<EOF > $cfg
 EOF
 
 new "start backend  -s init -f $cfg"
-# start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set oldex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><oldex xmlns="urn:example:clixon">str</oldex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -384,10 +360,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 #--------------------------------------
@@ -409,11 +382,10 @@ cat <<EOF > $cfg
 EOF
 
 new "start backend  -s init -f $cfg"
-# start new backend
-sudo $clixon_backend -s init -f $cfg
-if [ $? -ne 0 ]; then
-    err
-fi
+start_backend -s init -f $cfg
+
+new "waiting"
+sleep $RCWAIT
 
 new "Set oldex"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><config><oldex xmlns="urn:example:clixon">str</oldex></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
@@ -431,10 +403,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -z -f $cfg
-if [ $? -ne 0 ]; then
-    err "kill backend"
-fi
+stop_backend -f $cfg
 sudo pkill -u root -f clixon_backend
 
 rm -rf $dir