From 953326d39f365f78a1dcada6d9b4a7d77709f9c6 Mon Sep 17 00:00:00 2001
From: Olof hagsand <olof@hagsand.se>
Date: Sun, 11 Apr 2021 17:36:22 +0200
Subject: [PATCH] - Rewrote process control to simpler state model:
 stopped/running/exiting - Stricter CLICON_BACKEND_RESTCONF_PROCESS :  - if
 set, restconf daemon queries backend for its config  - if not set, restconf
 daemon reads its config from main config file

---
 CHANGELOG.md                                  |   7 +-
 apps/backend/backend_plugin_restconf.c        |  10 +
 apps/restconf/restconf_main_fcgi.c            |  16 +-
 apps/restconf/restconf_main_native.c          |  31 +-
 lib/src/clixon_proc.c                         | 298 ++++++++++++------
 test/lib.sh                                   |  23 +-
 test/test_restconf.sh                         |   3 +
 ...tconf_rpc.sh => test_restconf_internal.sh} | 251 +++++++++------
 ...pc2.sh => test_restconf_internal_cases.sh} | 172 +++++-----
 yang/clixon/clixon-config@2021-03-08.yang     |   2 +
 yang/clixon/clixon-lib@2021-03-08.yang        |  14 +-
 11 files changed, 517 insertions(+), 310 deletions(-)
 rename test/{test_restconf_rpc.sh => test_restconf_internal.sh} (64%)
 rename test/{test_restconf_rpc2.sh => test_restconf_internal_cases.sh} (72%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8626141..b5f10c15 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,8 +37,10 @@ Expected: April
 ### API changes on existing protocol/config features
 
 * Native RESTCONF mode
-  * Restconf "evhtp" mode MUST use libevhtp from https://github.com/clixon/clixon-libevhtp.git instead from criticalstack
-  * To configure native mode use: `configure --with-restconf=native`, changed from: `configure --with-restconf=evhtp`
+  * Renamed restconf "evhtp" mode to "native" mode
+    * To configure native mode use: `configure --with-restconf=native`, changed from: `configure --with-restconf=evhtp`
+  * Native mode MUST use libevhtp from https://github.com/clixon/clixon-libevhtp.git instead from criticalstack
+
 * NETCONF Hello message semantics has been made stricter according to RFC 6241 Sec 8.1, for example:
   * A client MUST send a <hello> element.
   * Each peer MUST send at least the base NETCONF capability, "urn:ietf:params:netconf:base:1.1" (or 1.0 for RFC 4741)
@@ -46,6 +48,7 @@ Expected: April
   * You can set `CLICON_NETCONF_HELLO_OPTIONAL` to true to use the old behavior of essentially ignoring hellos.
 * New clixon-lib@2020-03-08.yang revision
   * Changed: RPC process-control output to choice with status fields
+    * The fields are: active, description, command, status, starttime, pid (or just ok).
 * New clixon-config@2020-03-08.yang revision
   * Added: `CLICON_NETCONF_HELLO_OPTIONAL`
   * Added: `CLICON_CLI_AUTOCLI_EXCLUDE`
diff --git a/apps/backend/backend_plugin_restconf.c b/apps/backend/backend_plugin_restconf.c
index 536e6018..ceba7f04 100644
--- a/apps/backend/backend_plugin_restconf.c
+++ b/apps/backend/backend_plugin_restconf.c
@@ -238,12 +238,14 @@ restconf_pseudo_process_commit(clicon_handle    h,
 {
     int    retval = -1;
     cxobj *xtarget;
+    cxobj *xsource;
     cxobj *cx;
     int    enabled = 0;
     cxobj *xb;
     
     clicon_debug(1, "%s", __FUNCTION__);
     xtarget = transaction_target(td);
+    xsource = transaction_src(td);
     if (xpath_first(xtarget, NULL, "/restconf[enable='true']") != NULL)
 	enabled++;
     /* Get debug flag of restconf config, set the restconf start -D daemon flag according
@@ -271,10 +273,18 @@ restconf_pseudo_process_commit(clicon_handle    h,
 		/* A restart can terminate a restconf connection (cut the tree limb you are sitting on)
 		 * Specifically, the socket is terminated where the reply is sent, which will
 		 * cause the curl to fail.
+		 * Note that it should really be a START if the process is stopped, but the
+		 * commit code need not know any of that
 		 */
 		if (clixon_process_operation(h, RESTCONF_PROCESS, PROC_OP_RESTART, 0) < 0)
 		    goto done;
 	    }
+	    else if ((cx = xpath_first(xsource, NULL, "/restconf")) != NULL &&
+		     xml_flag(cx, XML_FLAG_CHANGE|XML_FLAG_DEL)){
+		/* Or something deleted */
+		if (clixon_process_operation(h, RESTCONF_PROCESS, PROC_OP_RESTART, 0) < 0)
+		    goto done;
+	    }
 	}
     }
     retval = 0;
diff --git a/apps/restconf/restconf_main_fcgi.c b/apps/restconf/restconf_main_fcgi.c
index 0c294d4d..2d5a29d4 100644
--- a/apps/restconf/restconf_main_fcgi.c
+++ b/apps/restconf/restconf_main_fcgi.c
@@ -431,15 +431,17 @@ main(int    argc,
     if (clixon_plugin_start_all(h) < 0)
 	goto done;
 
-    /* First try to get restconf config from local config-file */
-    if ((xrestconf1 = clicon_conf_restconf(h)) != NULL){
-	if ((ret = restconf_config_init(h, xrestconf1)) < 0)
-	    goto done;
-	if (ret == 1)
-	    configure_done = 1;
+    if (clicon_option_bool(h, "CLICON_BACKEND_RESTCONF_PROCESS") == 0){
+	/* If not read from backend, try to get restconf config from local config-file */
+	if ((xrestconf1 = clicon_conf_restconf(h)) != NULL){
+	    if ((ret = restconf_config_init(h, xrestconf1)) < 0)
+		goto done;
+	    if (ret == 1)
+		configure_done = 1;
+	}
     }
     /* If no local config, or it is disabled, try to query backend of config. */
-    if (!configure_done){
+    else {
 	/* Loop to wait for backend starting, try again if not done */
 	while (1){
 	    if (clicon_hello_req(h, &id) < 0){
diff --git a/apps/restconf/restconf_main_native.c b/apps/restconf/restconf_main_native.c
index 0ebb371a..bbb7fde1 100644
--- a/apps/restconf/restconf_main_native.c
+++ b/apps/restconf/restconf_main_native.c
@@ -1610,24 +1610,25 @@ restconf_clixon_init(clicon_handle h,
 	goto done;
     if (clicon_nsctx_global_set(h, nsctx_global) < 0)
 	goto done;
-
-    /* First try to get restconf config from local config-file */
     ret = 0;
-    if ((xrestconf = clicon_conf_restconf(h)) != NULL){
-	/*! Basic config init, set auth-type, pretty, etc ret 0 means disabled */
-	if ((ret = restconf_config_init(h, xrestconf)) < 0)
-	    goto done;
-	/* ret == 1 means this config is OK */
-	if (ret == 0){
-	    xrestconf = NULL; /* Dont free since it is part of conf tree */
-	}
-	else
-	    if ((*xrestconfp = xml_dup(xrestconf)) == NULL)
+    if (clicon_option_bool(h, "CLICON_BACKEND_RESTCONF_PROCESS") == 0){
+	/* If not read from backend, try to get restconf config from local config-file */
+	if ((xrestconf = clicon_conf_restconf(h)) != NULL){
+	    /*! Basic config init, set auth-type, pretty, etc ret 0 means disabled */
+	    if ((ret = restconf_config_init(h, xrestconf)) < 0)
 		goto done;
+	    /* ret == 1 means this config is OK */
+	    if (ret == 0){
+		xrestconf = NULL; /* Dont free since it is part of conf tree */
+	    }
+	    else
+		if ((*xrestconfp = xml_dup(xrestconf)) == NULL)
+		    goto done;
+	}
     }
-    /* If no local config, or it is disabled, try to query backend of config. */
-    if (ret == 0){
-	assert(*xrestconfp == NULL);
+    /* If no local config, or it is disabled, try to query backend of config. 
+     */
+    else {
 	if ((ret = restconf_clixon_backend(h, xrestconfp)) < 0)
 	    goto done;
 	if (ret == 0)
diff --git a/lib/src/clixon_proc.c b/lib/src/clixon_proc.c
index 99aafa35..dba3ed5c 100644
--- a/lib/src/clixon_proc.c
+++ b/lib/src/clixon_proc.c
@@ -34,20 +34,48 @@
   ***** END LICENSE BLOCK *****
 
   * Processes daemons
-  * States of processes:
+
   A description of process states.
-  It starts in a STOPPED state. On operation "start" or "restart" it gets a pid and goes into RUNNING:
-       STOPPED  --(re)start--> RUNNING
-  In RUNNING several things can happen:
-  - It is killed externally: the process then gets a SIGCHLD which triggers a wait and it goes into STOPPED:
-       RUNNING  --sigchld/wait-->  STOPPED
-  It is stopped due to an rpc or by config commit removing the config. In that case the parent 
-  process kills the process and enters into EXITING waiting for a SIGCHLD that triggers a wait:
-       RUNNING --stop--> EXITING --sigchld/wait--> STOPPED
-  It is restarted due to an rpc or config change (eg a server is added, a key modified, etc). Then 
-  a new process is started which enters RUNNING, while the old process (the dying clone) enters EXITING:
-       STOPPED  --restart--> RUNNING(newpid)
-       RUNNING --stop--> EXITING --sigchld/wait--> REMOVED (oldpid/clone)
+  An entity is a process_entry_t with a unique name. pids are created for "active" processes.
+  States:
+     STOPPED: pid=0,   No process running
+     RUNNING: pid set, Process started and believed to be running
+     EXITING: pid set, Process is killed by parent but not waited for
+   
+  Operations:
+     start, stop, restart
+
+  Transitions:
+     Process struct created by calling clixon_process_register() with static info such as name,
+     description, namespace, start arguments, etc. Starts in STOPPED state:
+       --> STOPPED
+
+     On operation "start" or "restart" it gets a pid and goes into RUNNING state:
+           STOPPED -- (re)start --> RUNNING(pid)
+
+     When running, several things may happen:
+     1. It is killed externally: the process gets a SIGCHLD triggers a wait and it goes to STOPPED:
+           RUNNING  --sigchld/wait-->  STOPPED
+
+     2. It is stopped due to a rpc or configuration remove: 
+        The parent kills the process and enters EXITING waiting for a SIGCHLD that triggers a wait,
+	therafter it goes to STOPPED
+           RUNNING --stop--> EXITING  --sigchld/wait--> STOPPED
+     
+     3. It is restarted due to rpc or config change (eg a server is added, a key modified, etc). 
+        The parent kills the process and enters EXITING waiting for a SIGCHLD that triggers a wait,
+	therafter a new process is started and it goes to RUNNING with a new pid
+
+           RUNNING --restart--> EXITING  --sigchld/wait + restart --> RUNNING(pid)
+
+      A complete state diagram is:
+
+      STOPPED  --(re)start-->     RUNNING(pid)
+          ^   <--1.wait(kill)---   |  ^
+	  |                   stop/|  | 
+          |                 restart|  | restart
+          |                        v  |
+          wait(stop) ------- EXITING(dying pid)
  */
 
 #ifdef HAVE_CONFIG_H
@@ -98,6 +126,14 @@
 /*
  * Types
  */
+/* Process state
+ */
+enum proc_state {
+    PROC_STATE_STOPPED,
+    PROC_STATE_RUNNING,
+    PROC_STATE_EXITING
+};
+typedef enum proc_state proc_state_t;
 
 /* Process entry list */
 struct process_entry_t {
@@ -108,10 +144,9 @@ struct process_entry_t {
     char         **pe_argv;     /* argv with command as element 0 and NULL-terminated */
     int            pe_argc;     /* Length of argc */
     pid_t          pe_pid;      /* Running process id (state) or 0 if dead (pid is set if exiting=1) */
-    int            pe_exiting;  /* If set process is in the process of dying needs reaping */
-    int            pe_clone;    /* Duplicate when restarting, delete when reaped */
-    pid_t          pe_status;   /* Status on exit as defined in waitpid */
-    proc_operation pe_op;       /* Operation pending? */
+    proc_operation pe_operation;/* Pending operation: stop/start/restart */
+    proc_state_t   pe_state;    /* stopped, running, exiting */
+    pid_t          pe_exit_status;/* Status on exit as defined in waitpid */
     struct timeval pe_starttime; /* Start time */
     proc_cb_t     *pe_callback; /* Wrapper function, may be called from process_operation  */
 };
@@ -307,14 +342,21 @@ clixon_proc_background(char       **argv,
  * Process management: start/stop registered processes for internal use
  */
 
+static const map_str2int proc_state_map[] = {
+    {"stopped",          PROC_STATE_STOPPED},
+    {"running",          PROC_STATE_RUNNING},
+    {"exiting",          PROC_STATE_EXITING},
+    {NULL,               -1}
+};
+
 /* Process operations
  */
 static const map_str2int proc_operation_map[] = {
-    {"none",                  PROC_OP_NONE},
-    {"start",                 PROC_OP_START},
-    {"stop",                  PROC_OP_STOP},
-    {"restart",               PROC_OP_RESTART},
-    {"status",                PROC_OP_STATUS},
+    {"none",                  PROC_OP_NONE},   /* Not state transition operator */
+    {"start",                 PROC_OP_START},  /* State transition operator */
+    {"stop",                  PROC_OP_STOP},   /* State transition operator */
+    {"restart",               PROC_OP_RESTART},/* State transition operator */
+    {"status",                PROC_OP_STATUS}, /* Not state transition operator */
     {NULL,                    -1}
 };
 
@@ -356,6 +398,7 @@ clixon_process_argv_get(clicon_handle h,
     return 0;
 }
 
+#ifdef NYI
 /*! Make a copy of process-entry struct 
  *
  * @param[in]  pe0   Original process-entry
@@ -415,6 +458,7 @@ clixon_process_register_dup(process_entry_t  *pe0,
     /* dealloc pe1 on error */
     return retval;
 }
+#endif
 
 /*! Register an internal process
  *
@@ -480,6 +524,11 @@ clixon_process_register(clicon_handle h,
 	}
     }
     pe->pe_callback = callback;
+    clicon_debug(1, "%s %s ----> %s", __FUNCTION__,
+		 pe->pe_name, 
+		 clicon_int2str(proc_state_map, PROC_STATE_STOPPED)
+		 );
+    pe->pe_state = PROC_STATE_STOPPED;
     ADDQ(pe, _proc_entry_list);
     retval = 0;
  done:
@@ -557,7 +606,7 @@ proc_op_run(pid_t pid0,
  *
  * @param[in]  h       clicon handle
  * @param[in]  name    Name of process
- * @param[in]  op      start, stop, restart, status
+ * @param[in]  op0     start, stop, restart, status
  * @param[in]  wrapit  If set, call potential callback, if false, dont call it
  * @retval -1  Error
  * @retval  0  OK
@@ -570,11 +619,12 @@ proc_op_run(pid_t pid0,
 int
 clixon_process_operation(clicon_handle  h,
 			 const char    *name,
-			 proc_operation op,
+			 proc_operation op0,
 			 int            wrapit)
 {
     int              retval = -1;
     process_entry_t *pe;
+    proc_operation   op;
     int              sched = 0; /* If set, process action should be scheduled, register a timeout */
 
     clicon_debug(1, "%s name:%s op:%s", __FUNCTION__, name, clicon_int2str(proc_operation_map, op));
@@ -583,17 +633,22 @@ clixon_process_operation(clicon_handle  h,
     pe = _proc_entry_list;
     do {
 	if (strcmp(pe->pe_name, name) == 0){
-	    /* Call wrapper function that eg changes op based on config */
+	    /* Call wrapper function that eg changes op1 based on config */
+	    op = op0;
 	    if (wrapit && pe->pe_callback != NULL)
 		if (pe->pe_callback(h, pe, &op) < 0)
 		    goto done;
-	    clicon_debug(1, "%s name: %s pid:%d op: %s", __FUNCTION__,
-			 name, pe->pe_pid, clicon_int2str(proc_operation_map, op));
+
 	    if (op == PROC_OP_START || op == PROC_OP_STOP || op == PROC_OP_RESTART){
-		pe->pe_op = op;
-		clicon_debug(1, "%s scheduling %s pid:%d", __FUNCTION__, name, pe->pe_pid);
+		pe->pe_operation = op;
+		clicon_debug(1, "%s scheduling name: %s pid:%d op: %s", __FUNCTION__,
+			     name, pe->pe_pid,
+			     clicon_int2str(proc_operation_map, pe->pe_operation));
 		sched++;
 	    }
+	    else{
+		clicon_debug(1, "%s name:%s op %s cancelled by wrwap", __FUNCTION__, name, clicon_int2str(proc_operation_map, op0));		
+	    }
 	    break; 	    /* hit break here */
 	}
 	pe = NEXTQ(process_entry_t *, pe);
@@ -632,8 +687,6 @@ clixon_process_status(clicon_handle  h,
     pe = _proc_entry_list;
     do {
 	if (strcmp(pe->pe_name, name) == 0){
-	    if (pe->pe_clone)
-		continue; /* this may be a dying duplicate */
 	    /* Check if running */
 	    run = 0;
 	    if (pe->pe_pid && proc_op_run(pe->pe_pid, &run) < 0)
@@ -642,8 +695,6 @@ clixon_process_status(clicon_handle  h,
 		    NETCONF_BASE_NAMESPACE, CLIXON_LIB_NS, run?"true":"false");
 	    if (pe->pe_description)
 		cprintf(cbret, "<description xmlns=\"%s\">%s</description>", CLIXON_LIB_NS, pe->pe_description);
-	    if (pe->pe_pid)
-		cprintf(cbret, "<pid xmlns=\"%s\">%u</pid>", CLIXON_LIB_NS, pe->pe_pid);
 	    cprintf(cbret, "<command xmlns=\"%s\">", CLIXON_LIB_NS);
 	    for (i=0; i<pe->pe_argc-1; i++){
 		if (i)
@@ -651,13 +702,17 @@ clixon_process_status(clicon_handle  h,
 		cprintf(cbret, "%s", pe->pe_argv[i]);
 	    }
 	    cprintf(cbret, "</command>");
-	    if (run && timerisset(&pe->pe_starttime)){
+	    cprintf(cbret, "<status xmlns=\"%s\">%s</status>", CLIXON_LIB_NS,
+		    clicon_int2str(proc_state_map, pe->pe_state));
+	    if (timerisset(&pe->pe_starttime)){
 		if (time2str(pe->pe_starttime, timestr, sizeof(timestr)) < 0){
 		    clicon_err(OE_UNIX, errno, "time2str");
 		    goto done;
 		}
 		cprintf(cbret, "<starttime xmlns=\"%s\">%s</starttime>", CLIXON_LIB_NS, timestr);
 	    }
+	    if (pe->pe_pid)
+		cprintf(cbret, "<pid xmlns=\"%s\">%u</pid>", CLIXON_LIB_NS, pe->pe_pid);
 	    cprintf(cbret, "</rpc-reply>");
 	    break; 	    /* hit break here */
 	}
@@ -694,7 +749,7 @@ clixon_process_start_all(clicon_handle h)
 	    if (pe->pe_callback(h, pe, &op) < 0)
 		goto done;
 	if (op == PROC_OP_START){
-	    pe->pe_op = op;
+	    pe->pe_operation = op;
 	    sched++;
 	}
 	pe = NEXTQ(process_entry_t *, pe);
@@ -723,73 +778,90 @@ clixon_process_sched(int           fd,
 {
     int              retval = -1;
     process_entry_t *pe;
-    process_entry_t *pe1;
-    proc_operation   op;
-    pid_t            newpid;
-    int              run;
+    int              isrunning; /* Process is actually running */
 
     clicon_debug(1, "%s",__FUNCTION__);
     if (_proc_entry_list == NULL)
 	goto ok;
     pe = _proc_entry_list;
     do {
-	clicon_debug(1, "%s name: %s pid:%d op: %s", __FUNCTION__,
-		     pe->pe_name, pe->pe_pid, clicon_int2str(proc_operation_map, pe->pe_op));
+	clicon_debug(1, "%s name: %s pid:%d %s --op:%s-->", __FUNCTION__,
+		     pe->pe_name, pe->pe_pid, clicon_int2str(proc_state_map, pe->pe_state), clicon_int2str(proc_operation_map, pe->pe_operation));
 	/* Execute pending operations and not already exiting */
-	if ((op = pe->pe_op) != PROC_OP_NONE &&
-	    pe->pe_exiting == 0){
-	    /* Check if running */
-	    run = 0;
-	    if (proc_op_run(pe->pe_pid, &run) < 0)
-		goto done;
-	    switch (op){
-	    case PROC_OP_STOP:
-		clicon_debug(1, "%s stop pid:%d", __FUNCTION__, pe->pe_pid);
-	    case PROC_OP_RESTART:
-		if (run){
-		    clicon_log(LOG_NOTICE, "Killing old process %s with pid: %d", pe->pe_name, pe->pe_pid);
-		    kill(pe->pe_pid, SIGTERM);
-		    /* Cant wait here because it would block the backend and terminating may involve
-		     * some protocol handling, instead SIGCHLD is receoved and 
-		     * clixon_process_waitpid is called that for waits/reaps the dead process */
-		    pe->pe_exiting = 1;
-		}
-		if (op == PROC_OP_STOP)
+	if (pe->pe_operation != PROC_OP_NONE){
+	    switch (pe->pe_state){
+	    case PROC_STATE_EXITING:
+		break; /* only clixon_process_waitpid can change state in exiting */
+	    case PROC_STATE_STOPPED:
+		switch (pe->pe_operation){
+		case PROC_OP_RESTART: /* stopped -> restart can happen if its externall stopped */
+		case PROC_OP_START:
+		    /* Check if actual running using kill(0) */
+		    isrunning = 0;
+		    if (proc_op_run(pe->pe_pid, &isrunning) < 0)
+			goto done;
+		    if (!isrunning)
+			if (clixon_proc_background(pe->pe_argv, pe->pe_netns, &pe->pe_pid) < 0)
+			    goto done;
+		    clicon_debug(1, "%s %s(%d) %s --%s--> %s", __FUNCTION__,
+				 pe->pe_name, pe->pe_pid,
+				 clicon_int2str(proc_state_map, pe->pe_state),
+				 clicon_int2str(proc_operation_map, pe->pe_operation),
+				 clicon_int2str(proc_state_map, PROC_STATE_RUNNING)
+				 );
+		    pe->pe_state = PROC_STATE_RUNNING;
+		    gettimeofday(&pe->pe_starttime, NULL);
+		    pe->pe_operation = PROC_OP_NONE;
 		    break;
-		if (!run){
+		default:
+		    break;
+		}
+		break;
+	    case PROC_STATE_RUNNING:
+		/* Check if actual running using kill(0) */
+		isrunning = 0;
+		if (proc_op_run(pe->pe_pid, &isrunning) < 0)
+		    goto done;
+		switch (pe->pe_operation){
+		case PROC_OP_STOP:
+		    clicon_debug(1, "%s stop pid:%d", __FUNCTION__, pe->pe_pid);
+		case PROC_OP_RESTART:
+		    if (isrunning){
+			clicon_log(LOG_NOTICE, "Killing old process %s with pid: %d", pe->pe_name, pe->pe_pid);
+			kill(pe->pe_pid, SIGTERM);
+			/* Cant wait here because it would block the backend and terminating may involve
+			 * some protocol handling, instead SIGCHLD is receoved and 
+			 * clixon_process_waitpid is called that for waits/reaps the dead process */
+		    }
+		    clicon_debug(1, "%s %s(%d) %s --%s--> %s", __FUNCTION__,
+				 pe->pe_name, pe->pe_pid,
+				 clicon_int2str(proc_state_map, pe->pe_state),
+				 clicon_int2str(proc_operation_map, pe->pe_operation),
+				 clicon_int2str(proc_state_map, PROC_STATE_EXITING)
+				 );
+		    pe->pe_state = PROC_STATE_EXITING; /* Keep operation stop/restart */
+		    break;
+		case PROC_OP_START:
+		    if (isrunning) /* Already runs */
+			break;
 		    if (clixon_proc_background(pe->pe_argv, pe->pe_netns, &pe->pe_pid) < 0)
 			goto done;
+		    clicon_debug(1, "%s %s(%d) %s --%s--> %s", __FUNCTION__,
+				 pe->pe_name, pe->pe_pid,
+				 clicon_int2str(proc_state_map, pe->pe_state),
+				 clicon_int2str(proc_operation_map, pe->pe_operation),
+				 clicon_int2str(proc_state_map, PROC_STATE_RUNNING)
+				 );
 		    gettimeofday(&pe->pe_starttime, NULL);
-		    clicon_debug(1, "%s started pid:%d", __FUNCTION__, pe->pe_pid);
-		}
-		else {
-		    /* This is the case where there is an existing process running.
-		     * it was killed above but still runs and needs to be reaped */
-		    if (clixon_proc_background(pe->pe_argv, pe->pe_netns, &newpid) < 0)
-			goto done;
-		    gettimeofday(&pe->pe_starttime, NULL);
-		    clicon_debug(1, "%s restart pid:%d -> %d", __FUNCTION__, pe->pe_pid, newpid);
-		    /* Create a new pe */
-		    if (clixon_process_register_dup(pe, &pe1) < 0)
-			goto done;
-		    pe->pe_clone = 1; /* Delete when reaped */
-		    pe1->pe_op = PROC_OP_NONE; /* Dont restart again */
-		    pe1->pe_pid = newpid;
-		}
-		break;
-	    case PROC_OP_START:
-		if (run) /* Already runs */
+		    pe->pe_operation = PROC_OP_NONE;
 		    break;
-		if (clixon_proc_background(pe->pe_argv, pe->pe_netns, &pe->pe_pid) < 0)
-		    goto done;
-		gettimeofday(&pe->pe_starttime, NULL);
-		clicon_debug(1, "%s started pid:%d", __FUNCTION__, pe->pe_pid);
-		break;
+		default:
+		    break;
+		}/* switch pe_state */
 	    default:
 		break;
-	    }
+	    } /* switch pe_state */
 	}
-	pe->pe_op = PROC_OP_NONE;
 	pe = NEXTQ(process_entry_t *, pe);
     } while (pe != _proc_entry_list);
  ok:
@@ -841,18 +913,50 @@ clixon_process_waitpid(clicon_handle h)
     clicon_debug(1, "%s", __FUNCTION__);
     pe = _proc_entry_list;
     do {
-	if (pe->pe_pid != 0){
-	    clicon_debug(1, "%s waitpid(%d)", __FUNCTION__, pe->pe_pid);
+	clicon_debug(1, "%s %s(%d) %s op:%s", __FUNCTION__,
+		     pe->pe_name, pe->pe_pid,
+		     clicon_int2str(proc_state_map, pe->pe_state),
+		     clicon_int2str(proc_operation_map, pe->pe_operation));
+	if (pe->pe_pid != 0
+	    && (pe->pe_state == PROC_STATE_RUNNING || pe->pe_state == PROC_STATE_EXITING)
+	    //	    && (pe->pe_operation == PROC_OP_STOP || pe->pe_operation == PROC_OP_RESTART)
+	    ){
+	    clicon_debug(1, "%s %s waitpid(%d)", __FUNCTION__, pe->pe_name, pe->pe_pid);
 	    if ((wpid = waitpid(pe->pe_pid, &status, WNOHANG)) == pe->pe_pid){
 		clicon_debug(1, "%s waitpid(%d) waited", __FUNCTION__, pe->pe_pid);
-		pe->pe_exiting = 0;
-		pe->pe_pid = 0;       /* mark as dead */
-		pe->pe_status = status;   
-		if (pe->pe_clone){
-		    /* Delete it */
-		    DELQ(pe, _proc_entry_list, process_entry_t *);
-		    clixon_process_delete_only(pe);
+		pe->pe_exit_status = status;
+		switch (pe->pe_operation){
+		case PROC_OP_NONE: /* Spontaneous / External termination */
+		case PROC_OP_STOP:
+		    clicon_debug(1, "%s %s(%d) %s --%s--> %s", __FUNCTION__,
+				 pe->pe_name, pe->pe_pid,
+				 clicon_int2str(proc_state_map, pe->pe_state),
+				 clicon_int2str(proc_operation_map, pe->pe_operation),
+				 clicon_int2str(proc_state_map, PROC_STATE_STOPPED)
+				 );
+		    pe->pe_state = PROC_STATE_STOPPED;
+		    pe->pe_pid = 0;       
+		    timerclear(&pe->pe_starttime);
+		    break;
+		case PROC_OP_RESTART:
+		    /* This is the case where there is an existing process running.
+		     * it was killed above but still runs and needs to be reaped */
+		    if (clixon_proc_background(pe->pe_argv, pe->pe_netns, &pe->pe_pid) < 0)
+			goto done;
+		    gettimeofday(&pe->pe_starttime, NULL);
+		    clicon_debug(1, "%s %s(%d) %s --%s--> %s", __FUNCTION__,
+				 pe->pe_name, pe->pe_pid,
+				 clicon_int2str(proc_state_map, pe->pe_state),
+				 clicon_int2str(proc_operation_map, pe->pe_operation),
+				 clicon_int2str(proc_state_map, PROC_STATE_RUNNING)
+				 );
+		    pe->pe_state = PROC_STATE_RUNNING;
+		    gettimeofday(&pe->pe_starttime, NULL);
+		    break;
+		default:
+		    break;
 		}
+		pe->pe_operation = PROC_OP_NONE;
 		break; /* pid is unique */
 	    }
 	    else
@@ -861,7 +965,7 @@ clixon_process_waitpid(clicon_handle h)
     	pe = NEXTQ(process_entry_t *, pe);
     } while (pe != _proc_entry_list);
     retval = 0;
-    // done:
+ done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     return retval;
 }
diff --git a/test/lib.sh b/test/lib.sh
index 50f45ced..4c8fc2ed 100755
--- a/test/lib.sh
+++ b/test/lib.sh
@@ -72,8 +72,12 @@ testname=
 # eg logging to a file: RCLOG="-l f/www-data/restconf.log"
 : ${RCLOG:=}
 
+# Namespace: netconf base
 BASENS='urn:ietf:params:xml:ns:netconf:base:1.0'
 
+# Namespace: Clixon lib
+LIBNS='xmlns="http://clicon.org/lib"'
+
 # Default netconf namespace statement, typically as placed on top-level <rpc xmlns=""
 DEFAULTONLY="xmlns=\"$BASENS\""
 
@@ -230,6 +234,7 @@ fi
 # error and exit,
 # arg1: expected
 # arg2: errmsg[optional]
+# Assumes: $dir and $expect are set
 function err(){
   echo -e "\e[31m\nError in Test$testnr [$testname]:"
   if [ $# -gt 0 ]; then 
@@ -248,6 +253,20 @@ function err(){
   exit -1 #$testnr
 }
 
+# Dont print diffs
+function err1(){
+  echo -e "\e[31m\nError in Test$testnr [$testname]:"
+  if [ $# -gt 0 ]; then 
+      echo "Expected: $1"
+      echo
+  fi
+  if [ $# -gt 1 ]; then 
+      echo "Received: $2"
+  fi
+  echo -e "\e[0m"
+  exit -1 #$testnr
+}
+
 # Test is previous test had valgrind errors if so quit
 function checkvalgrind(){
     if [ -f $valgrindfile ]; then
@@ -318,7 +337,7 @@ function start_restconf(){
     echo "sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $*"
     sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $* &
     if [ $? -ne 0 ]; then
-	err
+	err1 "expected 0" "$?"
     fi
 }
 
@@ -352,7 +371,7 @@ function wait_restconf(){
     while [[ $hdr != *"200 OK"* ]]; do
 #	echo "wait_restconf $i"
 	if [ $i -ge $DEMLOOP ]; then
-	    err "restconf timeout $DEMWAIT seconds"
+	    err1 "restconf timeout $DEMWAIT seconds"
 	fi
 	sleep $DEMSLEEP
 	hdr=$(curl $CURLOPTS $* $RCPROTO://localhost/restconf 2> /dev/null)
diff --git a/test/test_restconf.sh b/test/test_restconf.sh
index 924650ca..c6b355dc 100755
--- a/test/test_restconf.sh
+++ b/test/test_restconf.sh
@@ -51,9 +51,11 @@ if [ "${WITH_RESTCONF}" = "native" ]; then
     # Create server certs and CA
     cacerts $cakey $cacert
     servercerts $cakey $cacert $srvkey $srvcert
+    USEBACKEND=true
 else
     # Define default restconfig config: RESTCONFIG
     RESTCONFIG=$(restconf_config none false)
+    USEBACKEND=false
 fi
 
 # This is a fixed 'state' implemented in routing_backend. It is assumed to be always there
@@ -111,6 +113,7 @@ cat <<EOF > $cfg
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
+  <CLICON_BACKEND_RESTCONF_PROCESS>$USEBACKEND</CLICON_BACKEND_RESTCONF_PROCESS>
   $RESTCONFIG <!-- only fcgi -->
 </clixon-config>
 EOF
diff --git a/test/test_restconf_rpc.sh b/test/test_restconf_internal.sh
similarity index 64%
rename from test/test_restconf_rpc.sh
rename to test/test_restconf_internal.sh
index 993d3521..b335561b 100755
--- a/test/test_restconf_rpc.sh
+++ b/test/test_restconf_internal.sh
@@ -7,9 +7,8 @@
 # - on enable change, make the state as configured
 # - No restconf config means enable: false (extra rule)
 # See test_restconf_netns for network namespaces
+# See test_restconf_internal_cases for some special use-cases
 # XXX Lots of sleeps to remove race conditions. I am sure there are others way to fix this
-# XXX It is wrong to use $RESTCONF in clixon-config when using CLICON_BACKEND_RESTCONF_PROCESS
-# XXX the tests should be rewritten to use running datastore
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@@ -19,14 +18,21 @@ APPNAME=example
 cfg=$dir/conf.xml
 startupdb=$dir/startup_db
 
-# Define default restconfig config: RESTCONFIG
-RESTCONFIG=$(restconf_config none false)
+# Restconf debug
+RESTCONFDBG=$DBG
+RCPROTO=http # no ssl here
 
+if [ "${WITH_RESTCONF}" = "fcgi" ]; then
+    EXTRACONF="<CLICON_FEATURE>clixon-restconf:fcgi</CLICON_FEATURE>"
+else
+    EXTRACONF=""
+fi
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE> <!-- Use auth-type=none -->
+  $EXTRACONF
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
@@ -42,7 +48,6 @@ cat <<EOF > $cfg
   <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
   <!-- start restconf from backend -->
   <CLICON_BACKEND_RESTCONF_PROCESS>true</CLICON_BACKEND_RESTCONF_PROCESS>
-  $RESTCONFIG
 </clixon-config>
 EOF
 
@@ -58,65 +63,89 @@ module example {
 EOF
 
 # Subroutine send a process control RPC and tricks to echo process-id returned
-# Args:
-# 1: operation
-# 2: expectret  0: means expect pi 0 as return, else something else
-function testrpc()
+# Args, expected values of:
+# 0: ACTIVE: true or false
+# 1: STATUS: stopped/running/exiting
+# retvalue:
+# $pid
+function rpcstatus()
 {
-    operation=$1
-    expectret=$2
+    if [ $# -ne 2 ]; then
+	err1 "rpcstatus: # arguments: 2" "$#"
+    fi
+    active=$1
+    status=$2
     
     sleep $DEMSLEEP
-    new "send rpc $operation"
+    new "send rpc status"
     ret=$($clixon_netconf -qf $cfg<<EOF
 $DEFAULTHELLO
 <rpc $DEFAULTNS>
   <process-control xmlns="http://clicon.org/lib">
     <name>restconf</name>
-    <operation>$operation</operation>
+    <operation>status</operation>
   </process-control>
 </rpc>]]>]]>
 EOF
 )
-
-#    >&2 echo "ret:$ret" # debug
-
-    expect1="<pid xmlns=\"http://clicon.org/lib\">[0-9]*</pid>"
-    match=$(echo "$ret" | grep --null -Go "$expect1")
-#    >&2 echo "match:$match" # debug
+    # Check pid
+    expect="<pid xmlns=\"http://clicon.org/lib\">[0-9]*</pid>"
+    match=$(echo "$ret" | grep --null -Go "$expect")
     if [ -z "$match" ]; then
 	pid=0
     else
 	pid=$(echo "$match" | awk -F'[<>]' '{print $3}')
     fi
-    >&2 echo "pid:$pid" # debug
-
     if [ -z "$pid" ]; then
-	err "Running process" "$ret"
+	err "No pid return value" "$ret"
     fi
+    if $active; then
+	expect="^<rpc-reply $DEFAULTNS><active $LIBNS>$active</active><description $LIBNS>Clixon RESTCONF process</description><command $LIBNS>/www-data/clixon_restconf -f $cfg -D [0-9]</command><status $LIBNS>$status</status><starttime $LIBNS>20[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9]*Z</starttime><pid $LIBNS>$pid</pid></rpc-reply>]]>]]>$"
+    else
+	# inactive, no startime or pid
+	expect="^<rpc-reply $DEFAULTNS><active $LIBNS>$active</active><description $LIBNS>Clixon RESTCONF process</description><command $LIBNS>/www-data/clixon_restconf -f $cfg -D [0-9]</command><status $LIBNS>$status</status></rpc-reply>]]>]]>$"
+    fi
+    match=$(echo "$ret" | grep --null -Go "$expect")
+    if [ -z "$match" ]; then
+	err "$expect" "$ret"
+    fi
+}
+
+# Subroutine send a process control RPC and tricks to echo process-id returned
+# Args:
+# 1: operation   One of stop/start/restart
+function rpcoperation()
+{
+    operation=$1
+    
+    sleep $DEMSLEEP
+    new "send rpc $operation"
+    expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><process-control xmlns=\"http://clicon.org/lib\"><name>restconf</name><operation>$operation</operation></process-control></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok xmlns=\"http://clicon.org/lib\"/></rpc-reply>]]>]]>$"
 
-    new "check restconf retvalue"
-    if [ $operation = "status" ]; then
-	if [ $expectret -eq 0 ]; then
-	    if [ $pid -ne 0 ]; then
-		err "No process" "$pid"
-	    fi
-	else
-	    if [ $pid -eq 0 ]; then
-		err "Running process"
-	    fi
-	fi
-	echo "$pid" # cant use return that only uses 0-255
-    fi
     sleep $DEMSLEEP
 }
 
+# This test is confusing:
+# The whole restconf config is in clixon-config wich binds 0.0.0.0:80 which will be the only
+# config the restconf daemon ever reads.
+# However, enable (and debug) flag is stored in running db but only backend will ever read that.
+# It just controls how restconf is started, but thereafter the restconf daemon reads the static db in clixon-config file
+
 new "ENABLE true"
 # First basic operation with restconf enable is true
 cat<<EOF > $startupdb
 <${DATASTORE_TOP}>
    <restconf xmlns="http://clicon.org/restconf">
       <enable>true</enable>
+      <auth-type>none</auth-type>
+      <pretty>false</pretty>
+      <debug>$RESTCONFDBG</debug>
+      <socket>
+         <namespace>default</namespace>
+	 <address>0.0.0.0</address>
+	 <port>80</port>
+	 <ssl>false</ssl>
+      </socket>
    </restconf>
 </${DATASTORE_TOP}>
 EOF
@@ -134,18 +163,19 @@ if [ $BE -ne 0 ]; then
 
     new "start backend -s startup -f $cfg"
     start_backend -s startup -f $cfg
-
-    new "wait backend"
-    wait_backend
 fi
 
+new "wait backend"
+wait_backend
+
 # For debug
 #>&2 echo "curl $CURLOPTS -X POST -H \"Content-Type: application/yang-data+json\" $RCPROTO://localhost/restconf/operations/clixon-lib:process-control -d '{\"clixon-lib:input\":{\"name\":\"restconf\",\"operation\":\"status\"}}'"
 
 # Get pid of running process and check return xml
 new "1. Get rpc status"
-pid0=$(testrpc status 1) # Save pid0
-if [ $? -ne 0 ]; then echo "$pid0";exit -1; fi
+rpcstatus true running 
+pid0=$pid # Save pid0
+if [ $pid0 -eq 0 ]; then err "Pid" 0; fi
 
 new "check restconf process runnng using ps pid:$pid0"
 ps=$(ps -hp $pid0) 
@@ -167,8 +197,9 @@ new "try restconf rpc status"
 expectpart "$(curl $CURLOPTS -X POST -H "Content-Type: application/yang-data+json" $RCPROTO://localhost/restconf/operations/clixon-lib:process-control -d '{"clixon-lib:input":{"name":"restconf","operation":"status"}}')" 0 "HTTP/1.1 200 OK" '{"clixon-lib:output":' '"active":' '"pid":'
 
 new "2. Get status"
-pid1=$(testrpc status 1)
-if [ $? -ne 0 ]; then echo "$pid1";exit -1; fi
+rpcstatus true running
+pid1=$pid
+if [ $pid1 -eq 0 ]; then err "pid" 0; fi
 
 new "Check same pid"
 if [ "$pid0" -ne "$pid1" ]; then
@@ -179,29 +210,31 @@ new "try restconf rpc restart"
 expectpart "$(curl $CURLOPTS -X POST -H "Content-Type: application/yang-data+json" $RCPROTO://localhost/restconf/operations/clixon-lib:process-control -d '{"clixon-lib:input":{"name":"restconf","operation":"restart"}}')" 0 "HTTP/1.1 204 No Content"
 
 new "3. Get status"
-pid1=$(testrpc status 1)
-if [ $? -ne 0 ]; then echo "$pid1";exit -1; fi
+rpcstatus true running
+pid1=$pid
+if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
 
 new "check different pids"
 if [ "$pid0" -eq "$pid1" ]; then
-    err "not $pid0"
+    err1 "not $pid0" "$pid1"
 fi
 
 new "4. stop restconf RPC"
-testrpc stop 0
+rpcoperation stop
 if [ $? -ne 0 ]; then exit -1; fi
 
 new "5. Get rpc status stopped"
-pid=$(testrpc status 0)
-if [ $? -ne 0 ]; then echo "$pid";exit -1; fi
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 new "6. Start rpc again"
-testrpc start 0
+rpcoperation start
 if [ $? -ne 0 ]; then exit -1; fi
 
 new "7. Get rpc status"
-pid3=$(testrpc status 1)
-if [ $? -ne 0 ]; then echo "$pid3";exit -1; fi
+rpcstatus true running
+pid3=$pid
+if [ $pid3 -eq 0 ]; then err "Pid" 0; fi
 
 new "check restconf process running using ps"
 ps=$(ps -hp $pid3)
@@ -210,30 +243,32 @@ if [ -z "$ps" ]; then
 fi
 
 if [ $pid0 -eq $pid3 ]; then
-    err "A different pid" "same pid: $pid3"
+    err1 "A different pid" "same pid: $pid3"
 fi
 
 new "kill restconf"
 stop_restconf_pre
 
 new "8. start restconf RPC"
-testrpc start 0
+rpcoperation start
 if [ $? -ne 0 ]; then exit -1; fi
 
 new "9. check status RPC on"
-pid5=$(testrpc status 1) # Save pid5
-if [ $? -ne 0 ]; then echo "$pid5";exit -1; fi
+rpcstatus true running
+pid5=$pid
+if [ $pid5 -eq 0 ]; then err "Pid" 0; fi
 
 new "10. restart restconf RPC"
-testrpc restart 0
+rpcoperation restart
 if [ $? -ne 0 ]; then exit -1; fi
 
 new "11. Get restconf status rpc"
-pid7=$(testrpc status 1) # Save pid7
-if [ $? -ne 0 ]; then echo "$pid7";exit -1; fi
+rpcstatus true running
+pid7=$pid
+if [ $pid7 -eq 0 ]; then err "Pid" 0; fi
 
 if [ $pid5 -eq $pid7 ]; then
-    err "A different pid" "samepid: $pid7"
+    err1 "A different pid" "samepid: $pid7"
 fi
 
 if [ $BE -ne 0 ]; then
@@ -261,14 +296,17 @@ if [ $BE -ne 0 ]; then
     new "start backend -s none -f $cfg"
     start_backend -s none -f $cfg
 
-    new "waiting"
+    new "wait backend"
     wait_backend
 fi
 
-new "12. Get restconf (running) after restart"
-pid=$(testrpc status 1)
-if  [ valgrindtest -ne 2 ]; then # XXX does not work w backend valgrind test
-    if [ $? -ne 0 ]; then echo "$pid"; exit -1; fi
+new "wait restconf"
+wait_restconf
+
+if  [ $valgrindtest -ne 2 ]; then # Restart with same restconf pid does not work w backend valgrind test
+    new "12. Get restconf (running) after restart"
+    rpcstatus true running
+    if [ $pid -eq 0 ]; then err "Pid" 0; fi
 fi
 
 if [ $BE -ne 0 ]; then
@@ -283,14 +321,23 @@ if [ $BE -ne 0 ]; then
 fi
 #--------------------------
 
-# So far, restconf config enable flag has been true. Now change enable flag.
+# Now start with enable=false
 
-new "ENABLE false"
+new "enable false"
 # Second basic operation with restconf enable is false
 cat<<EOF > $startupdb
 <${DATASTORE_TOP}>
    <restconf xmlns="http://clicon.org/restconf">
       <enable>false</enable>
+      <auth-type>none</auth-type>
+      <pretty>false</pretty>
+      <debug>$RESTCONFDBG</debug>
+      <socket>
+         <namespace>default</namespace>
+	 <address>0.0.0.0</address>
+	 <port>80</port>
+	 <ssl>false</ssl>
+      </socket>
    </restconf>
 </${DATASTORE_TOP}>
 EOF
@@ -307,68 +354,74 @@ if [ $BE -ne 0 ]; then
     fi
     new "start backend -s startup -f $cfg"
     start_backend -s startup -f $cfg
-
-    new "waiting"
-    wait_backend
 fi
 
+new "wait backend"
+wait_backend
+
 new "13. check status RPC off"
-pid=$(testrpc status 0)
-if [ $? -ne 0 ]; then echo "$pid";exit -1; fi
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
-new "14. start restconf RPC"
-testrpc start 0
+new "14. start restconf RPC (but disabled)"
+rpcoperation start
 if [ $? -ne 0 ]; then exit -1; fi
 
-new "15. check status RPC off"
-pid=$(testrpc status 0)
-if [ $? -ne 0 ]; then echo "$pid";exit -1; fi
+new "15. check status RPC still off"
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 new "Enable restconf"
-expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><default-operation>merge</default-operation><target><candidate/></target><config><restconf xmlns=\"http://clicon.org/restconf\"><enable>true</enable></restconf></config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><default-operation>merge</default-operation><target><candidate/></target><config><restconf xmlns=\"http://clicon.org/restconf\"><enable>true</enable><debug>$RESTCONFDBG</debug></restconf></config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-new "netconf commit"
+new "commit enable"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-sleep $DEMSLEEP
-
 new "16. check status RPC on"
-pid=$(testrpc status 1)
-if [ $? -ne 0 ]; then echo "$pid";exit -1; fi
+rpcstatus true running
+pid1=$pid
+if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
 
-# Edit a field, eg debug
-new "Edit a restconf field via restconf"
-expectpart "$(curl $CURLOPTS -X PUT -H "Content-Type: application/yang-data+json" $RCPROTO://localhost/restconf/data/clixon-restconf:restconf/debug -d '{"clixon-restconf:debug":1}' )" 0 "HTTP/1.1 201 Created"
+new "wait restconf"
+wait_restconf
+
+# Edit a field, eg pretty to trigger a restart
+new "Edit a restconf field via restconf" # XXX fcgi fails here
+expectpart "$(curl $CURLOPTS -X PUT -H "Content-Type: application/yang-data+json" $RCPROTO://localhost/restconf/data/clixon-restconf:restconf/pretty -d '{"clixon-restconf:pretty":true}' )" 0 "HTTP/1.1 204 No Content"
 
 new "check status RPC new pid"
-pid1=$(testrpc status 1)
+rpcstatus true running
+pid2=$pid
+if [ $pid2 -eq 0 ]; then err "Pid" 0; fi
 
-if [ $? -ne 0 ]; then echo "$pid1";exit -1; fi
-if [ $pid -eq $pid1 ]; then
-    err "A different pid" "Same pid: $pid"
+if [ $pid1 -eq $pid2 ]; then
+    err1 "A different pid" "$pid1"
 fi
 
-sleep $DEMSLEEP
+new "wait restconf"
+wait_restconf
 
 new "Edit a non-restconf field via restconf"
 expectpart "$(curl $CURLOPTS -X POST -H "Content-Type: application/yang-data+json" $RCPROTO://localhost/restconf/data -d '{"example:val":"xyz"}' )" 0 "HTTP/1.1 201 Created"
 
-new "check status RPC same pid"
-pid2=$(testrpc status 1)
-if [ $? -ne 0 ]; then echo "$pid2";exit -1; fi
-if [ $pid1 -ne $pid2 ]; then
-    err "Same pid $pid1" "$pid2"
+new "17. check status RPC same pid"
+rpcstatus true running
+pid3=$pid
+if [ $pid3 -eq 0 ]; then err "Pid" 0; fi
+
+if [ $pid2 -ne $pid3 ]; then
+    err1 "Same pid $pid2" "$pid3"
 fi
 
 new "Disable restconf"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><default-operation>merge</default-operation><target><candidate/></target><config><restconf xmlns=\"http://clicon.org/restconf\"><enable>false</enable></restconf></config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-new "netconf commit"
+new "commit disable"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
 new "17. check status RPC off"
-pid=$(testrpc status 0)
-if [ $? -ne 0 ]; then echo "$pid";exit -1; fi
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 # Negative validation checks of clixon-restconf / socket
 
@@ -400,6 +453,8 @@ endtest
 
 # Set by restconf_config
 unset RESTCONFIG
+unset RESTCONFDBG
+unset RCPROTO
 
 rm -rf $dir
 
diff --git a/test/test_restconf_rpc2.sh b/test/test_restconf_internal_cases.sh
similarity index 72%
rename from test/test_restconf_rpc2.sh
rename to test/test_restconf_internal_cases.sh
index 4e399f9b..f3140d19 100755
--- a/test/test_restconf_rpc2.sh
+++ b/test/test_restconf_internal_cases.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Send restconf rpc:s when starting from backend
 # Two specific usecases that have been problematic are tested here
-# In comparison test_restconf_rpc.sh:
+# In comparison test_restconf_internal.sh:
 # - uses externally started restconf, here started by backend
 # - generic tests, here specific
 # The first usecases is: empty status message
@@ -26,14 +26,21 @@ cfg=$dir/conf.xml
 startupdb=$dir/startup_db
 
 # Restconf debug
-RESTCONFDBG=0
+RESTCONFDBG=$DBG
 RCPROTO=http # no ssl here
 
+if [ "${WITH_RESTCONF}" = "fcgi" ]; then
+    EXTRACONF="<CLICON_FEATURE>clixon-restconf:fcgi</CLICON_FEATURE>"
+else
+    EXTRACONF=""
+fi
+
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE> <!-- Use auth-type=none -->
+  $EXTRACONF
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
@@ -63,54 +70,54 @@ module example {
 }
 EOF
 
-function testrpc()
+# Subroutine send a process control RPC and tricks to echo process-id returned
+# Args, expected values of:
+# 0: ACTIVE: true or false
+# 1: STATUS: stopped/running/exiting
+# retvalue:
+# $pid
+# See also in test_restconf_internal.sh
+function rpcstatus()
 {
-    operation=$1
-    expectret=$2
+    if [ $# -ne 2 ]; then
+	err1 "rpcstatus: # arguments: 2" "$#"
+    fi
+    active=$1
+    status=$2
     
     sleep $DEMSLEEP
-    new "send rpc $operation"
+    new "send rpc status"
     ret=$($clixon_netconf -qf $cfg<<EOF
 $DEFAULTHELLO
 <rpc $DEFAULTNS>
   <process-control xmlns="http://clicon.org/lib">
     <name>restconf</name>
-    <operation>$operation</operation>
+    <operation>status</operation>
   </process-control>
 </rpc>]]>]]>
 EOF
 )
-
-#    >&2 echo "ret:$ret" # debug
-
-    expect1="<pid xmlns=\"http://clicon.org/lib\">[0-9]*</pid>"
-    match=$(echo "$ret" | grep --null -Go "$expect1")
-#    >&2 echo "match:$match" # debug
+    # Check pid
+    expect="<pid xmlns=\"http://clicon.org/lib\">[0-9]*</pid>"
+    match=$(echo "$ret" | grep --null -Go "$expect")
     if [ -z "$match" ]; then
 	pid=0
     else
 	pid=$(echo "$match" | awk -F'[<>]' '{print $3}')
     fi
-#    >&2 echo "pid:$pid" # debug
-
     if [ -z "$pid" ]; then
-	err "Running process" "$ret"
+	err "No pid return value" "$ret"
     fi
-
-    new "check restconf retvalue"
-    if [ $operation = "status" ]; then
-	if [ $expectret -eq 0 ]; then
-	    if [ $pid -ne 0 ]; then
-		err "No process" "$pid"
-	    fi
-	else
-	    if [ $pid -eq 0 ]; then
-		err "Running process"
-	    fi
-	fi
-	echo "$pid" # cant use return that only uses 0-255
+    if $active; then
+	expect="^<rpc-reply $DEFAULTNS><active $LIBNS>$active</active><description $LIBNS>Clixon RESTCONF process</description><command $LIBNS>/www-data/clixon_restconf -f $cfg -D [0-9]</command><status $LIBNS>$status</status><starttime $LIBNS>20[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9]*Z</starttime><pid $LIBNS>$pid</pid></rpc-reply>]]>]]>$"
+    else
+	# inactive, no startime or pid
+	expect="^<rpc-reply $DEFAULTNS><active $LIBNS>$active</active><description $LIBNS>Clixon RESTCONF process</description><command $LIBNS>/www-data/clixon_restconf -f $cfg -D [0-9]</command><status $LIBNS>$status</status></rpc-reply>]]>]]>$"
+    fi
+    match=$(echo "$ret" | grep --null -Go "$expect")
+    if [ -z "$match" ]; then
+	err "$expect" "$ret"
     fi
-    sleep $DEMSLEEP
 }
 
 # FIRST usecase
@@ -127,7 +134,6 @@ if [ $BE -ne 0 ]; then
     if [ $? -ne 0 ]; then
 	err
     fi
-
     new "start backend -s init -f $cfg"
     start_backend -s init -f $cfg
 fi
@@ -142,53 +148,33 @@ RESTCONFIG1=$(cat <<EOF
 EOF
 )
 
-LIBNS='xmlns="http://clicon.org/lib"'
-
-new "get status 1"
-expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><process-control xmlns=\"http://clicon.org/lib\"><name>restconf</name><operation>status</operation></process-control></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><active $LIBNS>false</active><description $LIBNS>Clixon RESTCONF process</description><command $LIBNS>/www-data/clixon_restconf -f $cfg -D $RESTCONFDBG</command></rpc-reply>]]>]]>$"
+new "1. get status"
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 new "enable minimal restconf, no server"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><target><candidate/></target><config>$RESTCONFIG1</config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-new "netconf commit"
+new "commit minimal server"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-# Get pid2
-new "get pid"
-pid2=$(testrpc status 1)
-echo "pid2:$pid2"
+new "2. get status, get pid1"
+rpcstatus true running
+pid1=$pid
+if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
 
-new "get status 2"
-ret=$($clixon_netconf -qf $cfg<<EOF
-$DEFAULTHELLO
-<rpc $DEFAULTNS>
-  <process-control xmlns="http://clicon.org/lib">
-    <name>restconf</name>
-    <operation>status</operation>
-  </process-control>
-</rpc>]]>]]>
-EOF
-)
-expect="^<rpc-reply $DEFAULTNS><active $LIBNS>true</active><description $LIBNS>Clixon RESTCONF process</description><pid $LIBNS>$pid2</pid><command $LIBNS>/www-data/clixon_restconf -f $cfg -D $RESTCONFDBG</command><starttime $LIBNS>20[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9]*Z</starttime>"
-match=$(echo "$ret" | grep --null -Go "$expect")
-if [ -z "$match" ]; then
-    err "$expect" "$ret"
-fi
-
-# Kill it
-new "kill $pid2"
-sudo kill $pid2
+new "kill $pid1 externally"
+sudo kill $pid1
 sleep $DEMSLEEP
 
-# Why kill it twice?
-# I should really debug this,... it happens in docker somethimes but its not the aim of the test
-new "kill $pid2 again"
-sudo kill $pid2
+# Why kill it twice? it happens in docker somethimes but is not the aim of the test
+new "kill $pid1 again"
+sudo kill $pid1 2> /dev/null
 sleep $DEMSLEEP
 
-new "Check killed"
-# Ensure no pid
-testrpc status 0 > /dev/null
+new "3. get status: Check killed"
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 RESTCONFIG2=$(cat <<EOF
 <restconf xmlns="http://clicon.org/restconf">
@@ -196,16 +182,15 @@ RESTCONFIG2=$(cat <<EOF
 </restconf>
 EOF
 )
-new "create a server"
+new "create server"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><target><candidate/></target><config>$RESTCONFIG2</config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-new "netconf commit"
+new "commit create server"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-# 3. get status
-
-new "get status 3"
-expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><process-control xmlns=\"http://clicon.org/lib\"><name>restconf</name><operation>status</operation></process-control></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><active $LIBNS>true</active><description $LIBNS>Clixon RESTCONF process</description><pid $LIBNS>"
+new "4. get status"
+rpcstatus true running
+if [ $pid -eq 0 ]; then err "Pid" 0; fi
 
 if [ $BE -ne 0 ]; then
     new "Kill backend"
@@ -238,8 +223,9 @@ fi
 new "wait backend"
 wait_backend
 
-new "get status 1"
-testrpc status 0 > /dev/null
+new "5. get status not started"
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 RESTCONFIG1=$(cat <<EOF
 <restconf xmlns="http://clicon.org/restconf">
@@ -255,7 +241,7 @@ EOF
 new "Create server with invalid address"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><target><candidate/></target><config>$RESTCONFIG1</config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-new "netconf commit"
+new "commit invalid server"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
 sleep $DEMSLEEP
@@ -285,6 +271,9 @@ if [ -n "$ret" ]; then
     err "No zombie process" "$ret"
 fi
 
+# THIRD usecase
+# NOTE this does not apply for fcgi where servers cant be "removed"
+if [ "${WITH_RESTCONF}" != "fcgi" ]; then
 new "3. restconf not removed"
 
 new "kill old restconf"
@@ -304,8 +293,9 @@ fi
 new "wait backend"
 wait_backend
 
-new "get status 1"
-testrpc status 0 > /dev/null
+new "6. get status stopped"
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
 
 RESTCONFIG1=$(cat <<EOF
 <restconf xmlns="http://clicon.org/restconf">
@@ -325,12 +315,14 @@ new "commit create"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
 # pid
-pid1=$(testrpc status 1)
-
+new "7. get status, get pid1"
+rpcstatus true running
+pid1=$pid
+if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
 sleep $DEMSLEEP
 
 new "Get restconf config 1"
-expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' $RCPROTO://localhost/restconf/data/clixon-restconf:restconf)" 0 "HTTP/1.1 200 OK" "<restconf xmlns=\"http://clicon.org/restconf\"><enable>true</enable><auth-type>none</auth-type><debug>0</debug><pretty>false</pretty><socket><namespace>default</namespace><address>0.0.0.0</address><port>80</port><ssl>false</ssl></socket></restconf>"
+expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' $RCPROTO://localhost/restconf/data/clixon-restconf:restconf)" 0 "HTTP/1.1 200 OK" "<restconf xmlns=\"http://clicon.org/restconf\"><enable>true</enable><auth-type>none</auth-type><debug>$RESTCONFDBG</debug><pretty>false</pretty><socket><namespace>default</namespace><address>0.0.0.0</address><port>80</port><ssl>false</ssl></socket></restconf>"
 
 # remove it
 new "Delete server"
@@ -339,15 +331,21 @@ expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS xmlns:nc=\"
 new "commit delete"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
-new "Get restconf config 2"
-expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' $RCPROTO://localhost/restconf/data/clixon-restconf:restconf)" 0 "HTTP/1.1 200 OK" "<restconf xmlns=\"http://clicon.org/restconf\"><enable>true</enable><auth-type>none</auth-type><debug>0</debug><pretty>false</pretty></restconf>"
+# Here restconf should have been restarted with no listener, the process is up but does not
+# reply on restconf
 
-pid2=$(testrpc status 1)
+new "8. get status, get different pid2"
+rpcstatus true running
+pid2=$pid
+if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
 
-if [ $pid1 -ne $pid2 ]; then
-    err "Same pid:$pid1" "$pid2"
+if [ $pid1 -eq $pid2 ]; then
+    err1 "A different pid" "same pid: $pid1"
 fi
 
+new "Get restconf config 2: no server"
+expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' $RCPROTO://localhost/restconf/data/clixon-restconf:restconf 2>&1)" 7 "Failed to connect" "Connection refused"
+
 if [ $BE -ne 0 ]; then
     new "Kill backend"
     # Check if premature kill
@@ -359,6 +357,8 @@ if [ $BE -ne 0 ]; then
     stop_backend -f $cfg
 fi
 
+fi # "${WITH_RESTCONF}" != "fcgi"
+
 new "endtest"
 endtest
 
diff --git a/yang/clixon/clixon-config@2021-03-08.yang b/yang/clixon/clixon-config@2021-03-08.yang
index 1dbf3ba7..5f30e9d1 100644
--- a/yang/clixon/clixon-config@2021-03-08.yang
+++ b/yang/clixon/clixon-config@2021-03-08.yang
@@ -726,6 +726,8 @@ module clixon-config {
 	    description
 		"If set, enable process-control of restconf daemon, ie start/stop restconf 
                  daemon internally from backend daemon.
+                 Also, if set, restconf daemon queries backend for its config
+                 if not set, restconf daemon reads its config from main config file
                  It uses clixon-restconf.yang for config and clixon-lib.yang for RPC
                  Process control of restconf daemon is as follows:
                  - on RPC start, if enable is true, start the service, if false, error or ignore it
diff --git a/yang/clixon/clixon-lib@2021-03-08.yang b/yang/clixon/clixon-lib@2021-03-08.yang
index c6fb9830..44c5c0c8 100644
--- a/yang/clixon/clixon-lib@2021-03-08.yang
+++ b/yang/clixon/clixon-lib@2021-03-08.yang
@@ -184,7 +184,10 @@ module clixon-lib {
 		    leaf active {
 			description
 			    "True if process is running, false if not. 
-                             More specifically, there is a process-id and it exists (in Linux: kill(pid,0)";
+                             More specifically, there is a process-id and it exists (in Linux: kill(pid,0).
+                             Note that this is actual state and status is administrative state,
+                             which means that changing the administrative state, eg stopped->running
+                             may not immediately switch active to true.";
 			type boolean;
 		    }
 		    leaf description {
@@ -196,8 +199,13 @@ module clixon-lib {
 			description "Start command with arguments";
 		    }
 		    leaf status {
-			description "Exit Status as defined by waitpid() - if exited";
-			type int32;
+			description
+			    "Administrative status (except on external kill where it enters stopped
+                             directly from running):
+                             stopped: pid=0,   No process running
+                             running: pid set, Process started and believed to be running
+                             exiting: pid set, Process is killed by parent but not waited for";
+			type string;
 		    }
 		    leaf starttime {
 			description "Time of starting process UTC";