Added functionality to restart an individual plugin.: New clixon-lib:restart-plugin RPC

CHANGELOG.md

@@ -50,6 +50,8 @@ Expected: May 2020
 
 ### Minor changes
 
+* Added functionality to restart an individual plugin.
+  * New clixon-lib:restart-plugin RPC
 * Added option `CLICON_YANG_UNKNOWN_ANYDATA` to treat unknown XML (wrt YANG) as anydata.
   * This is to be (very) forgiving but you need to accept eg unsynchronized YANG and XML
 * Compile-time option: `USE_CLIGEN44` for running clixon-45 with cligen-44.

apps/backend/backend_commit.c

@@ -811,6 +811,10 @@ from_client_validate(clicon_handle h,
 } /* from_client_validate */
 
 #ifdef RESTART_PLUGIN_RPC
+/*! Restart specific backend plugins without full backend restart
+ * Note, depending on plugin callbacks, there may be other dependencies which may make this
+ * difficult in the general case.
+ */
 int
 from_client_restart_one(clicon_handle h,
 			clixon_plugin *cp,
@@ -820,7 +824,6 @@ from_client_restart_one(clicon_handle h,
     char               *db = "tmp";
     transaction_data_t *td = NULL;
     plgreset_t         *resetfn;          /* Plugin auth */
-    trans_cb_t         *fn;
     int                 ret;
     cxobj              *xerr = NULL;
     yang_stmt          *yspec;
@@ -889,17 +892,10 @@ from_client_restart_one(clicon_handle h,
 	xml_apply_ancestor(xn, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
     }
     
-    /* 4. Call plugin transaction start callbacks */
+    /* Call plugin transaction start callbacks */
-    if ((fn = cp->cp_api.ca_trans_begin) != NULL){
+    if (plugin_transaction_begin_one(cp, h, td) < 0)
-	if ((retval = fn(h, (transaction_data)td)) < 0){
-	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
-		clicon_log(LOG_NOTICE, "%s: Plugin '%s' transaction_begin callback does not make clicon_err call on error", 
-			   __FUNCTION__, cp->cp_name);
-	    
 	goto fail;
-	}
+    /* Make generic validation on all new or changed data.
-    }
-    /* 5. Make generic validation on all new or changed data.
        Note this is only call that uses 3-values */
     if ((ret = generic_validate(h, yspec, td, &xerr)) < 0)
 	goto done;
@@ -908,42 +904,25 @@ from_client_restart_one(clicon_handle h,
 	    goto done;
 	goto fail;
     }
-    if ((fn = cp->cp_api.ca_trans_validate) != NULL){
+    /* Call validate callback in this plugin */
-	if ((retval = fn(h, (transaction_data)td)) < 0){
+    if (plugin_transaction_validate_one(cp, h, td) < 0)
-	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
-		clicon_log(LOG_NOTICE, "%s: Plugin '%s' transaction_validate callback does not make clicon_err call on error", 
-			   __FUNCTION__, cp->cp_name);
 	goto fail;
-	}
+    if (plugin_transaction_complete_one(cp, h, td) < 0)
-    }
-    if ((fn = cp->cp_api.ca_trans_complete) != NULL){
-	if ((retval = fn(h, (transaction_data)td)) < 0){
-	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
-		clicon_log(LOG_NOTICE, "%s: Plugin '%s' trans_complete callback does not make clicon_err call on error", 
-			   __FUNCTION__, cp->cp_name);
-	    
 	goto fail;
-	}
+    /* Call commit callback in this plugin */
-    }
+    if (plugin_transaction_commit_one(cp, h, td) < 0)
-    
-    if ((fn = cp->cp_api.ca_trans_commit) != NULL){
-	if (fn(h, (transaction_data)td) < 0){
-	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
-		clicon_log(LOG_NOTICE, "%s: Plugin '%s' trans_commit callback does not make clicon_err call on error", 
-			   __FUNCTION__, cp->cp_name);
 	goto fail;
-	}
+    if (plugin_transaction_commit_done_one(cp, h, td) < 0)
-    }
+	goto fail;
-    if ((fn = cp->cp_api.ca_trans_end) != NULL){
+    /* Finalize */
-	if ((retval = fn(h, (transaction_data)td)) < 0){
+    if (plugin_transaction_end_one(cp, h, td) < 0)
-	    if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
-		clicon_log(LOG_NOTICE, "%s: Plugin '%s' trans_end callback does not make clicon_err call on error", 
-			   __FUNCTION__, cp->cp_name);
 	goto fail;
-	}
-    }
     retval = 1;
  done:
+    if (td){
+	 xmldb_get0_free(h, &td->td_target);
+	 transaction_free(td);
+     }
     return retval;
  fail:
     retval = 0;

example/main/example_backend.c

@@ -65,23 +65,25 @@
 /*! Variable to control if reset code is run.
  * The reset code inserts "extra XML" which assumes ietf-interfaces is
  * loaded, and this is not always the case.
- * Therefore, the backend must be started with -- -r to enable the reset function
+ * Start backend with -- -r
  */
 static int _reset = 0;
 
 /*! Variable to control if state code is run
  * The state code adds extra non-config data
- * Therefore, the backend must be started with -- -s to enable the state function
+ * Start backend with -- -s
  */
 static int _state = 0;
 
 /*! File where state XML is read from, if _state is true -- -sS <file>
  * Primarily for testing
+ * Start backend with -- -sS <file>
  */
 static char *_state_file = NULL;
 
 /*! Read state file init on startup instead of on request
  * Primarily for testing
+ * Start backend with -- -siS <file>
  */
 static int _state_file_init = 0;
 static cxobj *_state_xstate = NULL;
@@ -89,15 +91,18 @@ static cxobj *_state_xstate = NULL;
 /*! Variable to control module-specific upgrade callbacks.
  * If set, call test-case for upgrading ietf-interfaces, otherwise call 
  * auto-upgrade
+ * Start backend with -- -u
  */
 static int _module_upgrade = 0;
 
 /*! Variable to control general-purpose upgrade callbacks.
+ * Start backend with -- -U
  */
 static int _general_upgrade = 0;
 
 /*! Variable to control transaction logging (for debug)
  * If set, call syslog for every transaction callback 
+ * Start backend with -- -t
  */
 static int _transaction_log = 0;
 

example/main/example_backend_nacm.c

@@ -63,6 +63,7 @@
 
 /*! Variable to control transaction logging (for debug)
  * If set, call syslog for every transaction callback
+ * Start backend with -- -t
  */
 static int _transaction_log = 0;
 

include/clixon_custom.h

@@ -105,7 +105,7 @@
  * Note, depending on plugin callbacks, there may be other dependencies which may make this
  * difficult in the general case.
  */
-#undef RESTART_PLUGIN_RPC
+#define RESTART_PLUGIN_RPC
 
 /*! Differentiate creating XML object body/element vs elenmet to reduce space
  */

test/test_transaction.sh

@@ -2,6 +2,8 @@
 # Transaction functionality
 # The test uses two backend plugins (main and nacm) that logs to a file and a
 # netconf client to push operation. The tests then look at the log.
+# The test assumes the two plugins recognize the -- -t argument which includes
+# that one of them fails at validation at one point
 # The tests are as follows (first five only callbacks per se; then data vector tests)
 # 1. Validate-only transaction
 # 2. Commit transaction
@@ -19,9 +21,6 @@
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
-# Which format to use as datastore format internally
-: ${format:=xml}
-
 APPNAME=example
 
 cfg=$dir/conf_yang.xml
@@ -77,11 +76,12 @@ cat <<EOF > $cfg
   <CLICON_SOCK>$dir/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
-  <CLICON_XMLDB_FORMAT>$format</CLICON_XMLDB_FORMAT>
 </clixon-config>
 EOF
 
 # Check statements in log
+# arg1: a statement to look for
+# arg2: expected line number
 checklog(){
     s=$1 # statement
     l0=$2 # linenr
@@ -321,5 +321,3 @@ stop_backend -f $cfg
 
 rm -rf $dir
 
-# unset conditional parameters 
-unset format

test/test_transaction_restart.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# Transaction functionality: restart single plugin and observe that only that plugin
+# gets callbacks
+# The test uses two backend plugins (main and nacm) that logs.
+# nacm is then restarted, not main
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyang=$dir/trans.yang
+flog=$dir/backend.log
+touch $flog
+
+# Used as a trigger for user-validittion errors, eg <a>$errnr</a> is invalid
+errnr=42
+
+cat <<EOF > $fyang
+module trans{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix ex;
+   /* Generic config data */
+   container table{
+      list parameter{
+         key name;
+	 leaf name{
+	    type string;
+	 }
+	 leaf value{
+	    type string;
+	 }
+      }
+   }
+}
+EOF
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
+  <!--CLICON_BACKEND_REGEXP>example_backend.so$</CLICON_BACKEND_REGEXP-->
+  <CLICON_NETCONF_DIR>/usr/local/lib/$APPNAME/netconf</CLICON_NETCONF_DIR>
+  <CLICON_RESTCONF_DIR>/usr/local/lib/$APPNAME/restconf</CLICON_RESTCONF_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>$dir/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+</clixon-config>
+EOF
+
+# Check statements in log
+# arg1: a statement to look for
+# arg2: expected line number
+checklog(){
+    s=$1 # statement
+    l0=$2 # linenr
+    new "Check $s in log"
+#    echo "grep \"transaction_log $s line:$l0\"  $flog"
+    t=$(grep -n "transaction_log $s" $flog)
+
+    if [ -z "$t" ]; then
+	echo -e "\e[31m\nError in Test$testnr [$testname]:"
+	if [ $# -gt 0 ]; then 
+	    echo "Not found in log"
+	    echo
+	fi
+	echo -e "\e[0m"
+	exit -1
+    fi
+    l1=$(echo "$t" | awk -F ":" '{print $1}')
+    if [ $l1 -ne $l0 ]; then
+	echo -e "\e[31m\nError in Test$testnr [$testname]:"
+	if [ $# -gt 0 ]; then 
+	    echo "Expected match on line $l0, found on $l1"
+	    echo
+	fi
+	echo -e "\e[0m"
+	exit -1
+    fi
+}
+
+new "test params: -f $cfg -l f$flog -- -t" # Fail on this
+# Bring your own backend
+if [ $BE -ne 0 ]; then
+    # kill old backend (if any)
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+    new "start backend  -s init -f $cfg -l f$flog -- -t /foo"
+    start_backend -s init -f $cfg -l f$flog -- -t /foo # -t means transaction logging (foo is dummy)
+
+    new "waiting"
+    wait_backend
+fi
+
+let nr=0
+
+new "Basic transaction to add top-level x"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><edit-config><target><candidate/></target><config><table xmlns='urn:example:clixon'><parameter><name>$nr</name></parameter></table></config></edit-config></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
+
+new "Commit base"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><commit/></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
+
+let line=13 # Skipping basic transaction. Sanity check, find one last transaction
+xml="<table xmlns=\"urn:example:clixon\"><parameter><name>0</name></parameter></table>"
+checklog "$nr nacm_end add: $xml" $line
+
+new "Send restart nacm plugin"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><restart-plugin xmlns="http://clicon.org/lib"><plugin>example_backend_nacm</plugin></restart-plugin></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>'
+
+# Now analyze log:
+# all transactions come from nacm plugin only.
+let nr++
+let line=14
+
+for op in begin validate complete commit commit_done end; do
+    checklog "$nr nacm_$op add: $xml" $line
+    let line++
+done
+
+# Negative test: restart a plugin that does not exist
+new "Send restart to nonexistatn plugin expect fail"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><restart-plugin xmlns="http://clicon.org/lib"><plugin>xxx</plugin></restart-plugin></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>bad-element</error-tag><error-info><bad-element>plugin</bad-element></error-info><error-severity>error</error-severity><error-message>No such plugin</error-message></rpc-error></rpc-reply>]]>]]>$'
+
+if [ $BE -eq 0 ]; then
+    exit # BE
+fi
+
+new "Kill backend"
+# Check if premature kill
+pid=$(pgrep -u root -f clixon_backend)
+if [ -z "$pid" ]; then
+    err "backend already dead"
+fi
+# kill backend
+stop_backend -f $cfg
+
+rm -rf $dir
+