- Restconf bind: continue with other sockets if bind fails, print address on log

- Multi-yang: Look in CLICON_YANG_MAIN_DIR, not only CLICON_MAIN_DIRs for old yangs
  - Backend -q quit option enhanced for multi-yang case

apps/backend/backend_commit.c

@@ -208,6 +208,14 @@ startup_common(clicon_handle       h,
 	if ((ret = xmldb_get0(h, db, YB_MODULE, NULL, "/", 0, &xt, msdiff, &xerr)) < 0)
 	    goto done;
 	if (ret == 0){     /* ret should not be 0 */
+	    /* Print upgraded db: -q backend switch for debugging/ showing upgraded config only */
+	    if (clicon_quit_upgrade_get(h) == 1){
+		xml_print(stderr, xerr);
+		clicon_err(OE_XML, 0, "invalid configuration before upgrade");
+		exit(0);  /* This is fairly abrupt , but need to avoid side-effects of rewinding 
+			   * See similar clause below
+			   */
+	    }
 	    if (clicon_xml2cbuf(cbret, xerr, 0, 0, -1) < 0)
 		goto done;
 	    goto fail;
@@ -236,10 +244,10 @@ startup_common(clicon_handle       h,
 	if (ret == 0)
 	    goto fail;
     }
-    /* Print upgraded db: -q backend switch */
+    /* Print upgraded db: -q backend switch for debugging/ showing upgraded config only */
     if (clicon_quit_upgrade_get(h) == 1){
 	/* bind yang */
-	if ((ret = (xml_bind_yang(xt, YB_MODULE, yspec, &xret)) < 1)){
+	if ((ret = xml_bind_yang(xt, YB_MODULE, yspec, &xret)) < 1){
 	    if (ret == 0){
 		/* invalid */
 		clicon_err(OE_XML, EFAULT, "invalid configuration");
@@ -249,7 +257,6 @@ startup_common(clicon_handle       h,
 		xml_print(stderr, xret);
 		clicon_err(OE_XML, 0, "%s: YANG binding error", __func__);
 	    }
-
 	}	/* sort yang */
 	else if (xml_sort_recurse(xt) < 0) {
 	    clicon_err(OE_XML, EFAULT, "Yang sort error");

apps/cli/cli_common.c

@@ -682,7 +682,7 @@ compare_xmls(cxobj *xc1,
     close(fd);
 
     if ((fd = mkstemp(filename2)) < 0){
-	clicon_err(OE_UNDEF, errno, "mkstemp: %s", strerror (errno));
+	clicon_err(OE_UNDEF, errno, "mkstemp: %s", strerror(errno));
 	goto done;
     }
     if ((f = fdopen(fd, "w")) == NULL)

apps/restconf/restconf_lib.c

@@ -645,7 +645,7 @@ restconf_config_init(clicon_handle h,
 /*! Create and bind restconf socket
  * 
  * @param[in]  netns0    Network namespace, special value "default" is same as NULL
- * @param[in]  addr      Address as string, eg "0.0.0.0", "::"
+ * @param[in]  addrstr   Address as string, eg "0.0.0.0", "::"
  * @param[in]  addrtype  One of inet:ipv4-address or inet:ipv6-address
  * @param[in]  port      TCP port
  * @param[in]  backlog  Listen backlog, queie of pending connections
@@ -654,7 +654,7 @@ restconf_config_init(clicon_handle h,
  */
 int
 restconf_socket_init(const char   *netns0,
-		     const char   *addr,
+		     const char   *addrstr,
 		     const char   *addrtype,
 		     uint16_t      port,
 		     int           backlog,
@@ -668,7 +668,7 @@ restconf_socket_init(const char   *netns0,
     size_t              sin_len;
     const char         *netns;
 
-    clicon_debug(1, "%s %s %s %s %hu", __FUNCTION__, netns0, addrtype, addr, port);
+    clicon_debug(1, "%s %s %s %s %hu", __FUNCTION__, netns0, addrtype, addrstr, port);
     /* netns default -> NULL */
     if (netns0 != NULL && strcmp(netns0, "default")==0)
 	netns = NULL;
@@ -679,14 +679,14 @@ restconf_socket_init(const char   *netns0,
         sin6.sin6_port   = htons(port);
         sin6.sin6_family = AF_INET6;
 
-        inet_pton(AF_INET6, addr, &sin6.sin6_addr);
+        inet_pton(AF_INET6, addrstr, &sin6.sin6_addr);
         sa = (struct sockaddr *)&sin6;
     }
     else if (strcmp(addrtype, "inet:ipv4-address") == 0) {
         sin_len             = sizeof(struct sockaddr_in);
         sin.sin_family      = AF_INET;
         sin.sin_port        = htons(port);
-        sin.sin_addr.s_addr = inet_addr(addr);
+        sin.sin_addr.s_addr = inet_addr(addrstr);
 
         sa = (struct sockaddr *)&sin;
     }
@@ -694,7 +694,7 @@ restconf_socket_init(const char   *netns0,
 	clicon_err(OE_XML, EINVAL, "Unexpected addrtype: %s", addrtype);
 	return -1;
     }
-    if (clixon_netns_socket(netns, sa, sin_len, backlog, flags, ss) < 0)
+    if (clixon_netns_socket(netns, sa, sin_len, backlog, flags, addrstr, ss) < 0)
 	goto done;
     clicon_debug(1, "%s ss=%d", __FUNCTION__, *ss);
     retval = 0;

apps/restconf/restconf_lib.h

@@ -84,7 +84,7 @@ char *restconf_uripath(clicon_handle h);
 int   restconf_drop_privileges(clicon_handle h, char *user);
 int   restconf_authentication_cb(clicon_handle h, void *req, int pretty, restconf_media media_out);
 int   restconf_config_init(clicon_handle h, cxobj *xrestconf);
-int   restconf_socket_init(const char *netns0, const char *addr, const char *addrtype, uint16_t port, int backlog, int flags, int *ss);
+int   restconf_socket_init(const char *netns0, const char *addrstr, const char *addrtype, uint16_t port, int backlog, int flags, int *ss);
 int   restconf_socket_extract(clicon_handle h, cxobj *xs, cvec *nsc, char **namespace, char **address, char **addrtype, uint16_t *port, uint16_t *ssl);
 
 #endif /* _RESTCONF_LIB_H_ */

apps/restconf/restconf_main_native.c

@@ -1338,7 +1338,6 @@ restconf_clixon_backend(clicon_handle h,
     goto done;
 }
 
-
 /*! Per-socket openssl inits
  * @param[in]  h        Clicon handle
  * @param[in]  xs       XML config of single restconf socket
@@ -1492,8 +1491,13 @@ restconf_openssl_init(clicon_handle h,
     if (xpath_vec(xrestconf, nsc, "socket", &vec, &veclen) < 0)
 	goto done;
     for (i=0; i<veclen; i++){
-	if (openssl_init_socket(h, vec[i], nsc) < 0)
+	if (openssl_init_socket(h, vec[i], nsc) < 0){
-	    goto done;
+	    /* Bind errors are ignored, proceed with next after log */
+	    if (clicon_errno == OE_UNIX && clicon_suberrno == EADDRNOTAVAIL)
+		clicon_err_reset();
+	    else
+		goto done;
+	}
     }
     retval = 1;
  done:

lib/clixon/clixon_netns.h

@@ -9,6 +9,6 @@
 /*
  * Prototypes
  */
-int clixon_netns_socket(const char *netns, struct sockaddr *sa, size_t sin_len, int backlog, int flags, int *sock);
+int clixon_netns_socket(const char *netns, struct sockaddr *sa, size_t sin_len, int backlog, int flags, const char *addrstr, int *sock);
 
 #endif  /* _CLIXON_NETNS_H_ */

lib/src/clixon_datastore_read.c

@@ -541,7 +541,7 @@ xmldb_readfile(clicon_handle    h,
 		    continue;
 		/* Add old/deleted yangs not present in the loaded/running yangspec. */
 		if ((ymod = yang_find_module_by_namespace_revision(yspec, ns, rev)) == NULL){
-		    /* Append it */
+		    /* YANG Module not found, look for it and append if found */
 		    if (yang_spec_parse_module(h, name, rev, yspec) < 0){
 			/* Special case: file-not-found errors */
 			if (clicon_suberrno == ENOENT){

lib/src/clixon_netns.c

@@ -112,6 +112,7 @@ get_sock(int  usock,
  * @param[in]  sa_len   Length of sa. Tecynicaliyu to be independent of sockaddr sa_len
  * @param[in]  backlog  Listen backlog, queie of pending connections
  * @param[in]  flags    Socket flags Or:ed in with the socket(2) type parameter
+ * @param[in]  addrstr  Address string for debug
  * @param[out] sock     Server socket (bound for accept)
  */
 static int
@@ -119,6 +120,7 @@ create_socket(struct sockaddr *sa,
 	      size_t           sin_len,		    
 	      int              backlog,
 	      int              flags,
+	      const char      *addrstr,
 	      int             *sock)
 {
     int    retval = -1;
@@ -153,7 +155,8 @@ create_socket(struct sockaddr *sa,
 	goto done;
     }
     if (bind(s, sa, sin_len) == -1) {
-	clicon_err(OE_UNIX, errno, "bind");
+	/* Note may be ignored in upper layer by checking for EADDRNOTAVAIL, see eg restconf_openssl_init */
+	clicon_err(OE_UNIX, errno, "bind(%s)", addrstr);
 	goto done;
     }
     if (listen(s, backlog ) < 0){
@@ -176,8 +179,9 @@ create_socket(struct sockaddr *sa,
  * @param[in]  netns    Network namespace
  * @param[in]  sa       Socketaddress
  * @param[in]  sa_len   Length of sa. Tecynicaliyu to be independent of sockaddr sa_len
- * @param[in]  backlog  Listen backlog, queie of pending connections
+ * @param[in]  backlog  Listen backlog, queue of pending connections
  * @param[in]  flags    Socket flags OR:ed in with the socket(2) type parameter
+ * @param[in]  addrstr  Address string for debug
  * @param[out] sock     Server socket (bound for accept)
  */
 static int
@@ -186,13 +190,14 @@ fork_netns_socket(const char      *netns,
 		  size_t           sin_len,		    
 		  int              backlog,
 		  int              flags,
+		  const char      *addrstr,
 		  int             *sock)
 {
-    int   retval = -1;
+    int         retval = -1;
-    int   sp[2] = {-1, -1};
+    int         sp[2] = {-1, -1};
-    pid_t child;
+    pid_t       child;
-    int   status = 0;
+    int         status = 0;
-    char  nspath[MAXPATHLEN]; /* Path to namespace file */
+    char        nspath[MAXPATHLEN]; /* Path to namespace file */
     struct stat st;
 
     clicon_debug(1, "%s %s", __FUNCTION__, netns);
@@ -228,7 +233,7 @@ fork_netns_socket(const char      *netns,
 #endif
 	close(fd);
 	/* Create socket in this namespace */
-	if (create_socket(sa, sin_len, backlog, flags, &s) < 0)
+	if (create_socket(sa, sin_len, backlog, flags, addrstr, &s) < 0)
 	    return -1;
 	/* Send socket to parent */
 	if (send_sock(sp[1], s) < 0)
@@ -257,6 +262,7 @@ fork_netns_socket(const char      *netns,
  * @param[in]  sa_len   Length of sa. Tecynicaliyu to be independent of sockaddr sa_len
  * @param[in]  backlog  Listen backlog, queie of pending connections
  * @param[in]  flags    Socket flags OR:ed in with the socket(2) type parameter
+ * @param[in]  addrstr  Address string for debug
  * @param[out] sock     Server socket (bound for accept)
  */
 int
@@ -265,19 +271,20 @@ clixon_netns_socket(const char      *netns,
 		    size_t           sin_len,		    
 		    int              backlog,
 		    int              flags,
+		    const char      *addrstr,
 		    int             *sock)
 {
     int    retval = -1;
     
     clicon_debug(1, "%s", __FUNCTION__);
     if (netns == NULL){
-	if (create_socket(sa, sin_len, backlog, flags, sock) < 0)
+	if (create_socket(sa, sin_len, backlog, flags, addrstr, sock) < 0)
 	    goto done;
 	goto ok;
     }
     else {
 #ifdef HAVE_SETNS
-	if (fork_netns_socket(netns, sa, sin_len, backlog, flags, sock) < 0)
+	if (fork_netns_socket(netns, sa, sin_len, backlog, flags, addrstr, sock) < 0)
 	    goto done;
 #else
 	clicon_err(OE_UNIX, errno, "No namespace support on platform: %s", netns);

lib/src/clixon_yang_parse_lib.c

@@ -828,7 +828,9 @@ yang_parse_find_match(clicon_handle h,
 		module);
     xc = NULL;
     while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) {
-	if (strcmp(xml_name(xc), "CLICON_YANG_DIR") != 0)
+	/* Skip if not yang dir */
+	if (strcmp(xml_name(xc), "CLICON_YANG_DIR") != 0 &&
+	    strcmp(xml_name(xc), "CLICON_YANG_MAIN_DIR") != 0)
 	    continue;
 	dir = xml_body(xc);
 	/* get all matching files in this directory */
@@ -1425,6 +1427,7 @@ yang_spec_parse_module(clicon_handle h,
     /* Do not load module if it already exists */
     if (yang_find_module_by_name_revision(yspec, name, revision) != NULL)
 	goto ok;
+    /* Find a yang module and parse it and all its submodules */
     if (yang_parse_module(h, name, revision, yspec) == NULL)
 	goto done;
     if (yang_parse_post(h, yspec, modmin) < 0)

test/test_restconf_internal_usecases.sh

@@ -16,6 +16,10 @@
 #   1. Start server
 #   2. Remove server
 #   3. Check status (Error: still up)
+# The fourth usecase is failing one of several sockets but still reach the working
+#   1. Start two servers, where one fails
+#   2. Reach one not the other
+#   (Wanted to bind an invalid port, but then such a port must be bound and later killed)
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@@ -28,6 +32,7 @@ startupdb=$dir/startup_db
 # Restconf debug
 RESTCONFDBG=$DBG
 RCPROTO=http # no ssl here
+INVALIDADDR=251.1.1.1 # used by fourth usecase as invalid
 
 if [ "${WITH_RESTCONF}" = "fcgi" ]; then
     EXTRACONF="<CLICON_FEATURE>clixon-restconf:fcgi</CLICON_FEATURE>"
@@ -122,7 +127,7 @@ EOF
 
 # FIRST usecase
 
-new "1. Empty status message"
+new "FIRST usecase: Empty status message"
 
 new "kill old restconf"
 stop_restconf_pre
@@ -163,14 +168,15 @@ rpcstatus true running
 pid1=$pid
 if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
 
-new "kill $pid1 externally"
+new "Check $pid1 exists"
-sudo kill $pid1
+while kill -0 $pid1 2> /dev/null; do
-sleep $DEMSLEEP
+    new "kill $pid1 externally"
+    sudo kill $pid1
+    sleep $DEMSLEEP
+done
+
+
 
-# Why kill it twice? it happens in docker somethimes but is not the aim of the test
-new "kill $pid1 again"
-sudo kill $pid1 2> /dev/null
-sleep $DEMSLEEP
 
 new "3. get status: Check killed"
 rpcstatus false stopped
@@ -204,7 +210,7 @@ if [ $BE -ne 0 ]; then
 fi
 
 # SECOND usecase
-new "2. zombie process on exit"
+new "SECOND usecase: zombie process on exit"
 
 new "kill old restconf"
 stop_restconf_pre
@@ -274,7 +280,7 @@ fi
 # THIRD usecase
 # NOTE this does not apply for fcgi where servers cant be "removed"
 if [ "${WITH_RESTCONF}" != "fcgi" ]; then
-new "3. restconf not removed"
+new "THIRD usecase: restconf not removed"
 
 new "kill old restconf"
 stop_restconf_pre
@@ -314,6 +320,9 @@ expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-confi
 new "commit create"
 expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
 
+new "wait restconf"
+wait_restconf
+
 # pid
 new "7. get status, get pid1"
 rpcstatus true running
@@ -359,6 +368,78 @@ fi
 
 fi # "${WITH_RESTCONF}" != "fcgi"
 
+# FOURTH usecase
+
+if [ "${WITH_RESTCONF}" != "fcgi" ]; then
+# Does not apply for fcgi where servers are configured in nginx
+
+new "FOURTH usecase. One server fails, others working"
+
+new "kill old restconf"
+stop_restconf_pre
+
+new "test params: -f $cfg"
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -z -f $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+
+    new "start backend -s init -f $cfg"
+    start_backend -s init -f $cfg
+fi
+new "wait backend"
+wait_backend
+
+new "9. get status stopped"
+rpcstatus false stopped
+if [ $pid -ne 0 ]; then err "Pid" "$pid"; fi
+
+RESTCONFIG1=$(cat <<EOF
+<restconf xmlns="http://clicon.org/restconf">
+   <enable>true</enable>
+   <debug>$RESTCONFDBG</debug>
+   <auth-type>none</auth-type>
+   <pretty>false</pretty>
+   <socket><namespace>default</namespace><address>0.0.0.0</address><port>80</port><ssl>false</ssl></socket>
+   <socket><namespace>default</namespace><address>$INVALIDADDR</address><port>8080</port><ssl>false</ssl></socket>
+</restconf>
+EOF
+)
+
+new "Create server"
+expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><target><candidate/></target><config>$RESTCONFIG1</config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
+
+new "commit create"
+expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
+
+new "wait restconf"
+wait_restconf
+
+# pid
+new "10. get status, get pid1"
+rpcstatus true running
+pid1=$pid
+if [ $pid1 -eq 0 ]; then err "Pid" 0; fi
+sleep $DEMSLEEP
+
+new "Get restconf config"
+expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' $RCPROTO://localhost/restconf/data/clixon-restconf:restconf)" 0 "HTTP/1.1 200 OK" "<restconf xmlns=\"http://clicon.org/restconf\"><enable>true</enable><auth-type>none</auth-type><debug>$RESTCONFDBG</debug><pretty>false</pretty><socket><namespace>default</namespace><address>0.0.0.0</address><port>80</port><ssl>false</ssl></socket><socket><namespace>default</namespace><address>$INVALIDADDR</address><port>8080</port><ssl>false</ssl></socket></restconf>"
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u root -f clixon_backend)
+    if [ -z "$pid" ]; then
+	err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+fi # "${WITH_RESTCONF}" != "fcgi"
+
 new "endtest"
 endtest
 

test/test_upgrade_checkold.sh

@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+# Test of CLICON_XMLDB_UPGRADE_CHECKOLD where original config in an upgrade scenario is tested,
+# not just current after upgrade
+# Assume read a config from startup of module A@2016 to be upgraded to A@2021 using auto-upgrade
+# Using CLICON_XMLDB_UPGRADE_CHECKOLD=true try all variations of:
+#   oldyang:  A@2016 exists or not
+#   modstate: startupdb has correct modstate or not
+#   xmlok:    startdb has the right "old" syntax or not "wrong"
+# These are 8 combinations. Only one is right, others give some variants of error messages
+# XXX remains to check all cases with CLICON_XMLDB_UPGRADE_CHECKOLD = false
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyang1=$dir/A@2016-01-01.yang
+fyang2=$dir/A@2021-01-01.yang
+changelog=$dir/changelog.xml # Module revision changelog
+
+# Create configuration
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XML_CHANGELOG>true</CLICON_XML_CHANGELOG>
+  <CLICON_XML_CHANGELOG_FILE>$changelog</CLICON_XML_CHANGELOG_FILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
+  <!-- given as -o option in clixon_backend start: CLICON_XMLDB_UPGRADE_CHECKOLD>true</CLICON_XMLDB_UPGRADE_CHECKOLD -->
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+</clixon-config>
+EOF
+
+# New yang
+cat <<EOF > $fyang2
+module A{
+  prefix a;
+  revision 2021-01-01;
+  namespace "urn:example:a";
+  container upgraded{
+  }
+}
+EOF
+
+# Changelog of example-a: 
+cat <<EOF > $changelog
+<changelogs xmlns="http://clicon.org/xml-changelog" xmlns:a="urn:example:a" xmlns:b="urn:example:b" >
+  <changelog>
+    <namespace>urn:example:a</namespace>
+    <revfrom>2016-01-01</revfrom>
+    <revision>2021-01-01</revision>
+    <step>
+      <name>0</name>
+      <op>rename</op>
+      <where>/a:old</where>
+      <tag>"upgraded"</tag>
+    </step>
+  </changelog>
+</changelogs>
+EOF
+
+# Arguments:
+# 1: expect return xml
+function testrun(){
+    checkold=$1
+    expectxml=$2
+
+    new "test params: -f $cfg"
+
+    new "start backend -D $DBG -s startup -f $cfg -q -l o"
+    expectpart "$(sudo $clixon_backend -D $DBG -o CLICON_XMLDB_UPGRADE_CHECKOLD=$checkold -s startup -f $cfg -q -l e 2>&1)" 0 "$expectxml"
+}
+
+# kill old backend (if any)
+new "kill old backend"
+sudo clixon_backend -zf $cfg
+if [ $? -ne 0 ]; then
+    err
+fi
+
+let j=1
+for checkold in true; do # XXX remains to check all cases with CLICON_XMLDB_UPGRADE_CHECKOLD = false
+for oldyang in true false; do
+    if $oldyang; then
+    # create old yang
+    cat <<EOF > $fyang1
+module A{
+  prefix a;
+  revision 2016-01-01;
+  namespace "urn:example:a";
+  container old{
+  }
+}
+EOF
+    else
+	rm -f $fyang1
+    fi
+    for modstate in true false; do
+	if $modstate; then
+	    modstatestr="<modules-state xmlns=\"urn:example:a\"><module-set-id>42</module-set-id><module><name>A</name><revision>2016-01-01</revision><namespace>urn:example:a</namespace></module></modules-state>"
+	else
+	    modstatestr=""
+	fi
+	for xml in true false; do
+	    if $xml; then
+		xmltag="old"
+		expectxml="<error-message>Failed to find YANG spec of XML node: $xmltag with parent: config in namespace: urn:example:a</error-message>"
+		if $oldyang; then
+		    if $modstate; then
+			expectxml="<upgraded xmlns=\"urn:example:a\"/>"
+		    fi
+		elif $modstate; then
+		    expectxml="<error-message>Internal error: No yang files found matching \"A@2016-01-01\" in the list of CLICON_YANG_DIRs</error-message>"
+		fi
+	    else # xml false
+		xmltag="wrong"
+		expectxml="<error-message>Failed to find YANG spec of XML node: $xmltag with parent: config in namespace: urn:example:a</error-message>"
+		if ! $oldyang; then
+		    if $modstate; then
+			expectxml="<error-message>Internal error: No yang files found matching \"A@2016-01-01\" in the list of CLICON_YANG_DIRs</error-message>"
+		    fi
+		fi
+	    fi
+	    cat <<EOF > $dir/startup_db
+	    <${DATASTORE_TOP}>
+	      $modstatestr
+	      <$xmltag xmlns="urn:example:a"/>
+	    </${DATASTORE_TOP}>
+EOF
+	    # Here is actual call
+	    new "$j. checkold:$checkold oldyang:$oldyang modstate:$modstate xmlok:$xml"
+	    testrun "$checkold" "$expectxml"
+	    let j++
+	done
+    done
+done
+done
+rm -rf $dir
+
+unset j
+unset xml
+unset xmltag
+unset oldyang
+unset modstate
+unset modstatestr
+unset fyang1
+unset fyang2
+
+new "endtest"
+endtest

test/test_upgrade_interfaces.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Upgrade a module by registering a manually programmed callback
 # The usecase is inspired by the ietf-interfaces upgrade from
-# 2014-05-08 to 2018-02-20.
+# 2014-05-08 to 2016-01-01 to 2018-02-20.
 # That includes moving parts from interfaces-state to interfaces and then
 # deprecating the whole /interfaces-state tree.
 # A preliminary change list is in Appendix A of
@@ -29,6 +29,28 @@ cfg=$dir/conf.xml
 if2014=$dir/interfaces@2014-05-08.yang
 if2018=$dir/interfaces@2018-02-20.yang
 
+# Create configuration
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_FEATURE>interfaces:if-mib</CLICON_FEATURE>
+  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
+  <CLICON_XML_CHANGELOG>false</CLICON_XML_CHANGELOG>
+  <CLICON_XMLDB_UPGRADE_CHECKOLD>false</CLICON_XMLDB_UPGRADE_CHECKOLD>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+</clixon-config>
+EOF
+
 # Original simplified version - note all is config to allow for storing in
 # datastore
 cat <<EOF > $if2014
@@ -104,7 +126,6 @@ module interfaces{
 	}
     }
 }
-
 EOF
 
 cat <<EOF > $if2018
@@ -229,27 +250,6 @@ cat <<EOF > $dir/startup_db
 </${DATASTORE_TOP}>
 EOF
 
-# Create configuration
-cat <<EOF > $cfg
-<clixon-config xmlns="http://clicon.org/config">
-  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
-  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
-  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
-  <CLICON_FEATURE>interfaces:if-mib</CLICON_FEATURE>
-  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
-  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
-  <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
-  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
-  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
-  <CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
-  <CLICON_XML_CHANGELOG>false</CLICON_XML_CHANGELOG>
-  <CLICON_XMLDB_UPGRADE_CHECKOLD>false</CLICON_XMLDB_UPGRADE_CHECKOLD>
-  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
-  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
-  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-</clixon-config>
-EOF
-
 # Start from startup and upgrade, check running 
 function testrun(){
     runxml=$1 

test/test_upgrade_quit.sh

@@ -16,6 +16,28 @@ cfg=$dir/conf.xml
 if2014=$dir/interfaces@2014-05-08.yang
 if2018=$dir/interfaces@2018-02-20.yang
 
+# Create configuration
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_FEATURE>interfaces:if-mib</CLICON_FEATURE>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
+  <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
+  <CLICON_XML_CHANGELOG>false</CLICON_XML_CHANGELOG>
+  <CLICON_XMLDB_UPGRADE_CHECKOLD>true</CLICON_XMLDB_UPGRADE_CHECKOLD>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+</clixon-config>
+EOF
+
 # Original simplified version - note all is config to allow for storing in
 # datastore
 cat <<EOF > $if2014
@@ -236,28 +258,6 @@ cat <<EOF > $dir/startup_db
 </${DATASTORE_TOP}>
 EOF
 
-# Create configuration
-cat <<EOF > $cfg
-<clixon-config xmlns="http://clicon.org/config">
-  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
-  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
-  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
-  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
-  <CLICON_FEATURE>interfaces:if-mib</CLICON_FEATURE>
-  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
-  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
-  <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
-  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
-  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
-  <CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
-  <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
-  <CLICON_XML_CHANGELOG>false</CLICON_XML_CHANGELOG>
-  <CLICON_XMLDB_UPGRADE_CHECKOLD>true</CLICON_XMLDB_UPGRADE_CHECKOLD>
-  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
-  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
-  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-</clixon-config>
-EOF
 
 
 # This is 2014 syntax