Merge branch 'master' of https://github.com/clicon/clixon

apps/backend/backend_main.c

@@ -843,6 +843,13 @@ main(int    argc,
 	    goto done;
 	/* if status = STARTUP_INVALID, cbret contains info */
     }
+    /*
+     * Disable unknown to anydata auto-creation after startup
+     */
+    if (clicon_option_bool(h, "CLICON_YANG_UNKNOWN_ANYDATA") == 1){
+	clicon_option_bool_set(h, "CLICON_YANG_UNKNOWN_ANYDATA", 0);
+	xml_bind_yang_unknown_anydata(0);
+    }
     /* Merge extra XML from file and reset function to running  
      */
     if (status == STARTUP_OK && startup_mode != SM_NONE){

docker/main/Makefile.in

@@ -75,7 +75,7 @@ push:
 # Note tests will kill the daemons started in the start scrips
 test: docker
 	./cleanup.sh ; PORT=8080 ./start.sh # kill (ignore error) and the start it
-	sudo docker exec -it clixon-system bash -c 'cd /usr/local/bin/test && ./all.sh'
+	sudo docker exec -t clixon-system bash -c 'cd /usr/local/bin/test && ./all.sh'
 
 depend:
 

test/cicd/Makefile.in

@@ -40,6 +40,7 @@ SHELL	= /bin/sh
 
 .PHONY: all clean distclean depend install uninstall
 
+# Optional specific LOGDIR, eg "LOGDIR=/var/log make"
 
 # Include "site.mk" file if it exists and define the HOSTS variables
 # eg :
@@ -54,7 +55,11 @@ all: $(HOSTS)
 
 # Real hosts reachable by ssh
 $(HOSTS): 
-	./cicd.sh $@ 2>&1 | tee $@.log
+ifdef LOGDIR
+	./cicd.sh $@ 2>&1 > $(LOGDIR)/$@.log
+else
+	./cicd.sh $@ 2>&1 | tee /$@.log
+endif
 
 clean:
 	rm -f *.log

test/test_yang_anydata.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 # Test YANG ANYDATA:
 # Also Test CLICON_YANG_UNKNOWN_ANYDATA: Treat unknown XML/JSON nodes as anydata. 
-# Test matric is three dimensions:
+# Test matrix is three dimensions:
-# 1. YANG spec:  u-elements are a) anydata or b) unknown
+# 1. YANG spec:  Add elements denotend with "u" which are either a) defined as anydata or b) unknown
-# 2. Access is made to top-elements: a) top-level, b) in container, c) in list
+# 2. Make access to elements as: a) top-level, b) in container, c) in list
 # 3. data is in a) startup b) netconf 3) state
 
 # Load an XML file with unknown yang constructs that is labelled as anydata
@@ -48,6 +48,14 @@ module any{
      }
      anydata u4;
    }
+   rpc myrpc {
+	input {
+	    anydata u7;
+	}
+	output {
+	    anydata u8;
+	}
+   }
 }
 EOF
 
@@ -73,6 +81,8 @@ module unknown{
        }
      }
    }
+   rpc myrpc {
+   }
 }
 EOF
 
@@ -81,7 +91,7 @@ XMLA='<ca xmlns="urn:example:any"><b><k>22</k><u3><u31>42</u31></u3></b><u2><u21
 
 XMLU='<cu xmlns="urn:example:unknown"><b><k>22</k><u3><u31>42</u31></u3></b><u2><u21>a string</u21></u2></cu><u1 xmlns="urn:example:unknown"><u11>23</u11></u1>'
 
-# Full state
+# Full state with unknowns
 STATE0='<sa xmlns="urn:example:any"><sb><k>22</k><u5>55</u5></sb><u4><u5>a string</u5></u4></sa><su xmlns="urn:example:unknown"><sb><k>22</k><u5>55</u5></sb><u4><u5>a string</u5></u4></su>'
 
 # Partial state with unknowns removed in the unknown module
@@ -99,11 +109,16 @@ testrun()
     unknown=$2
 
     if $unknown; then # treat unknown as anydata or not
-	unknownreply="<rpc-reply><ok/></rpc-reply>]]>]]>"
+	if $startup; then # If startup
 	    XML="$XMLA$XMLU"
 	else
-	unknownreply="<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown-element</error-tag><error-info><bad-element>u1</bad-element></error-info><error-severity>error</error-severity><error-message>Unassigned yang spec</error-message></rpc-error></rpc-reply>]]>]]>"
 	    XML="$XMLA"
+	    unknownreply="<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown-element</error-tag><error-info><bad-element>u1</bad-element></error-info><error-severity>error</error-severity><error-message>Unassigned yang spec</error-message></rpc-error></rpc-reply>]]>]]>"
+	fi
+    else
+	XML="$XMLA"
+	unknownreply="<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown
+-element</error-tag><error-info><bad-element>u1</bad-element></error-info><error-severity>error</error-severity><error-message>Unassigned yang spec</error-message></rpc-error></rpc-reply>]]>]]>"
     fi
 
     if $startup; then # get config from startup
@@ -127,6 +142,7 @@ testrun()
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
+  <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
   <CLICON_YANG_UNKNOWN_ANYDATA>$unknown</CLICON_YANG_UNKNOWN_ANYDATA>
   $F
 </clixon-config>
@@ -159,6 +175,16 @@ EOF
 	new "waiting"
 	wait_backend
     fi
+    if [ $RC -ne 0 ]; then
+	new "kill old restconf daemon"
+	stop_restconf_pre
+
+	new "start restconf daemon"
+	start_restconf -f $cfg
+
+    fi
+    new "wait restconf"
+    wait_restconf
 
 
     if ! $startup; then # If not startup, add xml using netconf
@@ -185,24 +211,45 @@ EOF
     new "cli commit"
     expectpart "$($clixon_cli -1 -f $cfg commit)" 0 "^$"
 
-    if $unknown; then
+
-	STATE="$STATE0" # full state
+    new "restconf get config"
-    else
+    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+xml" $RCPROTO://localhost/restconf/data?content=config)" 0 "HTTP/1.1 200 OK" "$XML"
-	STATE="$STATE1" # partial state
+    
-    fi
+    # Save partial state in state file with unknown removed (positive test)
-    echo "$STATE" > $fstate 
+    echo "$STATE1" > $fstate 
 
     new "Get state (positive test)"
-    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get content="nonconfig"></get></rpc>]]>]]>' "^<rpc-reply><data>$STATE</data></rpc-reply>]]>]]>"
+    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get content="nonconfig"></get></rpc>]]>]]>' "^<rpc-reply><data>$STATE1</data></rpc-reply>]]>]]>"
 
-    echo "$STATE0" > $fstate # full state
+    new "restconf get state(positive)"
+    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+xml" $RCPROTO://localhost/restconf/data?content=nonconfig)" 0 "HTTP/1.1 200 OK" "$STATE1"
+
+    # full state with unknowns
+    echo "$STATE0" > $fstate 
 
     new "Get state (negative test)"
-    if $unknown; then
-	expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get content="nonconfig"></get></rpc>]]>]]>' "^<rpc-reply><data>$STATE</data></rpc-reply>]]>]]>"
-    else
     expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get content="nonconfig"></get></rpc>]]>]]>' "error-message>Failed to find YANG spec of XML node: u5 with parent: sb in namespace: urn:example:unknown. Internal error, state callback returned invalid XML: example_backend</error-message>"
+
+	new "restconf get state(negative)"
+    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+xml" $RCPROTO://localhost/restconf/data?content=nonconfig)" 0 "HTTP/1.1 412 Precondition Failed" "<error-tag>operation-failed</error-tag><error-info><bad-element>u5</bad-element></error-info>"
+
+    # RPC:s take "not-supported" as OK: syntax OK and according to mdeol, just not implemented in
+    # server. But "unknown-element" as truly unknwon.
+    # (Would need to add a handler to get a proper OK)
+    new "Not supported RPC"
+    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><myrpc xmlns="urn:example:any"></myrpc></rpc>]]>]]>' '<error-tag>operation-not-supported</error-tag>'
+
+    new "anydata RPC"
+    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><myrpc xmlns="urn:example:any"><u7><u8>88</u8></u7></myrpc></rpc>]]>]]>' '<error-tag>operation-not-supported</error-tag>'
+
+    new "unknown RPC"
+    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><myrpc xmlns="urn:example:unknown"><u7><u8>88</u8></u7></myrpc></rpc>]]>]]>' '<error-tag>unknown-element</error-tag>'
+
+    if [ $RC -ne 0 ]; then
+	new "Kill restconf daemon"
+	stop_restconf
     fi
+
     if [ $BE -ne 0 ]; then
 	new "Kill backend"
 	# Check if premature kill

test/vagrant/Makefile.in

@@ -40,6 +40,7 @@ SHELL	= /bin/sh
 
 .PHONY: all clean distclean depend install uninstall
 
+# Optional specific LOGDIR, eg "LOGDIR=/var/log make"
 
 # The "site.mk" file if it exists and define the VAGRANTS variables
 # eg:
@@ -59,7 +60,11 @@ logs:
 # Local vagrant hosts eg generic/centos8. The vagrantdir is a subdirectory and logs will
 # appear eg as generic/centos8.log
 $(VAGRANTS): logs
+ifdef LOGDIR
+	./vagrant.sh $@ 2>&1 > "$(LOGDIR)/$(subst /,-,$@).log"
+else
 	./vagrant.sh $@ 2>&1 | tee "logs/$(subst /,-,$@).log"
+endif
 
 destroy:
 	for i in $(VAGRANTS) ; \

test/vagrant/vagrant.sh

@@ -96,6 +96,7 @@ system=$($sshcmd uname) # we use the release "hack" instead
 # Some release have packages, some need to be built from source
 buildfcgi=false
 buildevhtp=false
+buildcmake=false # Some releases (eg centos/7) has too old cmake to build libevhtp
 case $release in
     openbsd)
 	# packages for building
@@ -154,7 +155,8 @@ case $release in
 	    evhtp)
 		$sshcmd sudo yum install -y libevent openssl
 		buildevhtp=true
-		$sshcmd sudo yum install -y libevent-devel cmake openssl-devel
+		buildcmake=true # Actually, only necessary on centos/7
+		$sshcmd sudo yum install -y libevent-devel openssl-devel
 		;;
 	esac
 	;;
@@ -249,12 +251,29 @@ case ${with_restconf} in
 	. ./nginx.sh $dir $idfile $port $wwwuser
 	;;
     evhtp)
+	if $buildcmake; then
+	    $sshcmd "test -d cmake || sudo git clone https://gitlab.kitware.com/cmake/cmake.git"
+	    $sshcmd "(cd cmake; sudo ./bootstrap)"
+    	    $sshcmd "(cd cmake; sudo make)"
+	    $sshcmd "(cd cmake; sudo make install)"
+	fi
 	if $buildevhtp; then
+	    if true; then
+		$sshcmd << 'EOF'
+		test -d libevhtp || sudo git clone https://github.com/criticalstack/libevhtp.git
+		cd libevhtp/build; 
+		CMAKE=$(which cmake)
+		sudo $CMAKE -DEVHTP_DISABLE_REGEX=ON -DEVHTP_DISABLE_EVTHR=ON ..
+		sudo make
+		sudo make install
+EOF
+	    else
 		$sshcmd "test -d libevhtp || sudo git clone https://github.com/criticalstack/libevhtp.git"
-	    $sshcmd	"(cd libevhtp/build; sudo cmake -DEVHTP_DISABLE_REGEX=ON -DEVHTP_DISABLE_EVTHR=ON ..)"
+		$sshcmd "(cd libevhtp/build; sudo /usr/local/bin/cmake -DEVHTP_DISABLE_REGEX=ON -DEVHTP_DISABLE_EVTHR=ON ..)"
 		$sshcmd "(cd libevhtp/build; sudo make)"
 		$sshcmd "(cd libevhtp/build; sudo make install)"
 	    fi
+	fi
 	;;
 esac
 

yang/clixon/clixon-config@2020-06-17.yang

@@ -334,7 +334,7 @@ module clixon-config {
 	    type boolean;
 	    default false;
 	    description
-		"Treat unknown XML/JSON nodes as anydata. 
+		"Treat unknown XML/JSON nodes as anydata when loading from startup db.
                  This does not apply to namespaces, which means a top-level node: xxx:yyy
                  is accepted only if yyy is unknown, not xxx";
 	}