* Added [Vagrant tests](test/vagrant/README.md)

* Fixed memleak in restconf-fcgi

.gitignore

@@ -18,6 +18,7 @@ example/*/Makefile
 lib/Makefile
 lib/*/Makefile
 test/Makefile
+test/*/Makefile
 yang/Makefile
 yang/*/Makefile
 autom4te.cache/
@@ -58,4 +59,6 @@ util/clixon_util_xpath
 util/clixon_util_yang
 
 test/site.sh
+test/vagrant/site.mk
+test/cicd/site.mk
 doc/html

CHANGELOG.md

@@ -49,6 +49,7 @@ Expected: July 2020
   
 ### Minor changes
 
+* Added [Vagrant tests](test/vagrant/README.md)
 * Added new function `clicon_xml2str()` to complement xml_print and others that returns a malloced string.
 * Added new function `xml_child_index_each()` to iterate over the children of an XML node according to the order defined by an explicit index variable. This is a complement to `xml_child_each()` which iterates using the default order.
 

apps/restconf/restconf_fcgi_lib.c

@@ -246,8 +246,8 @@ clixon_restconf_params_set(clicon_handle h,
 {
     int   retval = -1;
     int   i;
-    char *param;
+    char *param = NULL;
-    char *val;
+    char *val = NULL;
 
     clicon_debug(1, "%s", __FUNCTION__);
     for (i = 0; envp[i] != NULL; i++){ /* on the form <param>=<value> */
@@ -256,6 +256,14 @@ clixon_restconf_params_set(clicon_handle h,
 	clicon_debug(1, "%s param:%s val:%s", __FUNCTION__, param, val);
 	if (clixon_restconf_param_set(h, param, val) < 0)
 	    goto done;
+	if (param){
+	    free(param);
+	    param = NULL;
+	}
+	if (val){
+	    free(val);
+	    val = NULL;
+	}
     }
     retval = 0;
  done:
@@ -274,16 +282,19 @@ clixon_restconf_params_clear(clicon_handle h,
 {
     int   retval = -1;
     int   i;
-    char *param;
+    char *param = NULL;
-    char *val;
 
     clicon_debug(1, "%s", __FUNCTION__);
     for (i = 0; envp[i] != NULL; i++){ /* on the form <param>=<value> */
-	if (clixon_strsplit(envp[i], '=', &param, &val) < 0)
+	if (clixon_strsplit(envp[i], '=', &param, NULL) < 0)
 	    goto done;
-	clicon_debug(1, "%s param:%s val:%s", __FUNCTION__, param, val);
+	clicon_debug(1, "%s param:%s", __FUNCTION__, param);
 	if (clixon_restconf_param_del(h, param) < 0)
 	    goto done;
+	if (param){
+	    free(param);
+	    param = NULL;
+	}
     }
     retval = 0;
  done:

configure

@@ -5325,7 +5325,7 @@ _ACEOF
 
 
 
-ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/cicd/Makefile"
+ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/cicd/Makefile test/vagrant/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -6045,6 +6045,7 @@ do
     "doc/Makefile") CONFIG_FILES="$CONFIG_FILES doc/Makefile" ;;
     "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;;
     "test/cicd/Makefile") CONFIG_FILES="$CONFIG_FILES test/cicd/Makefile" ;;
+    "test/vagrant/Makefile") CONFIG_FILES="$CONFIG_FILES test/vagrant/Makefile" ;;
 
   *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
   esac

configure.ac

@@ -326,5 +326,6 @@ AC_OUTPUT(Makefile
 	  doc/Makefile
 	  test/Makefile
 	  test/cicd/Makefile
+  	  test/vagrant/Makefile	
 )
 

test/README.md

@@ -19,6 +19,10 @@ this directory is executed.
 
 There are also [manual cicd scripts here](cicd/README.md)
 
+## Vagrant
+
+[Vagrant scripts are here](vagrant/README.md)
+
 ## Getting started
 
 You need to build and install the clixon utility programs before running the tests as some of the tests rely on them:

test/cicd/Makefile.in

@@ -41,25 +41,19 @@ SHELL	= /bin/sh
 .PHONY: all clean distclean depend install uninstall
 
 
-HOSTS=
+# Include "site.mk" file if it exists and define the HOSTS variables
-# The "hosts" file must exist and define the HOSTS variable
 # eg :
 # HOSTS += vandal.hagsand.com # i86_32 ubuntu
 # ...
--include hosts
+HOSTS=
+-include site.mk
 
-SCRIPTS  = cligen-mk.sh
+.PHONY: all clean distclean depend install uninstall $(HOSTS) $(VAGRANTS)
-SCRIPTS += clixon-mk.sh
-SCRIPTS += clixon-config.sh
 
-.PHONY: all clean distclean depend install uninstall $(HOSTS)
+all: $(HOSTS) $(VAGRANTS)
-
-all: $(HOSTS)
 
+# Real hosts reachable by ssh
 $(HOSTS): 
-	for s in $(SCRIPTS); do \
-	(scp $$s $@:/tmp/ ; ssh $@ chmod 750 /tmp/$$s || exit 1) \
-	done; 
 	./cicd.sh $@ 2>&1 | tee $@.log
 
 clean:

test/cicd/README.md

@@ -1,15 +1,13 @@
-CICD scripts
+Hosts scripts
-============
+=============
-Manual scripts for running committed code on a set of hosts.
+Manual scripts for running committed code on a set of remote hosts accessible with ssh.
 
 The script then uses a Makefile and logs in to each host, pulls from
 git, configure, makes and runs through the tests. Make is used to get
-concurrency - non-trivial with bash, eg with `make -j 10`
+concurrency, eg with `make -j 10`
 
-Note there are other cicd scripts than this, such as the the "travis" scrips.
+The Makefile contains a configurable HOSTS variable, which is defined
-
+in a "site.mk" file. You must add such a file, eg:
-The Makefile contains a configurable HOSTS variable, which ius defined
-in a "hosts" file. You must add such a file, eg:
 ```
    HOSTS += vandal.hagsand.com # i86_32 ubuntu
 ```

test/cicd/cicd.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 # CI/CD script complementing trevor github
-# Login in to a number of hosts and fo the following:
+# Triggered from Makefile
+# Login in to a number of hosts and do the following:
 # 0. Create and transfer sub-scripts used in main script: cligen-mk.sh clixon-mk.sh clixon-config.sh
 # 1. pull latest version
 # 2. Run configure
@@ -27,14 +28,23 @@ fi
 
 h=$1
 
+SCRIPTS="cligen-mk.sh clixon-mk.sh clixon-config.sh"
+
+# Copy test scripts to remote machine
+scp $SCRIPTS $h:/tmp/
+ssh -t $h "(cd /tmp; chmod 750 $SCRIPTS)"
+
+# pull git changes and build cligen
 ssh -t $h "test -d src || mkdir src"
 ssh -t $h "test -d src/cligen || (cd src;git clone https://github.com/olofhagsand/cligen.git)"
 ssh -t $h "(cd src/cligen;git pull)"
 ssh -t $h "(cd src/cligen;./configure)"
 ssh -t $h "(cd src/cligen; /tmp/cligen-mk.sh)"
+# pull git changes and build clixon
 ssh -t $h "test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)"
 ssh -t $h "(cd src/clixon;git pull)"
 ssh -t $h "(cd src/clixon; /tmp/clixon-config.sh)"
 ssh -t $h "(cd src/clixon; /tmp/clixon-mk.sh)"
 ssh -t $h sudo ldconfig
+# Run clixon test suite
 ssh -t $h "(cd src/clixon/test; ./sum.sh)"

test/test_api_path.sh

@@ -12,14 +12,14 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
 : ${clixon_util_path:=clixon_util_path -a -D $DBG}
 
+# Number of list/leaf-list entries
+: ${nr:=100}
+
 if [ $nr -lt 2 ] ; then
     echo "nr==$nr must be > 1"
     exit
 fi
 
-# Number of list/leaf-list entries
-: ${nr:=100}
-
 # Number of tests to generate XML for
 max=7
 

test/vagrant/Makefile.in

@@ -0,0 +1,73 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# 
+# Copyright (C) 2009-2019 Olof Hagsand
+# Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+#
+# This file is part of CLIXON
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 3 or later (the "GPL"),
+# in which case the provisions of the GPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of the GPL, and not to allow others to
+# use your version of this file under the terms of Apache License version 2, 
+# indicate your decision by deleting the provisions above and replace them with
+# the notice and other provisions required by the GPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the Apache License version 2 or the GPL.
+#
+# ***** END LICENSE BLOCK *****
+#
+
+VPATH       	= @srcdir@
+srcdir  	= @srcdir@
+top_srcdir  	= @top_srcdir@
+
+SHELL	= /bin/sh
+
+.PHONY: all clean distclean depend install uninstall
+
+
+# The "site.mk" file if it exists and define the VAGRANTS variables
+# eg:
+# VAGRANTS += freebsd/FreeBSD-12.1-STABLE
+
+VAGRANTS =
+
+-include site.mk
+
+.PHONY: all clean distclean depend install uninstall $(VAGRANTS)
+
+all: $(VAGRANTS)
+
+# Local vagrant hosts
+$(VAGRANTS):
+	./vagrant.sh $@ destroy 2>&1 | tee $@.log
+
+clean:
+	rm -f *.log
+
+distclean: clean
+	rm -f Makefile *~ .depend
+
+depend:
+
+install-include:
+
+install:
+
+uninstall:
+

test/vagrant/README.md

@@ -0,0 +1,28 @@
+Vagrant scripts
+===============
+Scripts for booting local vagrant hosts, installing clixon and running clixon tests
+
+The script then uses a Makefile and logs in to each host, pulls from
+git, configure, makes and runs through the tests. Make is used to get
+concurrency - eg with `make -j 10`
+
+The Makefile contains a configurable VAGRANTS variable, which is defined
+in a "site.mk" file. You can add such a file, eg:
+```
+  VAGRANTS += freebsd/FreeBSD-12.1-STABLE
+  VAGRANTS += generic/centos8
+```
+
+Logs appear in : <hostname>.log.
+
+You can also run a single vagrant test as follows:
+```
+  vagrant.sh freebsd/FreeBSD-12.1-STABLE
+```
+
+The current status is as follows
+* freebsd/FreeBSD-12.1-STABLE
+* generic/centos8 - some remaining nginx issue
+* generic/opensuse42 - fastcgi is not installed
+
+See more Vagrant boxes at [https://vagrantcloud.com/search]).

test/vagrant/nginx.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# Nginx config script. There are different variants of nginx configs, just off-loading
+# this to a separate script to hide the complexity
+
+set -eux
+
+if [ $# -ne 3 ]; then 
+    echo "usage: $0 <dir> <idfile> <port>"
+    exit -1
+fi
+dir=$1
+idfile=$2
+port=$3
+
+sshcmd="ssh -o StrictHostKeyChecking=no -i $idfile -p $port vagrant@127.0.0.1"
+scpcmd="scp -o StrictHostKeyChecking=no -p -i $idfile -P $port"
+
+if $($sshcmd test -d /etc/nginx/conf.d) ; then
+    confd=true
+else
+    confd=false
+fi
+
+if $confd; then # conf.d nginx config
+cat <<EOF > $dir/default.conf
+#
+server {
+        listen 80 default_server;
+        listen localhost:80 default_server;
+        listen [::]:80 default_server;
+	server_name localhost;
+	server_name _;
+	location / {
+	    fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
+	    include fastcgi_params;
+        }
+	location /streams {
+	    fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
+	    include fastcgi_params;
+ 	    proxy_http_version 1.1;
+	    proxy_set_header Connection "";
+        }
+}
+EOF
+    $scpcmd $dir/default.conf vagrant@127.0.0.1:
+cat<<EOF > $dir/startnginx.sh
+  sudo cp default.conf /etc/nginx/conf.d/
+#  if [ ! -d /run/nginx ]; then
+#    sudo mkdir /run/nginx
+#  fi
+  # Start nginx
+  /usr/sbin/nginx -c /etc/nginx/nginx.conf
+  >&2 echo "nginx started"
+
+EOF
+
+else # full nginx config
+
+# Nginx conf file
+cat<<'EOF' > $dir/nginx.conf
+#
+worker_processes  1;
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+    #                  '$status $body_bytes_sent "$http_referer" '
+    #                  '"$http_user_agent" "$http_x_forwarded_for"';
+
+    #access_log  logs/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    #keepalive_timeout  0;
+    keepalive_timeout  65;
+
+    #gzip  on;
+  server {
+        listen 80 default_server;
+        listen localhost:80 default_server;
+        listen [::]:80 default_server;
+	server_name localhost;
+	server_name _;
+	location / {
+	    fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
+	    include fastcgi_params;
+        }
+	location /streams {
+	    fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
+	    include fastcgi_params;
+ 	    proxy_http_version 1.1;
+	    proxy_set_header Connection "";
+        }
+  }
+}
+EOF
+    $scpcmd $dir/nginx.conf vagrant@127.0.0.1:
+cat<<EOF > $dir/startnginx.sh
+    #!/usr/bin/env bash
+    # start nginx
+    sudo cp nginx.conf /usr/local/etc/nginx/
+    if [ ! $(grep nginx_enable /etc/rc.conf) ]; then
+	sudo sh -c ' echo 'nginx_enable="YES"' >> /etc/rc.conf'
+    fi
+    sudo /usr/local/etc/rc.d/nginx restart
+EOF
+
+fi # full nginx config
+
+chmod a+x $dir/startnginx.sh
+$scpcmd $dir/startnginx.sh vagrant@127.0.0.1:
+$sshcmd ./startnginx.sh
+

test/vagrant/vagrant.sh

@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+# Script for running cligen and clixon test scripts on local vagrant virtual hosts
+# 1. Create a vagrant host based on "box" argument
+# 2. setup host for clixon
+# 3. Compile and install clixon
+# 4. Run tests
+# Example run:  ./vagrant.sh generic/centos8 2>&1 | tee cilog
+
+set -eux # x
+
+if [ $# -ne 1 -a $# -ne 2 ]; then 
+    echo "usage: $0 <box> [destroy]\n <box> as defined in https://vagrantcloud.com/search"
+    exit -1
+fi
+
+box=$1 # As defined in https://vagrantcloud.com/search
+if  [ $# -eq 2 ]; then
+    destroy=true
+else
+    destroy=false
+fi
+host=$(echo "$box"|awk -F'/' '{print $2}')
+dir=$box
+# XXX: ad-hoc to get (linus) release from boxname
+# using lsb_release is too heavyweight in many cases
+release=$(echo "$host" | grep -io "[a-z]*" | head -1 | tr '[:upper:]' '[:lower:]')
+
+# example box="freebsd/FreeBSD-12.1-STABLE"
+test -d $dir || mkdir -p $dir
+
+# Write a freebsd vagrant file
+cat<<EOF > $dir/Vagrantfile
+Vagrant.configure("2") do |config|
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "$box"
+  config.ssh.shell = "sh" # freebsd
+  config.vm.define "$host"
+  config.vm.hostname = "$host"
+end
+EOF
+
+# Start vagrant
+if $destroy; then
+    (cd $dir; vagrant destroy -f)
+fi
+(cd $dir; vagrant up)
+
+# Get ssh config to make proper ssh/scp calls to local vagrant host
+cfg=$(cd $dir; vagrant ssh-config $host)
+idfile=$(echo "$cfg" |grep "IdentityFile"|awk '{print $2}')
+port=$(echo "$cfg" |grep "Port"|awk '{print $2}')
+# make ssh and scp shorthand commands using vagrant-generated keys
+sshcmd="ssh -o StrictHostKeyChecking=no -i $idfile -p $port vagrant@127.0.0.1"
+scpcmd="scp -p -o StrictHostKeyChecking=no -i $idfile -P $port"
+ssh-keygen -f "$HOME/.ssh/known_hosts" -R "[127.0.0.1]:$port"
+echo "$sshcmd"
+
+system=$($sshcmd uname)
+
+case $system in
+    FreeBSD)
+	# packages for building
+	$sshcmd sudo pkg install -y git gmake bash
+	# cligen
+	$sshcmd sudo pkg install -y bison flex
+	# clixon
+	$sshcmd sudo pkg install -y fcgi-devkit nginx # FreeBSD
+	;;
+    Linux)
+	case $release in
+	    centos) # centos 8
+		# packages for building
+		$sshcmd sudo yum install -y git
+		# cligen
+		$sshcmd sudo yum install -y bison flex
+		# clixon 
+		$sshcmd sudo yum install -y fcgi-devel nginx
+		# clixon utilities
+		$sshcmd sudo yum install -y libcurl-devel
+		;;
+	    opensuse) # opensuse42
+		# clixon 
+		$sshcmd sudo zypper install -y nginx
+		;;
+	    *)
+		;;
+	esac
+	;;
+    *)
+	echo "Unknown system: $system"
+	;;
+esac
+
+# Hide all complex nginx config in sub-script
+. ./nginx.sh $dir $idfile $port
+
+# Setup cligen and clixon
+# 'EOF' means dont expand $
+cat<<'EOF' > $dir/setup.sh
+#!/usr/bin/env bash
+set -eux # x
+
+if [ $# -ne 1 ]; then 
+    echo "usage: $0 <release>"
+    exit -1
+fi
+release=$1
+# create user & group
+if [ ! $(id -u clicon) ]; then 
+   if [ $release = "freebsd" ]; then
+      sudo pw useradd clicon; 
+      sudo pw group mod clicon -m vagrant; 
+      sudo pw group mod clicon -m www; 
+   else  
+      sudo useradd clicon;
+      sudo usermod -a -G clicon vagrant;
+      sudo usermod -a -G clicon nginx; # nginx?
+   fi
+fi
+
+# cligen
+test -d src || mkdir src
+test -d src/cligen || (cd src;git clone https://github.com/olofhagsand/cligen.git)
+cd src/cligen
+git pull
+
+if [ $release = "freebsd" ]; then
+    ./configure
+    MAKE=$(which gmake)
+else
+    ./configure --prefix=/usr
+    MAKE=$(which make)
+fi
+echo "MAKE:$MAKE"
+$MAKE clean
+$MAKE -j10
+sudo $MAKE install
+
+# Clixon
+cd
+test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)
+cd src/clixon
+git pull
+
+if [ $release = "freebsd" ]; then
+    LDFLAGS=-L/usr/local/lib ./configure --with-wwwuser=www --enable-optyangs
+else
+   # Problems with su not having "sbin" in path on centos when when we run tests later
+    ./configure --sbindir=/usr/sbin --libdir=/usr/lib --with-wwwuser=nginx --enable-optyangs
+fi
+$MAKE clean
+$MAKE -j10
+sudo $MAKE install
+(cd example; $MAKE)
+(cd util; $MAKE)
+(cd example; sudo $MAKE install)
+(cd util; sudo $MAKE install)
+sudo ldconfig
+cd test
+echo "#!/usr/bin/env bash" > ./site.sh
+if [ $release = "freebsd" ]; then
+echo "wwwuser=www" >> ./site.sh
+echo "make=gmake" >> ./site.sh
+echo 'SKIPLIST="test_api.sh"' >> ./site.sh
+else
+echo "wwwuser=nginx" >> ./site.sh
+fi
+EOF
+chmod a+x $dir/setup.sh
+
+# config and setup cligen and clixon
+$scpcmd $dir/setup.sh vagrant@127.0.0.1:
+$sshcmd ./setup.sh $release
+
+# Run tests
+$sshcmd "(cd src/cligen/test; ./sum.sh)"
+$sshcmd "(cd src/clixon/test; ./sum.sh)"
+
+# destroy vm
+if $destroy; then
+    (cd $dir; vagrant destroy -f)
+fi