experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* Added [Vagrant tests](test/vagrant/README.md)

Browse source 
* Fixed memleak in restconf-fcgi
This commit is contained in:
Olof Hagsand 2020-06-08 08:16:42 +00:00
parent 7ffa2920a4
commit ab73794292
14 changed files with 458 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -18,6 +18,7 @@ example/*/Makefile
lib/Makefile
lib/*/Makefile
test/Makefile
test/*/Makefile
yang/Makefile
yang/*/Makefile
autom4te.cache/
@ -58,4 +59,6 @@ util/clixon_util_xpath
util/clixon_util_yang
test/site.sh
test/vagrant/site.mk
test/cicd/site.mk
doc/html

1
CHANGELOG.md
View file

@ -49,6 +49,7 @@ Expected: July 2020
### Minor changes
* Added [Vagrant tests](test/vagrant/README.md)
* Added new function `clicon_xml2str()` to complement xml_print and others that returns a malloced string.
* Added new function `xml_child_index_each()` to iterate over the children of an XML node according to the order defined by an explicit index variable. This is a complement to `xml_child_each()` which iterates using the default order.

23
apps/restconf/restconf_fcgi_lib.c
View file

@ -246,8 +246,8 @@ clixon_restconf_params_set(clicon_handle h,
{
int retval = -1;
int i;
char *param;
char *val;
char *param = NULL;
char *val = NULL;
clicon_debug(1, "%s", __FUNCTION__);
for (i = 0; envp[i] != NULL; i++){ /* on the form <param>=<value> */
@ -256,6 +256,14 @@ clixon_restconf_params_set(clicon_handle h,
clicon_debug(1, "%s param:%s val:%s", __FUNCTION__, param, val);
if (clixon_restconf_param_set(h, param, val) < 0)
goto done;
if (param){
free(param);
param = NULL;
}
if (val){
free(val);
val = NULL;
}
}
retval = 0;
done:
@ -274,16 +282,19 @@ clixon_restconf_params_clear(clicon_handle h,
{
int retval = -1;
int i;
char *param;
char *val;
char *param = NULL;
clicon_debug(1, "%s", __FUNCTION__);
for (i = 0; envp[i] != NULL; i++){ /* on the form <param>=<value> */
if (clixon_strsplit(envp[i], '=', &param, &val) < 0)
if (clixon_strsplit(envp[i], '=', &param, NULL) < 0)
goto done;
clicon_debug(1, "%s param:%s val:%s", __FUNCTION__, param, val);
clicon_debug(1, "%s param:%s", __FUNCTION__, param);
if (clixon_restconf_param_del(h, param) < 0)
goto done;
if (param){
free(param);
param = NULL;
}
}
retval = 0;
done:

3
configure vendored
View file

@ -5325,7 +5325,7 @@ _ACEOF
ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/cicd/Makefile"
ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/cicd/Makefile test/vagrant/Makefile"
cat >confcache <<\_ACEOF
# This file is a shell script that caches the results of configure
@ -6045,6 +6045,7 @@ do
"doc/Makefile") CONFIG_FILES="$CONFIG_FILES doc/Makefile" ;;
"test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;;
"test/cicd/Makefile") CONFIG_FILES="$CONFIG_FILES test/cicd/Makefile" ;;
"test/vagrant/Makefile") CONFIG_FILES="$CONFIG_FILES test/vagrant/Makefile" ;;
*) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
esac

3
configure.ac
View file

@ -325,6 +325,7 @@ AC_OUTPUT(Makefile
yang/optional/Makefile
doc/Makefile
test/Makefile
test/cicd/Makefile
test/cicd/Makefile
test/vagrant/Makefile
)

4
test/README.md
View file

@ -19,6 +19,10 @@ this directory is executed.
There are also [manual cicd scripts here](cicd/README.md)
## Vagrant
[Vagrant scripts are here](vagrant/README.md)
## Getting started
You need to build and install the clixon utility programs before running the tests as some of the tests rely on them:

18
test/cicd/Makefile.in
View file

@ -41,25 +41,19 @@ SHELL = /bin/sh
.PHONY: all clean distclean depend install uninstall
HOSTS=
# The "hosts" file must exist and define the HOSTS variable
# Include "site.mk" file if it exists and define the HOSTS variables
# eg :
# HOSTS += vandal.hagsand.com # i86_32 ubuntu
# ...
-include hosts
HOSTS=
-include site.mk
SCRIPTS = cligen-mk.sh
SCRIPTS += clixon-mk.sh
SCRIPTS += clixon-config.sh
.PHONY: all clean distclean depend install uninstall $(HOSTS) $(VAGRANTS)
.PHONY: all clean distclean depend install uninstall $(HOSTS)
all: $(HOSTS)
all: $(HOSTS) $(VAGRANTS)
# Real hosts reachable by ssh
$(HOSTS):
for s in $(SCRIPTS); do \
(scp $$s $@:/tmp/ ; ssh $@ chmod 750 /tmp/$$s || exit 1) \
done;
./cicd.sh $@ 2>&1 | tee $@.log
clean:

14
test/cicd/README.md
View file

@ -1,15 +1,13 @@
CICD scripts
============
Manual scripts for running committed code on a set of hosts.
Hosts scripts
=============
Manual scripts for running committed code on a set of remote hosts accessible with ssh.
The script then uses a Makefile and logs in to each host, pulls from
git, configure, makes and runs through the tests. Make is used to get
concurrency - non-trivial with bash, eg with `make -j 10`
concurrency, eg with `make -j 10`
Note there are other cicd scripts than this, such as the the "travis" scrips.
The Makefile contains a configurable HOSTS variable, which ius defined
in a "hosts" file. You must add such a file, eg:
The Makefile contains a configurable HOSTS variable, which is defined
in a "site.mk" file. You must add such a file, eg:
```
HOSTS += vandal.hagsand.com # i86_32 ubuntu
```

12
test/cicd/cicd.sh
View file

@ -1,6 +1,7 @@
#!/usr/bin/env bash
# CI/CD script complementing trevor github
# Login in to a number of hosts and fo the following:
# Triggered from Makefile
# Login in to a number of hosts and do the following:
# 0. Create and transfer sub-scripts used in main script: cligen-mk.sh clixon-mk.sh clixon-config.sh
# 1. pull latest version
# 2. Run configure
@ -27,14 +28,23 @@ fi
h=$1
SCRIPTS="cligen-mk.sh clixon-mk.sh clixon-config.sh"
# Copy test scripts to remote machine
scp $SCRIPTS $h:/tmp/
ssh -t $h "(cd /tmp; chmod 750 $SCRIPTS)"
# pull git changes and build cligen
ssh -t $h "test -d src || mkdir src"
ssh -t $h "test -d src/cligen || (cd src;git clone https://github.com/olofhagsand/cligen.git)"
ssh -t $h "(cd src/cligen;git pull)"
ssh -t $h "(cd src/cligen;./configure)"
ssh -t $h "(cd src/cligen; /tmp/cligen-mk.sh)"
# pull git changes and build clixon
ssh -t $h "test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)"
ssh -t $h "(cd src/clixon;git pull)"
ssh -t $h "(cd src/clixon; /tmp/clixon-config.sh)"
ssh -t $h "(cd src/clixon; /tmp/clixon-mk.sh)"
ssh -t $h sudo ldconfig
# Run clixon test suite
ssh -t $h "(cd src/clixon/test; ./sum.sh)"

6
test/test_api_path.sh
View file

@ -12,14 +12,14 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
: ${clixon_util_path:=clixon_util_path -a -D $DBG}
# Number of list/leaf-list entries
: ${nr:=100}
if [ $nr -lt 2 ] ; then
echo "nr==$nr must be > 1"
exit
fi
# Number of list/leaf-list entries
: ${nr:=100}
# Number of tests to generate XML for
max=7

73
test/vagrant/Makefile.in Normal file
View file

@ -0,0 +1,73 @@
#
# ***** BEGIN LICENSE BLOCK *****
#
# Copyright (C) 2009-2019 Olof Hagsand
# Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC(Netgate)
#
# This file is part of CLIXON
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Alternatively, the contents of this file may be used under the terms of
# the GNU General Public License Version 3 or later (the "GPL"),
# in which case the provisions of the GPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of the GPL, and not to allow others to
# use your version of this file under the terms of Apache License version 2,
# indicate your decision by deleting the provisions above and replace them with
# the notice and other provisions required by the GPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the Apache License version 2 or the GPL.
#
# ***** END LICENSE BLOCK *****
#
VPATH = @srcdir@
srcdir = @srcdir@
top_srcdir = @top_srcdir@
SHELL = /bin/sh
.PHONY: all clean distclean depend install uninstall
# The "site.mk" file if it exists and define the VAGRANTS variables
# eg:
# VAGRANTS += freebsd/FreeBSD-12.1-STABLE
VAGRANTS =
-include site.mk
.PHONY: all clean distclean depend install uninstall $(VAGRANTS)
all: $(VAGRANTS)
# Local vagrant hosts
$(VAGRANTS):
./vagrant.sh $@ destroy 2>&1 | tee $@.log
clean:
rm -f *.log
distclean: clean
rm -f Makefile *~ .depend
depend:
install-include:
install:
uninstall:

28
test/vagrant/README.md Normal file
View file

@ -0,0 +1,28 @@
Vagrant scripts
===============
Scripts for booting local vagrant hosts, installing clixon and running clixon tests
The script then uses a Makefile and logs in to each host, pulls from
git, configure, makes and runs through the tests. Make is used to get
concurrency - eg with `make -j 10`
The Makefile contains a configurable VAGRANTS variable, which is defined
in a "site.mk" file. You can add such a file, eg:
```
VAGRANTS += freebsd/FreeBSD-12.1-STABLE
VAGRANTS += generic/centos8
```
Logs appear in : <hostname>.log.
You can also run a single vagrant test as follows:
```
vagrant.sh freebsd/FreeBSD-12.1-STABLE
```
The current status is as follows
* freebsd/FreeBSD-12.1-STABLE
* generic/centos8 - some remaining nginx issue
* generic/opensuse42 - fastcgi is not installed
See more Vagrant boxes at [https://vagrantcloud.com/search]).

119
test/vagrant/nginx.sh Executable file
View file

@ -0,0 +1,119 @@
#!/usr/bin/env bash
# Nginx config script. There are different variants of nginx configs, just off-loading
# this to a separate script to hide the complexity
set -eux
if [ $# -ne 3 ]; then
echo "usage: $0 <dir> <idfile> <port>"
exit -1
fi
dir=$1
idfile=$2
port=$3
sshcmd="ssh -o StrictHostKeyChecking=no -i $idfile -p $port vagrant@127.0.0.1"
scpcmd="scp -o StrictHostKeyChecking=no -p -i $idfile -P $port"
if $($sshcmd test -d /etc/nginx/conf.d) ; then
confd=true
else
confd=false
fi
if $confd; then # conf.d nginx config
cat <<EOF > $dir/default.conf
#
server {
listen 80 default_server;
listen localhost:80 default_server;
listen [::]:80 default_server;
server_name localhost;
server_name _;
location / {
fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
include fastcgi_params;
}
location /streams {
fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
include fastcgi_params;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
}
EOF
$scpcmd $dir/default.conf vagrant@127.0.0.1:
cat<<EOF > $dir/startnginx.sh
sudo cp default.conf /etc/nginx/conf.d/
# if [ ! -d /run/nginx ]; then
# sudo mkdir /run/nginx
# fi
# Start nginx
/usr/sbin/nginx -c /etc/nginx/nginx.conf
>&2 echo "nginx started"
EOF
else # full nginx config
# Nginx conf file
cat<<'EOF' > $dir/nginx.conf
#
worker_processes 1;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
server {
listen 80 default_server;
listen localhost:80 default_server;
listen [::]:80 default_server;
server_name localhost;
server_name _;
location / {
fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
include fastcgi_params;
}
location /streams {
fastcgi_pass unix:/www-data/fastcgi_restconf.sock;
include fastcgi_params;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
}
}
EOF
$scpcmd $dir/nginx.conf vagrant@127.0.0.1:
cat<<EOF > $dir/startnginx.sh
#!/usr/bin/env bash
# start nginx
sudo cp nginx.conf /usr/local/etc/nginx/
if [ ! $(grep nginx_enable /etc/rc.conf) ]; then
sudo sh -c ' echo 'nginx_enable="YES"' >> /etc/rc.conf'
fi
sudo /usr/local/etc/rc.d/nginx restart
EOF
fi # full nginx config
chmod a+x $dir/startnginx.sh
$scpcmd $dir/startnginx.sh vagrant@127.0.0.1:
$sshcmd ./startnginx.sh

183
test/vagrant/vagrant.sh Executable file
View file

@ -0,0 +1,183 @@
#!/usr/bin/env bash
# Script for running cligen and clixon test scripts on local vagrant virtual hosts
# 1. Create a vagrant host based on "box" argument
# 2. setup host for clixon
# 3. Compile and install clixon
# 4. Run tests
# Example run: ./vagrant.sh generic/centos8 2>&1 | tee cilog
set -eux # x
if [ $# -ne 1 -a $# -ne 2 ]; then
echo "usage: $0 <box> [destroy]\n <box> as defined in https://vagrantcloud.com/search"
exit -1
fi
box=$1 # As defined in https://vagrantcloud.com/search
if [ $# -eq 2 ]; then
destroy=true
else
destroy=false
fi
host=$(echo "$box"|awk -F'/' '{print $2}')
dir=$box
# XXX: ad-hoc to get (linus) release from boxname
# using lsb_release is too heavyweight in many cases
release=$(echo "$host" | grep -io "[a-z]*" | head -1 | tr '[:upper:]' '[:lower:]')
# example box="freebsd/FreeBSD-12.1-STABLE"
test -d $dir || mkdir -p $dir
# Write a freebsd vagrant file
cat<<EOF > $dir/Vagrantfile
Vagrant.configure("2") do |config|
# Every Vagrant development environment requires a box. You can search for
# boxes at https://vagrantcloud.com/search.
config.vm.box = "$box"
config.ssh.shell = "sh" # freebsd
config.vm.define "$host"
config.vm.hostname = "$host"
end
EOF
# Start vagrant
if $destroy; then
(cd $dir; vagrant destroy -f)
fi
(cd $dir; vagrant up)
# Get ssh config to make proper ssh/scp calls to local vagrant host
cfg=$(cd $dir; vagrant ssh-config $host)
idfile=$(echo "$cfg" |grep "IdentityFile"|awk '{print $2}')
port=$(echo "$cfg" |grep "Port"|awk '{print $2}')
# make ssh and scp shorthand commands using vagrant-generated keys
sshcmd="ssh -o StrictHostKeyChecking=no -i $idfile -p $port vagrant@127.0.0.1"
scpcmd="scp -p -o StrictHostKeyChecking=no -i $idfile -P $port"
ssh-keygen -f "$HOME/.ssh/known_hosts" -R "[127.0.0.1]:$port"
echo "$sshcmd"
system=$($sshcmd uname)
case $system in
FreeBSD)
# packages for building
$sshcmd sudo pkg install -y git gmake bash
# cligen
$sshcmd sudo pkg install -y bison flex
# clixon
$sshcmd sudo pkg install -y fcgi-devkit nginx # FreeBSD
;;
Linux)
case $release in
centos) # centos 8
# packages for building
$sshcmd sudo yum install -y git
# cligen
$sshcmd sudo yum install -y bison flex
# clixon
$sshcmd sudo yum install -y fcgi-devel nginx
# clixon utilities
$sshcmd sudo yum install -y libcurl-devel
;;
opensuse) # opensuse42
# clixon
$sshcmd sudo zypper install -y nginx
;;
*)
;;
esac
;;
*)
echo "Unknown system: $system"
;;
esac
# Hide all complex nginx config in sub-script
. ./nginx.sh $dir $idfile $port
# Setup cligen and clixon
# 'EOF' means dont expand $
cat<<'EOF' > $dir/setup.sh
#!/usr/bin/env bash
set -eux # x
if [ $# -ne 1 ]; then
echo "usage: $0 <release>"
exit -1
fi
release=$1
# create user & group
if [ ! $(id -u clicon) ]; then
if [ $release = "freebsd" ]; then
sudo pw useradd clicon;
sudo pw group mod clicon -m vagrant;
sudo pw group mod clicon -m www;
else
sudo useradd clicon;
sudo usermod -a -G clicon vagrant;
sudo usermod -a -G clicon nginx; # nginx?
fi
fi
# cligen
test -d src || mkdir src
test -d src/cligen || (cd src;git clone https://github.com/olofhagsand/cligen.git)
cd src/cligen
git pull
if [ $release = "freebsd" ]; then
./configure
MAKE=$(which gmake)
else
./configure --prefix=/usr
MAKE=$(which make)
fi
echo "MAKE:$MAKE"
$MAKE clean
$MAKE -j10
sudo $MAKE install
# Clixon
cd
test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)
cd src/clixon
git pull
if [ $release = "freebsd" ]; then
LDFLAGS=-L/usr/local/lib ./configure --with-wwwuser=www --enable-optyangs
else
# Problems with su not having "sbin" in path on centos when when we run tests later
./configure --sbindir=/usr/sbin --libdir=/usr/lib --with-wwwuser=nginx --enable-optyangs
fi
$MAKE clean
$MAKE -j10
sudo $MAKE install
(cd example; $MAKE)
(cd util; $MAKE)
(cd example; sudo $MAKE install)
(cd util; sudo $MAKE install)
sudo ldconfig
cd test
echo "#!/usr/bin/env bash" > ./site.sh
if [ $release = "freebsd" ]; then
echo "wwwuser=www" >> ./site.sh
echo "make=gmake" >> ./site.sh
echo 'SKIPLIST="test_api.sh"' >> ./site.sh
else
echo "wwwuser=nginx" >> ./site.sh
fi
EOF
chmod a+x $dir/setup.sh
# config and setup cligen and clixon
$scpcmd $dir/setup.sh vagrant@127.0.0.1:
$sshcmd ./setup.sh $release
# Run tests
$sshcmd "(cd src/cligen/test; ./sum.sh)"
$sshcmd "(cd src/clixon/test; ./sum.sh)"
# destroy vm
if $destroy; then
(cd $dir; vagrant destroy -f)
fi