experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of https://github.com/clicon/clixon

Browse source
This commit is contained in:
Olof hagsand 2020-07-12 10:59:23 +02:00
commit 94ad44a2ea
13 changed files with 324 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

9
apps/restconf/restconf_api_evhtp.c
View file

@ -65,6 +65,15 @@
#include "restconf_lib.h"
#include "restconf_api.h" /* Virtual api */
/* evhtp_safe_free is a macro that may not be present in a libevhtp release
*/
#ifndef evhtp_safe_free
#define evhtp_safe_free(_var, _freefn) do { \
_freefn((_var)); \
(_var) = NULL; \
} while (0)
#endif
/*! Add HTTP header field name and value to reply, evhtp specific
* @param[in] req Evhtp http request handle
* @param[in] name HTTP header field name

2
apps/restconf/restconf_handle.h
View file

@ -37,7 +37,7 @@
*/
#ifndef _RESTCONF_HANDLE_H_
#define _RESCTONF_HANDLE_H_
#define _RESTCONF_HANDLE_H_
/*
* Prototypes

4
lib/clixon/clixon_handle.h
View file

@ -46,11 +46,7 @@
Note that its contents is different dependending on if invoked from a
cli/backend/netconf or other plugin. But this is hidden under-the-hood.
*/
#if 1 /* SANITY CHECK */
typedef struct {float a;} *clicon_handle;
#else
typedef void *clicon_handle;
#endif
/* The dynamicically loadable plugin object handle (should be in clixon_plugin.h) */
typedef void *plghndl_t;

1
test/config.sh.in
View file

@ -4,4 +4,5 @@
WITH_RESTCONF=@with_restconf@ # evhtp, fcgi or ""
WITH_LIBXML2=@with_libxml2@ # yes or ""
CXX=@CXX@
CC=@CC@

3
test/test_api.sh
View file

@ -210,7 +210,8 @@ clixon_plugin_init(clicon_handle h)
EOF
new "compile $cfile"
cc -g -Wall -rdynamic -fPIC -shared $cfile -o $sofile
# -I /usr/local_include for eg freebsd
expectpart "$($CC -g -Wall -rdynamic -fPIC -shared -I/usr/local/include $cfile -o $sofile)" 0 ""
new "test params: -s running -f $cfg"

6
test/test_c++.sh
View file

@ -63,9 +63,6 @@ class netconf_test
{
private:
struct clixon_plugin_api api;
plginit2_t *ca_init;
plgstart_t *ca_start;
plgexit_t *ca_exit;
public:
netconf_test(plginit2_t* init, plgstart_t* start, plgexit_t* exit, const char* str = "c++ netconf test") : api()
@ -138,7 +135,8 @@ clixon_plugin_api* clixon_plugin_init(clicon_handle h)
EOF
new "C++ compile"
expectpart "$($CXX -g -Wall -rdynamic -fPIC -shared $cfile -o $dir/backend/c++.so)" 0 ""
# -I /usr/local_include for eg freebsd
expectpart "$($CXX -g -Wall -rdynamic -fPIC -shared -I/usr/local/include $cfile -o $dir/backend/c++.so)" 0 ""
new "test params: -f $cfg"
if [ $BE -ne 0 ]; then

3
test/test_perf.sh
View file

@ -105,6 +105,9 @@ done
echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
# Now take large config file and write it via netconf to candidate
new "test time exists"
expectpart "$(time -p ls)" 0
new "netconf write large config"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'

18
test/vagrant/Makefile.in
View file

@ -49,18 +49,24 @@ VAGRANTS =
-include site.mk
.PHONY: all clean distclean depend install uninstall $(VAGRANTS)
.PHONY: all clean distclean depend install uninstall $(VAGRANTS) destroy
all: $(VAGRANTS)
logs:
test -d logs || mkdir -p logs
# Local vagrant hosts eg generic/centos8. The vagrantdir is a subdirectory and logs will
# appear eg as generic/centos8.log
$(VAGRANTS):
test -d $@ || mkdir -p $@
./vagrant.sh $@ destroy 2>&1 | tee $@.log
$(VAGRANTS): logs
./vagrant.sh $@ 2>&1 | tee "logs/$(subst /,-,$@).log"
clean:
rm -f *.log
destroy:
for i in $(VAGRANTS) ; \
do ./vagrant.sh $$i destroy; done
clean: destroy
rm -f logs/*.log
distclean: clean
rm -f Makefile *~ .depend

13
test/vagrant/README.md
View file

@ -7,7 +7,7 @@ git, configure, makes and runs through the tests. Make is used to get
concurrency - eg with `make -j 10`
The Makefile contains a configurable VAGRANTS variable, which is defined
in a "site.mk" file. You can add such a file, eg:
in a `site.mk` file. You can add such a file, eg:
```
VAGRANTS += freebsd/FreeBSD-12.1-STABLE
VAGRANTS += generic/centos8
@ -15,7 +15,7 @@ in a "site.mk" file. You can add such a file, eg:
Beware memory exhaustion if you run too many simultaneously.
Logs appear in : <dir>/<hostname>.log.
Logs appear in : `<dir>/<hostname>.log.`
You can also run a single vagrant test as follows:
```
@ -23,9 +23,12 @@ You can also run a single vagrant test as follows:
```
The current status is as follows
* freebsd/FreeBSD-12.1-STABLE
* generic/opensuse42
* generic/centos8 - one state test fails
* freebsd/FreeBSD-12.1-STABLE - OK
* ubuntu/xenial64 - OK
* generic/opensuse42 - nginx: [emerg] getgrnam("www-data") failed in /etc/nginx/nginx.conf:2
* generic/centos8 - Error in test_perf_state.sh errcode=255
*
For other vagrant boxes, see [search vagrant boxes](https://vagrantcloud.com/search)

75
test/vagrant/clixon.sh Executable file
View file

@ -0,0 +1,75 @@
#!/usr/bin/env bash
# Setup cligen and clixon
set -ux
#!/usr/bin/env bash
set -eux # x
if [ $# -ne 3 ]; then
echo "usage: $0 <release> <wwwuser> <with_restconf>"
exit -1
fi
release=$1
wwwuser=$2
with_restconf=$3
# create user & group
if [ ! $(id -u clicon) ]; then
if [ $release = "freebsd" ]; then
sudo pw useradd clicon -d /nonexistent -s /usr/sbin/nologin;
sudo pw group mod clicon -m vagrant; # start clixon tests as this users
sudo pw group mod clicon -m $wwwuser;
elif [ $release = "alpine" ]; then
sudo adduser -D -H clicon
sudo adduser $wwwuser clicon
else
sudo useradd -M -U clicon;
sudo usermod -a -G clicon vagrant; # start clixon tests as this users
sudo usermod -a -G clicon $wwwuser;
fi
fi
# cligen
test -d src || mkdir src
test -d src/cligen || (cd src;git clone https://github.com/clicon/cligen.git)
cd src/cligen
git pull
if [ $release = "freebsd" ]; then
./configure
MAKE=$(which gmake)
else
./configure --prefix=/usr
MAKE=$(which make)
fi
echo "MAKE:$MAKE"
$MAKE clean
$MAKE -j10
sudo $MAKE install
# Clixon
cd
test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)
cd src/clixon
git pull
if [ $release = "freebsd" ]; then
LDFLAGS=-L/usr/local/lib ./configure --with-cligen=/usr/local --enable-optyangs --with-restconf=${with_restconf}
else
# Problems with su not having "sbin" in path on centos when when we run tests later
./configure --sbindir=/usr/sbin --libdir=/usr/lib --enable-optyangs --with-restconf=${with_restconf}
fi
$MAKE clean
$MAKE -j10
sudo $MAKE install
(cd example; $MAKE)
(cd util; $MAKE)
(cd example; sudo $MAKE install)
(cd util; sudo $MAKE install)
sudo ldconfig
cd test
echo "#!/usr/bin/env bash" > ./site.sh
if [ $release = "freebsd" ]; then
echo "make=gmake" >> ./site.sh
fi

26
test/vagrant/mem.sh Executable file
View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
reset='\033[0m'
# White Background
BG='\033[47m'
# Black Foreground
FG='\033[0;30m'
printf "%+5s %+6s %+6s %s\n" PID VIRT RES VBOX
pids=$(ps -eo pid,cmd | grep VBoxHeadless |awk '{print $1}')
for pid in $pids; do
# echo "ps -o rss,vsize,cmd -h -p $pid"
line=$(ps -o rss,vsize,cmd -h -p $pid)
if [ -z "$line" ]; then
continue;
fi
rss=$(echo "$line"| awk '{print $1}')
let rss=rss/1000
virt=$(echo "$line"| awk '{print $2}')
let virt=virt/1000
rest=$(echo "$line"| sed 's/^.*--comment//' | sed 's/ --startvm.*$//' | awk -F- '{print $1 "-" $2}')
printf "%+5s %+5sM %+5sM %s\n" $pid $virt $rss $rest
done

1
test/vagrant/nginx.sh
View file

@ -69,6 +69,7 @@ EOF
$scpcmd $dir/nginx.conf vagrant@127.0.0.1:
cat<<'EOF' > $dir/startnginx.sh
#!/usr/bin/env bash
set -x
if [ $# -ne 0 -a $# -ne 1 ]; then
echo "usage: $0 [<prefix>"]
exit

310
test/vagrant/vagrant.sh
View file

@ -6,7 +6,7 @@
# 4. Run tests
# Example run: ./vagrant.sh generic/centos8 2>&1 | tee cilog
set -eux # x
set -eux #
if [ $# -ne 1 -a $# -ne 2 ]; then
echo "usage: $0 <box> [destroy]\n <box> as defined in https://vagrantcloud.com/search"
@ -14,41 +14,73 @@ if [ $# -ne 1 -a $# -ne 2 ]; then
fi
box=$1 # As defined in https://vagrantcloud.com/search
#with_restconf=fcgi
: ${with_restconf:=evhtp}
VCPUS=1
MEM=1024
# This is a hack just to get the linux release for provisioning
linuxrelease()
{
box=$1
release="unknown"
for r in freebsd openbsd opensuse ubuntu centos coreos alpine debian arch gentoo fedora rhel; do
# -i ignore case
if [ -n "$(echo "$box" | grep -io "$r")" ]; then
release=$r
break
fi
done
echo "$release"
}
if [ $# -eq 2 ]; then
destroy=true
else
destroy=false
fi
host=$(echo "$box"|awk -F'/' '{print $2}')
dir=$box
# XXX: ad-hoc to get (linux) release from boxname
# using lsb_release is too heavyweight in many cases
release=$(echo "$host" | grep -io "[a-z]*" | head -1 | tr '[:upper:]' '[:lower:]')
# Convert eg centos/8 -> centos-8 and use that as dir and hostname
host=$(echo "$box"|sed -e "s/\//-/")
dir=$host
wwwuser=www-data
# example box="freebsd/FreeBSD-12.1-STABLE"
# XXX ad.hoc to get release (lsb-release is too heavyweight)
release=$(linuxrelease $box)
echo "release:$release"
test -d $dir || mkdir -p $dir
# Write a freebsd vagrant file
# Write a vagrant file
cat<<EOF > $dir/Vagrantfile
Vagrant.configure("2") do |config|
# Every Vagrant development environment requires a box. You can search for
# boxes at https://vagrantcloud.com/search.
config.vm.box = "$box"
if Vagrant.has_plugin?("vagrant-vbguest")
config.vbguest.auto_update = false
end
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.box_check_update = true
config.ssh.shell = "sh" # freebsd
config.vm.define "$host"
config.vm.hostname = "$host"
config.vm.provider "virtualbox" do |v|
v.memory = $MEM
v.cpus = $VCPUS
end
end
EOF
# Start vagrant
if $destroy; then
(cd $dir; vagrant destroy -f)
exit 0
fi
(cd $dir; vagrant up)
echo "vagrant is up -----------------"
# Get ssh config to make proper ssh/scp calls to local vagrant host
cfg=$(cd $dir; vagrant ssh-config $host)
idfile=$(echo "$cfg" |grep "IdentityFile"|awk '{print $2}')
@ -59,156 +91,182 @@ scpcmd="scp -p -o StrictHostKeyChecking=no -i $idfile -P $port"
ssh-keygen -f "$HOME/.ssh/known_hosts" -R "[127.0.0.1]:$port"
echo "$sshcmd"
system=$($sshcmd uname)
system=$($sshcmd uname) # we use the release "hack" instead
# Some release have packages, some need to be built from source
buildfcgi=false
case $system in
FreeBSD)
buildevhtp=false
case $release in
openbsd)
# packages for building
$sshcmd sudo pkg install -y git gmake bash
# cligen
$sshcmd sudo pkg install -y bison flex
# Add www user for nginx
# Add restconf user
if [ ! $($sshcmd id -u $wwwuser) ]; then
$sshcmd sudo pw useradd $wwwuser -d /nonexistent -s /usr/sbin/nologin
fi
$sshcmd sudo pkg install -y fcgi-devkit nginx
case ${with_restconf} in
fcgi)
$sshcmd sudo pkg install -y fcgi-devkit nginx
;;
evhtp)
;;
esac
;;
freebsd)
# packages for building
$sshcmd sudo pkg install -y git gmake bash
# cligen
$sshcmd sudo pkg install -y bison flex
# Add restconf user
if [ ! $($sshcmd id -u $wwwuser) ]; then
$sshcmd sudo pw useradd $wwwuser -d /nonexistent -s /usr/sbin/nologin
fi
case ${with_restconf} in
fcgi)
$sshcmd sudo pkg install -y fcgi-devkit nginx
;;
evhtp)
$sshcmd sudo pkg install -y libevent cmake libevhtp
;;
esac
;;
Linux)
# nginx restconf user: $wwwuser
centos)
# add restconf user: $wwwuser
if [ ! $($sshcmd id -u $wwwuser) ]; then
$sshcmd sudo useradd -M $wwwuser
fi
case $release in
centos) # centos 8
# packages for building
$sshcmd sudo yum install -y git
# cligen
$sshcmd sudo yum install -y bison flex
# clixon
$sshcmd sudo yum install -y fcgi-devel nginx
# clixon utilities
$sshcmd sudo yum install -y libcurl-devel
# packages for building
$sshcmd sudo yum install -y git
# cligen
$sshcmd sudo yum install -y bison flex
# clixon utilities
$sshcmd sudo yum install -y time libcurl-devel gcc-c++
# restconf
case ${with_restconf} in
fcgi)
buildfcgi=true # build fcgi from source
$sshcmd sudo yum install -y epel-release
# $sshcmd sudo yum update
$sshcmd sudo yum install -y nginx
;;
opensuse) # opensuse42
# packages for building
$sshcmd sudo zypper install -y git
# cligen
$sshcmd sudo zypper install -y bison flex
# clixon
evhtp)
$sshcmd sudo yum install -y libevent openssl
buildevhtp=true
$sshcmd sudo yum install -y libevent-devel cmake openssl-devel
;;
esac
;;
opensuse) # opensuse42
# restconf user: $wwwuser
if [ ! $($sshcmd id -u $wwwuser) ]; then
$sshcmd sudo useradd -M -U $wwwuser
fi
# packages for building
$sshcmd sudo zypper install -y git
# cligen
$sshcmd sudo zypper install -y bison flex
# clixon utilities
$sshcmd sudo zypper install -y libcurl-devel gcc-c++
# restconf
case ${with_restconf} in
fcgi)
$sshcmd sudo zypper install -y nginx
buildfcgi=true # build fcgi from source
# clixon utilities
$sshcmd sudo zypper install -y libcurl-devel
# packages for building fcgi
$sshcmd sudo zypper install -y autoconf automake libtool
;;
*) # ubuntu/apt based
# cligen
$sshcmd sudo apt install -y bison flex
# clixon
$sshcmd sudo apt install -y libfcgi-dev nginx
# clixon utilities
$sshcmd sudo apt install -y libcurl4-openssl-dev
evhtp)
;;
esac
;;
ubuntu) # ubuntu/apt based
$sshcmd sudo apt install -y git
# restconf user: $wwwuser
if [ ! $($sshcmd id -u $wwwuser) ]; then
$sshcmd sudo useradd -M $wwwuser
fi
# cligen
$sshcmd sudo apt install -y bison flex make
# clixon utilities
$sshcmd sudo apt install -y libcurl4-openssl-dev
$sshcmd sudo apt install -y g++
# restconf
case ${with_restconf} in
fcgi)
buildfcgi=true # some ubuntu dont have fcgidev
$sshcmd sudo apt install -y nginx
;;
evhtp)
$sshcmd sudo apt install -y libevent-2.1
buildevhtp=true
$sshcmd sudo apt install -y libevent-dev cmake libssl-dev
;;
esac
;;
alpine)
if [ ! $($sshcmd id -u $wwwuser) ]; then
$sshcmd sudo adduser -D -H $wwwuser
fi
$sshcmd sudo apk add --update git make build-base gcc flex bison curl-dev g++
# restconf
case ${with_restconf} in
fcgi)
$sshcmd sudo apk add --update nginx fcgi-dev
;;
evhtp)
;;
esac
;;
arch)
$sshcmd sudo pacman -S --noconfirm git
# restconf
case ${with_restconf} in
fcgi)
$sshcmd sudo pacman -S --noconfirm nginx fcgi
;;
evhtp)
;;
esac
;;
*)
echo "Unknown system: $system"
echo "Unknown release: $release"
;;
esac
# Some platforms dont have fcgi, build the source (should all?)
if $buildfcgi; then
$sshcmd "test -d fcgi2 || git clone https://github.com/FastCGI-Archives/fcgi2"
$sshcmd "(cd fcgi2; ./autogen.sh; ./configure; make; sudo make install)"
test -d $dir/fcgi2 || (cd $dir;git clone https://github.com/FastCGI-Archives/fcgi2)
(cd $dir/fcgi2; ./autogen.sh; rm -rf .git)
$scpcmd -r $dir/fcgi2 vagrant@127.0.0.1:
$sshcmd "(cd fcgi2; ./configure --prefix=/usr; make; sudo make install)"
fi
# Hide all complex nginx config in sub-script
. ./nginx.sh $dir $idfile $port $wwwuser
case ${with_restconf} in
fcgi)
# Hide all complex nginx config in sub-script
. ./nginx.sh $dir $idfile $port $wwwuser
;;
evhtp)
if $buildevhtp; then
$sshcmd "test -d libevhtp || sudo git clone https://github.com/criticalstack/libevhtp.git"
$sshcmd "(cd libevhtp/build; sudo cmake -DEVHTP_DISABLE_REGEX=ON -DEVHTP_DISABLE_EVTHR=ON ..)"
$sshcmd "(cd libevhtp/build; sudo make)"
$sshcmd "(cd libevhtp/build; sudo make install)"
fi
;;
esac
# Setup cligen and clixon
# This is a script generated at the original host, then copied to the target and run there.
# 'EOF' means dont expand $
cat<<'EOF' > $dir/setup.sh
#!/usr/bin/env bash
set -eux # x
if [ $# -ne 2 ]; then
echo "usage: $0 <release> <wwwuser>"
exit -1
fi
release=$1
wwwuser=$2
# create user & group
if [ ! $(id -u clicon) ]; then
if [ $release = "freebsd" ]; then
sudo pw useradd clicon -d /nonexistent -s /usr/sbin/nologin;
sudo pw group mod clicon -m vagrant; # start clixon tests as this users
sudo pw group mod clicon -m $wwwuser;
else
sudo useradd -M -U clicon;
sudo usermod -a -G clicon vagrant; # start clixon tests as this users
sudo usermod -a -G clicon $wwwuser;
fi
fi
# cligen
test -d src || mkdir src
test -d src/cligen || (cd src;git clone https://github.com/clicon/cligen.git)
cd src/cligen
git pull
if [ $release = "freebsd" ]; then
./configure
MAKE=$(which gmake)
else
./configure --prefix=/usr
MAKE=$(which make)
fi
echo "MAKE:$MAKE"
$MAKE clean
$MAKE -j10
sudo $MAKE install
# Clixon
cd
test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)
cd src/clixon
git pull
if [ $release = "freebsd" ]; then
LDFLAGS=-L/usr/local/lib ./configure --with-cligen=/usr/local --enable-optyangs
else
# Problems with su not having "sbin" in path on centos when when we run tests later
./configure --sbindir=/usr/sbin --libdir=/usr/lib --enable-optyangs
fi
$MAKE clean
$MAKE -j10
sudo $MAKE install
(cd example; $MAKE)
(cd util; $MAKE)
(cd example; sudo $MAKE install)
(cd util; sudo $MAKE install)
sudo ldconfig
cd test
echo "#!/usr/bin/env bash" > ./site.sh
if [ $release = "freebsd" ]; then
echo "make=gmake" >> ./site.sh
echo 'SKIPLIST="test_api.sh"' >> ./site.sh
fi
EOF
chmod a+x $dir/setup.sh
# config and setup cligen and clixon
$scpcmd $dir/setup.sh vagrant@127.0.0.1:
$sshcmd ./setup.sh $release $wwwuser
$scpcmd ./clixon.sh vagrant@127.0.0.1:
$sshcmd ./clixon.sh $release $wwwuser ${with_restconf}
# Run tests
$sshcmd "(cd src/cligen/test; ./sum.sh)"
$sshcmd "(cd src/clixon/test; ./sum.sh)"
# destroy vm
if $destroy; then
(cd $dir; vagrant destroy -f)
fi
#if $destroy; then
# (cd $dir; vagrant destroy -f)
#fi