diff --git a/apps/restconf/restconf_api_evhtp.c b/apps/restconf/restconf_api_evhtp.c index b626d295..289036c2 100644 --- a/apps/restconf/restconf_api_evhtp.c +++ b/apps/restconf/restconf_api_evhtp.c @@ -65,6 +65,15 @@ #include "restconf_lib.h" #include "restconf_api.h" /* Virtual api */ +/* evhtp_safe_free is a macro that may not be present in a libevhtp release + */ +#ifndef evhtp_safe_free +#define evhtp_safe_free(_var, _freefn) do { \ + _freefn((_var)); \ + (_var) = NULL; \ +} while (0) +#endif + /*! Add HTTP header field name and value to reply, evhtp specific * @param[in] req Evhtp http request handle * @param[in] name HTTP header field name diff --git a/apps/restconf/restconf_handle.h b/apps/restconf/restconf_handle.h index 7d286099..467165ea 100644 --- a/apps/restconf/restconf_handle.h +++ b/apps/restconf/restconf_handle.h @@ -37,7 +37,7 @@ */ #ifndef _RESTCONF_HANDLE_H_ -#define _RESCTONF_HANDLE_H_ +#define _RESTCONF_HANDLE_H_ /* * Prototypes diff --git a/test/vagrant/Makefile.in b/test/vagrant/Makefile.in index 671ca482..bcee54ee 100644 --- a/test/vagrant/Makefile.in +++ b/test/vagrant/Makefile.in @@ -49,18 +49,24 @@ VAGRANTS = -include site.mk -.PHONY: all clean distclean depend install uninstall $(VAGRANTS) +.PHONY: all clean distclean depend install uninstall $(VAGRANTS) destroy all: $(VAGRANTS) +logs: + test -d logs || mkdir -p logs + # Local vagrant hosts eg generic/centos8. The vagrantdir is a subdirectory and logs will # appear eg as generic/centos8.log -$(VAGRANTS): - test -d $@ || mkdir -p $@ - ./vagrant.sh $@ destroy 2>&1 | tee $@.log +$(VAGRANTS): logs + ./vagrant.sh $@ 2>&1 | tee "logs/$(subst /,-,$@).log" -clean: - rm -f *.log +destroy: + for i in $(VAGRANTS) ; \ + do ./vagrant.sh $$i destroy; done + +clean: destroy + rm -f logs/*.log distclean: clean rm -f Makefile *~ .depend diff --git a/test/vagrant/README.md b/test/vagrant/README.md index a150c213..91173494 100644 --- a/test/vagrant/README.md +++ b/test/vagrant/README.md @@ -7,7 +7,7 @@ git, configure, makes and runs through the tests. Make is used to get concurrency - eg with `make -j 10` The Makefile contains a configurable VAGRANTS variable, which is defined -in a "site.mk" file. You can add such a file, eg: +in a `site.mk` file. You can add such a file, eg: ``` VAGRANTS += freebsd/FreeBSD-12.1-STABLE VAGRANTS += generic/centos8 @@ -15,7 +15,7 @@ in a "site.mk" file. You can add such a file, eg: Beware memory exhaustion if you run too many simultaneously. -Logs appear in : /.log. +Logs appear in : `/.log.` You can also run a single vagrant test as follows: ``` @@ -23,9 +23,12 @@ You can also run a single vagrant test as follows: ``` The current status is as follows -* freebsd/FreeBSD-12.1-STABLE -* generic/opensuse42 -* generic/centos8 - one state test fails +* freebsd/FreeBSD-12.1-STABLE - OK +* ubuntu/xenial64 - OK +* generic/opensuse42 - nginx: [emerg] getgrnam("www-data") failed in /etc/nginx/nginx.conf:2 + +* generic/centos8 - Error in test_perf_state.sh errcode=255 +* For other vagrant boxes, see [search vagrant boxes](https://vagrantcloud.com/search) diff --git a/test/vagrant/clixon.sh b/test/vagrant/clixon.sh new file mode 100755 index 00000000..07687444 --- /dev/null +++ b/test/vagrant/clixon.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Setup cligen and clixon + +set -ux + +#!/usr/bin/env bash +set -eux # x + +if [ $# -ne 3 ]; then + echo "usage: $0 " + exit -1 +fi +release=$1 +wwwuser=$2 +with_restconf=$3 + +# create user & group +if [ ! $(id -u clicon) ]; then + if [ $release = "freebsd" ]; then + sudo pw useradd clicon -d /nonexistent -s /usr/sbin/nologin; + sudo pw group mod clicon -m vagrant; # start clixon tests as this users + sudo pw group mod clicon -m $wwwuser; + elif [ $release = "alpine" ]; then + sudo adduser -D -H clicon + sudo adduser $wwwuser clicon + else + sudo useradd -M -U clicon; + sudo usermod -a -G clicon vagrant; # start clixon tests as this users + sudo usermod -a -G clicon $wwwuser; + fi +fi + +# cligen +test -d src || mkdir src +test -d src/cligen || (cd src;git clone https://github.com/clicon/cligen.git) +cd src/cligen +git pull + +if [ $release = "freebsd" ]; then + ./configure + MAKE=$(which gmake) +else + ./configure --prefix=/usr + MAKE=$(which make) +fi +echo "MAKE:$MAKE" +$MAKE clean +$MAKE -j10 +sudo $MAKE install + +# Clixon +cd +test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git) +cd src/clixon +git pull + +if [ $release = "freebsd" ]; then + LDFLAGS=-L/usr/local/lib ./configure --with-cligen=/usr/local --enable-optyangs --with-restconf=${with_restconf} +else + # Problems with su not having "sbin" in path on centos when when we run tests later + ./configure --sbindir=/usr/sbin --libdir=/usr/lib --enable-optyangs --with-restconf=${with_restconf} +fi +$MAKE clean +$MAKE -j10 +sudo $MAKE install +(cd example; $MAKE) +(cd util; $MAKE) +(cd example; sudo $MAKE install) +(cd util; sudo $MAKE install) +sudo ldconfig +cd test +echo "#!/usr/bin/env bash" > ./site.sh +if [ $release = "freebsd" ]; then + echo "make=gmake" >> ./site.sh + echo 'SKIPLIST="test_api.sh test_c++.sh"' >> ./site.sh +fi diff --git a/test/vagrant/mem.sh b/test/vagrant/mem.sh new file mode 100755 index 00000000..106e1d8b --- /dev/null +++ b/test/vagrant/mem.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +reset='\033[0m' + +# White Background +BG='\033[47m' + +# Black Foreground +FG='\033[0;30m' + +printf "%+5s %+6s %+6s %s\n" PID VIRT RES VBOX +pids=$(ps -eo pid,cmd | grep VBoxHeadless |awk '{print $1}') +for pid in $pids; do +# echo "ps -o rss,vsize,cmd -h -p $pid" + line=$(ps -o rss,vsize,cmd -h -p $pid) + if [ -z "$line" ]; then + continue; + fi + rss=$(echo "$line"| awk '{print $1}') + let rss=rss/1000 + virt=$(echo "$line"| awk '{print $2}') + let virt=virt/1000 + rest=$(echo "$line"| sed 's/^.*--comment//' | sed 's/ --startvm.*$//' | awk -F- '{print $1 "-" $2}') + printf "%+5s %+5sM %+5sM %s\n" $pid $virt $rss $rest + +done + diff --git a/test/vagrant/nginx.sh b/test/vagrant/nginx.sh index bc4b239c..119970d2 100755 --- a/test/vagrant/nginx.sh +++ b/test/vagrant/nginx.sh @@ -69,6 +69,7 @@ EOF $scpcmd $dir/nginx.conf vagrant@127.0.0.1: cat<<'EOF' > $dir/startnginx.sh #!/usr/bin/env bash + set -x if [ $# -ne 0 -a $# -ne 1 ]; then echo "usage: $0 ["] exit diff --git a/test/vagrant/vagrant.sh b/test/vagrant/vagrant.sh index fb7c7df8..a9d18196 100755 --- a/test/vagrant/vagrant.sh +++ b/test/vagrant/vagrant.sh @@ -6,7 +6,7 @@ # 4. Run tests # Example run: ./vagrant.sh generic/centos8 2>&1 | tee cilog -set -eux # x +set -eux # if [ $# -ne 1 -a $# -ne 2 ]; then echo "usage: $0 [destroy]\n as defined in https://vagrantcloud.com/search" @@ -14,41 +14,73 @@ if [ $# -ne 1 -a $# -ne 2 ]; then fi box=$1 # As defined in https://vagrantcloud.com/search + +#with_restconf=fcgi +: ${with_restconf:=evhtp} + +VCPUS=1 +MEM=1024 + +# This is a hack just to get the linux release for provisioning +linuxrelease() +{ + box=$1 + release="unknown" + for r in freebsd openbsd opensuse ubuntu centos coreos alpine debian arch gentoo fedora rhel; do + # -i ignore case + if [ -n "$(echo "$box" | grep -io "$r")" ]; then + release=$r + break + fi + done + echo "$release" +} + if [ $# -eq 2 ]; then destroy=true else destroy=false fi -host=$(echo "$box"|awk -F'/' '{print $2}') -dir=$box -# XXX: ad-hoc to get (linux) release from boxname -# using lsb_release is too heavyweight in many cases -release=$(echo "$host" | grep -io "[a-z]*" | head -1 | tr '[:upper:]' '[:lower:]') +# Convert eg centos/8 -> centos-8 and use that as dir and hostname +host=$(echo "$box"|sed -e "s/\//-/") +dir=$host wwwuser=www-data -# example box="freebsd/FreeBSD-12.1-STABLE" +# XXX ad.hoc to get release (lsb-release is too heavyweight) +release=$(linuxrelease $box) +echo "release:$release" + test -d $dir || mkdir -p $dir -# Write a freebsd vagrant file +# Write a vagrant file cat< $dir/Vagrantfile Vagrant.configure("2") do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. config.vm.box = "$box" + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + config.vm.synced_folder ".", "/vagrant", disabled: true config.vm.box_check_update = true config.ssh.shell = "sh" # freebsd config.vm.define "$host" config.vm.hostname = "$host" + config.vm.provider "virtualbox" do |v| + v.memory = $MEM + v.cpus = $VCPUS + end end EOF # Start vagrant if $destroy; then (cd $dir; vagrant destroy -f) + exit 0 fi (cd $dir; vagrant up) - +echo "vagrant is up -----------------" # Get ssh config to make proper ssh/scp calls to local vagrant host cfg=$(cd $dir; vagrant ssh-config $host) idfile=$(echo "$cfg" |grep "IdentityFile"|awk '{print $2}') @@ -59,156 +91,182 @@ scpcmd="scp -p -o StrictHostKeyChecking=no -i $idfile -P $port" ssh-keygen -f "$HOME/.ssh/known_hosts" -R "[127.0.0.1]:$port" echo "$sshcmd" -system=$($sshcmd uname) +system=$($sshcmd uname) # we use the release "hack" instead +# Some release have packages, some need to be built from source buildfcgi=false -case $system in - FreeBSD) +buildevhtp=false +case $release in + openbsd) # packages for building $sshcmd sudo pkg install -y git gmake bash # cligen $sshcmd sudo pkg install -y bison flex - # Add www user for nginx + # Add restconf user if [ ! $($sshcmd id -u $wwwuser) ]; then $sshcmd sudo pw useradd $wwwuser -d /nonexistent -s /usr/sbin/nologin fi - $sshcmd sudo pkg install -y fcgi-devkit nginx + case ${with_restconf} in + fcgi) + $sshcmd sudo pkg install -y fcgi-devkit nginx + ;; + evhtp) + ;; + esac + ;; + freebsd) + # packages for building + $sshcmd sudo pkg install -y git gmake bash + # cligen + $sshcmd sudo pkg install -y bison flex + # Add restconf user + if [ ! $($sshcmd id -u $wwwuser) ]; then + $sshcmd sudo pw useradd $wwwuser -d /nonexistent -s /usr/sbin/nologin + fi + case ${with_restconf} in + fcgi) + $sshcmd sudo pkg install -y fcgi-devkit nginx + ;; + evhtp) + $sshcmd sudo pkg install -y libevent cmake libevhtp + ;; + esac ;; - Linux) - # nginx restconf user: $wwwuser + centos) + # add restconf user: $wwwuser if [ ! $($sshcmd id -u $wwwuser) ]; then $sshcmd sudo useradd -M $wwwuser fi - case $release in - centos) # centos 8 - # packages for building - $sshcmd sudo yum install -y git - # cligen - $sshcmd sudo yum install -y bison flex - # clixon - $sshcmd sudo yum install -y fcgi-devel nginx - # clixon utilities - $sshcmd sudo yum install -y libcurl-devel + # packages for building + $sshcmd sudo yum install -y git + # cligen + $sshcmd sudo yum install -y bison flex + # clixon utilities + $sshcmd sudo yum install -y time libcurl-devel gcc-c++ + # restconf + case ${with_restconf} in + fcgi) + buildfcgi=true # build fcgi from source + $sshcmd sudo yum install -y epel-release + # $sshcmd sudo yum update + $sshcmd sudo yum install -y nginx ;; - opensuse) # opensuse42 - # packages for building - $sshcmd sudo zypper install -y git - # cligen - $sshcmd sudo zypper install -y bison flex - # clixon + evhtp) + $sshcmd sudo yum install -y libevent openssl + buildevhtp=true + $sshcmd sudo yum install -y libevent-devel cmake openssl-devel + ;; + esac + ;; + opensuse) # opensuse42 + # restconf user: $wwwuser + if [ ! $($sshcmd id -u $wwwuser) ]; then + $sshcmd sudo useradd -M -U $wwwuser + fi + # packages for building + $sshcmd sudo zypper install -y git + # cligen + $sshcmd sudo zypper install -y bison flex + # clixon utilities + $sshcmd sudo zypper install -y libcurl-devel gcc-c++ + # restconf + case ${with_restconf} in + fcgi) $sshcmd sudo zypper install -y nginx buildfcgi=true # build fcgi from source - # clixon utilities - $sshcmd sudo zypper install -y libcurl-devel - # packages for building fcgi - $sshcmd sudo zypper install -y autoconf automake libtool ;; - *) # ubuntu/apt based - # cligen - $sshcmd sudo apt install -y bison flex - # clixon - $sshcmd sudo apt install -y libfcgi-dev nginx - # clixon utilities - $sshcmd sudo apt install -y libcurl4-openssl-dev + evhtp) + ;; + esac + ;; + ubuntu) # ubuntu/apt based + $sshcmd sudo apt install -y git + # restconf user: $wwwuser + if [ ! $($sshcmd id -u $wwwuser) ]; then + $sshcmd sudo useradd -M $wwwuser + fi + # cligen + $sshcmd sudo apt install -y bison flex make + # clixon utilities + $sshcmd sudo apt install -y libcurl4-openssl-dev + $sshcmd sudo apt install -y g++ + # restconf + case ${with_restconf} in + fcgi) + buildfcgi=true # some ubuntu dont have fcgidev + $sshcmd sudo apt install -y nginx + ;; + evhtp) + $sshcmd sudo apt install -y libevent-2.1 + buildevhtp=true + $sshcmd sudo apt install -y libevent-dev cmake libssl-dev + + ;; + esac + ;; + alpine) + if [ ! $($sshcmd id -u $wwwuser) ]; then + $sshcmd sudo adduser -D -H $wwwuser + fi + $sshcmd sudo apk add --update git make build-base gcc flex bison curl-dev g++ + + # restconf + case ${with_restconf} in + fcgi) + $sshcmd sudo apk add --update nginx fcgi-dev + ;; + evhtp) + ;; + esac + ;; + arch) + $sshcmd sudo pacman -S --noconfirm git + # restconf + case ${with_restconf} in + fcgi) + $sshcmd sudo pacman -S --noconfirm nginx fcgi + ;; + evhtp) ;; esac ;; *) - echo "Unknown system: $system" + echo "Unknown release: $release" ;; esac # Some platforms dont have fcgi, build the source (should all?) if $buildfcgi; then - $sshcmd "test -d fcgi2 || git clone https://github.com/FastCGI-Archives/fcgi2" - $sshcmd "(cd fcgi2; ./autogen.sh; ./configure; make; sudo make install)" + test -d $dir/fcgi2 || (cd $dir;git clone https://github.com/FastCGI-Archives/fcgi2) + (cd $dir/fcgi2; ./autogen.sh; rm -rf .git) + $scpcmd -r $dir/fcgi2 vagrant@127.0.0.1: + $sshcmd "(cd fcgi2; ./configure --prefix=/usr; make; sudo make install)" fi -# Hide all complex nginx config in sub-script -. ./nginx.sh $dir $idfile $port $wwwuser +case ${with_restconf} in + fcgi) + # Hide all complex nginx config in sub-script + . ./nginx.sh $dir $idfile $port $wwwuser + ;; + evhtp) + if $buildevhtp; then + $sshcmd "test -d libevhtp || sudo git clone https://github.com/criticalstack/libevhtp.git" + $sshcmd "(cd libevhtp/build; sudo cmake -DEVHTP_DISABLE_REGEX=ON -DEVHTP_DISABLE_EVTHR=ON ..)" + $sshcmd "(cd libevhtp/build; sudo make)" + $sshcmd "(cd libevhtp/build; sudo make install)" + fi + ;; +esac # Setup cligen and clixon -# This is a script generated at the original host, then copied to the target and run there. -# 'EOF' means dont expand $ -cat<<'EOF' > $dir/setup.sh -#!/usr/bin/env bash -set -eux # x - -if [ $# -ne 2 ]; then - echo "usage: $0 " - exit -1 -fi -release=$1 -wwwuser=$2 -# create user & group -if [ ! $(id -u clicon) ]; then - if [ $release = "freebsd" ]; then - sudo pw useradd clicon -d /nonexistent -s /usr/sbin/nologin; - sudo pw group mod clicon -m vagrant; # start clixon tests as this users - sudo pw group mod clicon -m $wwwuser; - else - sudo useradd -M -U clicon; - sudo usermod -a -G clicon vagrant; # start clixon tests as this users - sudo usermod -a -G clicon $wwwuser; - fi -fi - -# cligen -test -d src || mkdir src -test -d src/cligen || (cd src;git clone https://github.com/clicon/cligen.git) -cd src/cligen -git pull - -if [ $release = "freebsd" ]; then - ./configure - MAKE=$(which gmake) -else - ./configure --prefix=/usr - MAKE=$(which make) -fi -echo "MAKE:$MAKE" -$MAKE clean -$MAKE -j10 -sudo $MAKE install - -# Clixon -cd -test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git) -cd src/clixon -git pull - -if [ $release = "freebsd" ]; then - LDFLAGS=-L/usr/local/lib ./configure --with-cligen=/usr/local --enable-optyangs -else - # Problems with su not having "sbin" in path on centos when when we run tests later - ./configure --sbindir=/usr/sbin --libdir=/usr/lib --enable-optyangs -fi -$MAKE clean -$MAKE -j10 -sudo $MAKE install -(cd example; $MAKE) -(cd util; $MAKE) -(cd example; sudo $MAKE install) -(cd util; sudo $MAKE install) -sudo ldconfig -cd test -echo "#!/usr/bin/env bash" > ./site.sh -if [ $release = "freebsd" ]; then - echo "make=gmake" >> ./site.sh - echo 'SKIPLIST="test_api.sh"' >> ./site.sh -fi -EOF -chmod a+x $dir/setup.sh - -# config and setup cligen and clixon -$scpcmd $dir/setup.sh vagrant@127.0.0.1: -$sshcmd ./setup.sh $release $wwwuser +$scpcmd ./clixon.sh vagrant@127.0.0.1: +$sshcmd ./clixon.sh $release $wwwuser ${with_restconf} # Run tests $sshcmd "(cd src/cligen/test; ./sum.sh)" $sshcmd "(cd src/clixon/test; ./sum.sh)" # destroy vm -if $destroy; then - (cd $dir; vagrant destroy -f) -fi +#if $destroy; then +# (cd $dir; vagrant destroy -f) +#fi