From 9e2bdb8c8e01e46177fa988558147371a8eba6f8 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 18 Jul 2020 15:52:12 +0200 Subject: [PATCH] split test_perf into test_perf_netconf, _restconf and _cli --- test/test_perf_cli.sh | 171 ++++++++++++++++ test/{test_perf.sh => test_perf_netconf.sh} | 73 +------ test/test_perf_restconf.sh | 216 ++++++++++++++++++++ 3 files changed, 388 insertions(+), 72 deletions(-) create mode 100755 test/test_perf_cli.sh rename test/{test_perf.sh => test_perf_netconf.sh} (76%) create mode 100755 test/test_perf_restconf.sh diff --git a/test/test_perf_cli.sh b/test/test_perf_cli.sh new file mode 100755 index 00000000..388f9a70 --- /dev/null +++ b/test/test_perf_cli.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Scaling/ performance tests for CLI +# Lists (and leaf-lists) +# Add, get and delete entries + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +# Which format to use as datastore format internally +: ${format:=xml} + +# Number of list/leaf-list entries in file +: ${perfnr:=10000} + +# Number of requests made get/put +: ${perfreq:=10} + +# time function (this is a mess to get right on freebsd/linux) +# -f %e gives elapsed wall clock time but is not available on all systems +# so we use time -p for POSIX compliance and awk to get wall clock time +# Note sometimes time -p is used and sometimes $TIMEFN, cant get it to work same everywhere +: ${TIMEFN:=time -p} # portability: 2>&1 | awk '/real/ {print $2}' + +APPNAME=example + +cfg=$dir/scaling-conf.xml +fyang=$dir/scaling.yang +fconfig=$dir/large.xml +fconfig2=$dir/large2.xml + +cat < $fyang +module scaling{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix ex; + container x { + list y { + key "a"; + leaf a { + type int32; + } + leaf b { + type int32; + } + } + leaf-list c { + type string; + } + } +} +EOF + +cat < $cfg + + $cfg + $dir + /usr/local/share/clixon + $fyang + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/example/$APPNAME.pidfile + false + $dir + false + $format + example + /usr/local/lib/example/cli + /usr/local/lib/example/clispec + 0 + ietf-netconf:startup + +EOF + +new "test params: -f $cfg" +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s +fi + +new "waiting" +wait_backend + +new "generate config with $perfnr list entries" +echo -n "" > $fconfig +for (( i=0; i<$perfnr; i++ )); do + echo -n "$i$i" >> $fconfig +done +echo "]]>]]>" >> $fconfig + +# Now take large config file and write it via netconf to candidate +new "test time exists" +expectpart "$(time -p ls)" 0 + +new "netconf write large config" +expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# Here, there are $perfnr entries in candidate +new "netconf write large config again" +expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# Now commit it from candidate to running +new "netconf commit large config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# CLI get (XXX why does this take so much time?) +# See: EXPAND_ONLY_INTERACTIVE in cligen. If set it is acceptable but there are some side-effects +new "cli get $perfreq small config 1 key index" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg show conf xml x y $rnd > /dev/null +done } 2>&1 | awk '/real/ {print $2}' + +# CLI add +new "cli add $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg set x y $rnd b $rnd +done } 2>&1 | awk '/real/ {print $2}' + +new "cli get large config" +$TIMEFN $clixon_cli -1f $cfg show config xml 2>&1 > /dev/null | awk '/real/ {print $2}' + +# Delete entries (last since entries are removed from db) +# netconf +new "cli delete $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg delete x y $rnd +done } 2>&1 | awk '/real/ {print $2}' + +# Now do leaf-lists instead of leafs + +#new "generate leaf-list config" +echo -n "replace" > $fconfig2 +for (( i=0; i<$perfnr; i++ )); do + echo -n "$i" >> $fconfig2 +done +echo "]]>]]>" >> $fconfig2 + +new "netconf replace large list-leaf config" +expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig2" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +new "netconf commit large leaf-list config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# XXX No leafref cli tests + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=$(pgrep -u root -f clixon_backend) +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +rm -rf $dir + +# unset conditional parameters +unset format +unset perfnr +unset perfreq diff --git a/test/test_perf.sh b/test/test_perf_netconf.sh similarity index 76% rename from test/test_perf.sh rename to test/test_perf_netconf.sh index 888e35f6..0cc49414 100755 --- a/test/test_perf.sh +++ b/test/test_perf_netconf.sh @@ -86,17 +86,6 @@ fi new "waiting" wait_backend -if [ $RC -ne 0 ]; then - new "kill old restconf daemon" - stop_restconf_pre - - new "start restconf daemon" - start_restconf -f $cfg - - new "waiting" - wait_restconf -fi - new "generate config with $perfnr list entries" echo -n "" > $fconfig for (( i=0; i<$perfnr; i++ )); do @@ -148,59 +137,13 @@ new "netconf add $perfreq small config" echo "$rnd$rnd]]>]]>" done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' -# RESTCONF get -new "restconf get $perfreq small config 1 key index" -{ time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - curl $CURLOPTS -X GET $RCPROTO://localhost/restconf/data/scaling:x/y=$rnd > /dev/null -done } 2>&1 | awk '/real/ {print $2}' - -# RESTCONF put -# Reference: -# i686 format=xml perfnr=10000/100 time: 38/29s 20190425 WITH/OUT startup copying -# i686 format=tree perfnr=10000/100 time: 72/64s 20190425 WITH/OUT startup copying -new "restconf add $perfreq small config" -{ time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - curl $CURLOPTS -X PUT $RCPROTO://localhost/restconf/data/scaling:x/y=$rnd -d '{"scaling:y":{"a":"'$rnd'","b":"'$rnd'"}}' -done } 2>&1 | awk '/real/ {print $2}' - -# CLI get (XXX why does this take so much time?) -# See: EXPAND_ONLY_INTERACTIVE in cligen. If set it is acceptable but there are some side-effects -new "cli get $perfreq small config 1 key index" -{ time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - $clixon_cli -1 -f $cfg show conf xml x y $rnd > /dev/null -done } 2>&1 | awk '/real/ {print $2}' - -# CLI add -new "cli add $perfreq small config" -{ time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - $clixon_cli -1 -f $cfg set x y $rnd b $rnd -done } 2>&1 | awk '/real/ {print $2}' - # Instead of many small entries, get one large in netconf and restconf # cli? new "netconf get large config" expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" '^00112233' 2>&1 | awk '/real/ {print $2}' -new "restconf get large config" -# XXX for some reason cannot expand $TIMEFN next two tests, need keep variable? -$TIMEFN curl $CURLOPTS -X GET $RCPROTO://localhost/restconf/data 2>&1 > /dev/null | awk '/real/ {print $2}' - -new "cli get large config" -$TIMEFN $clixon_cli -1f $cfg show config xml 2>&1 > /dev/null | awk '/real/ {print $2}' - # Delete entries (last since entries are removed from db) -# netconf -new "cli delete $perfreq small config" -{ time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - $clixon_cli -1 -f $cfg delete x y $rnd -done } 2>&1 | awk '/real/ {print $2}' - -#new "netconf discard-changes" +new "netconf discard-changes" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "netconf delete $perfreq small config" @@ -212,15 +155,6 @@ done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' #new "netconf discard-changes" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" -# XXX This takes time -# 18.69 without startup feature -# 21.98 with startup -new "restconf delete $perfreq small config" -{ time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - curl $CURLOPTS -X DELETE $RCPROTO://localhost/restconf/data/scaling:x/y=$rnd -done > /dev/null; } 2>&1 | awk '/real/ {print $2}' - # Now do leaf-lists istead of leafs #new "generate leaf-list config" @@ -251,11 +185,6 @@ expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>" '^01' 2>&1 | awk '/real/ {print $2}' -if [ $RC -ne 0 ]; then - new "Kill restconf daemon" - stop_restconf -fi - if [ $BE -eq 0 ]; then exit # BE fi diff --git a/test/test_perf_restconf.sh b/test/test_perf_restconf.sh new file mode 100755 index 00000000..850670c1 --- /dev/null +++ b/test/test_perf_restconf.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# Scaling/ performance tests +# CLI/Netconf/Restconf +# Lists (and leaf-lists) +# Add, get and delete entries + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +# Which format to use as datastore format internally +: ${format:=xml} + +# Number of list/leaf-list entries in file +: ${perfnr:=10000} + +# Number of requests made get/put +: ${perfreq:=10} + +# time function (this is a mess to get right on freebsd/linux) +# -f %e gives elapsed wall clock time but is not available on all systems +# so we use time -p for POSIX compliance and awk to get wall clock time +# Note sometimes time -p is used and sometimes $TIMEFN, cant get it to work same everywhere +: ${TIMEFN:=time -p} # portability: 2>&1 | awk '/real/ {print $2}' + +APPNAME=example + +cfg=$dir/scaling-conf.xml +fyang=$dir/scaling.yang +fconfig=$dir/large.xml +fconfig2=$dir/large2.xml + +cat < $fyang +module scaling{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix ex; + container x { + list y { + key "a"; + leaf a { + type int32; + } + leaf b { + type int32; + } + } + leaf-list c { + type string; + } + } +} +EOF + +cat < $cfg + + $cfg + $dir + /usr/local/share/clixon + $fyang + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/example/$APPNAME.pidfile + false + $dir + false + $format + example + /usr/local/lib/example/cli + /usr/local/lib/example/clispec + 0 + ietf-netconf:startup + +EOF + +new "test params: -f $cfg" +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s +fi + +new "waiting" +wait_backend + +if [ $RC -ne 0 ]; then + new "kill old restconf daemon" + stop_restconf_pre + + new "start restconf daemon" + start_restconf -f $cfg + + new "waiting" + wait_restconf +fi + +new "generate config with $perfnr list entries" +echo -n "" > $fconfig +for (( i=0; i<$perfnr; i++ )); do + echo -n "$i$i" >> $fconfig +done +echo "]]>]]>" >> $fconfig + +# Now take large config file and write it via netconf to candidate +new "test time exists" +expectpart "$(time -p ls)" 0 + +new "netconf write large config" +expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# Here, there are $perfnr entries in candidate +new "netconf write large config again" +expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# Now commit it from candidate to running +new "netconf commit large config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +# RESTCONF get +new "restconf get $perfreq small config 1 key index" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + curl $CURLOPTS -X GET $RCPROTO://localhost/restconf/data/scaling:x/y=$rnd > /dev/null +done } 2>&1 | awk '/real/ {print $2}' + +# RESTCONF put +# Reference: +# i686 format=xml perfnr=10000/100 time: 38/29s 20190425 WITH/OUT startup copying +# i686 format=tree perfnr=10000/100 time: 72/64s 20190425 WITH/OUT startup copying +new "restconf add $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + curl $CURLOPTS -X PUT $RCPROTO://localhost/restconf/data/scaling:x/y=$rnd -d '{"scaling:y":{"a":"'$rnd'","b":"'$rnd'"}}' +done } 2>&1 | awk '/real/ {print $2}' + +new "restconf get large config" +# XXX for some reason cannot expand $TIMEFN next two tests, need keep variable? +$TIMEFN curl $CURLOPTS -X GET $RCPROTO://localhost/restconf/data 2>&1 > /dev/null | awk '/real/ {print $2}' + +# Delete entries (last since entries are removed from db) +# netconf +new "cli delete $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg delete x y $rnd +done } 2>&1 | awk '/real/ {print $2}' + +#new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# XXX This takes time +# 18.69 without startup feature +# 21.98 with startup +new "restconf delete $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + curl $CURLOPTS -X DELETE $RCPROTO://localhost/restconf/data/scaling:x/y=$rnd +done > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +# Now do leaf-lists istead of leafs + +#new "generate leaf-list config" +echo -n "replace" > $fconfig2 +for (( i=0; i<$perfnr; i++ )); do + echo -n "$i" >> $fconfig2 +done +echo "]]>]]>" >> $fconfig2 + +new "netconf replace large list-leaf config" +expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig2" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +new "netconf commit large leaf-list config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +new "netconf add $perfreq small leaf-list config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + echo "$rnd]]>]]>" +done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +new "netconf add small leaf-list config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 'x]]>]]>' "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +new "netconf commit small leaf-list config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" 2>&1 | awk '/real/ {print $2}' + +new "netconf get large leaf-list config" +expecteof "time -p $clixon_netconf -qf $cfg" 0 "]]>]]>" '^01' 2>&1 | awk '/real/ {print $2}' + +if [ $RC -ne 0 ]; then + new "Kill restconf daemon" + stop_restconf +fi + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=$(pgrep -u root -f clixon_backend) +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +rm -rf $dir + +# unset conditional parameters +unset format +unset perfnr +unset perfreq