experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* Optimizations

Browse source 
* Reduced memory for attribute and body objects, see `XML_NEW_DIFFERENTIATE` compile-time option.
  * Optimized cbuf handling in parsing and xml2cbuf functions.
  * Optimized xml scanner to read strings rather than single chars
  * Optimized xml_merge for the case of disjunct trees.
This commit is contained in:
Olof hagsand 2020-04-28 22:31:58 +02:00
parent 9a8c6cf3e6
commit 94cf4a88b3
24 changed files with 477 additions and 257 deletions
Download patch file Download diff file Expand all files Collapse all files

4
test/test_cli.sh
View file

@ -25,9 +25,9 @@ cat <<EOF > $cfg
<CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
<CLICON_YANG_MODULE_MAIN>clixon-example</CLICON_YANG_MODULE_MAIN>
<CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>

18
test/test_perf_mem.sh
View file

@ -1,11 +1,14 @@
#!/usr/bin/env bash
# Backend Memory tests, footprint using the clixon-conf state statistics
# Create a large datastore, load it and measure
# Baseline: (thinkpad laptop) running db:
# 100K objects: 500K mem: 74M
# 1M objects: 5M mem: 747M
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
# ENable this for massif memory profiling
# Enable this for massif memory profiling
#clixon_backend="valgrind --tool=massif clixon_backend"
clixon_util_xpath=clixon_util_xpath
@ -92,12 +95,14 @@ testrun(){
pid=$(cat $pidfile)
new "netconf get state"
res=$(echo "<rpc><get><filter type=\"xpath\" select=\"/cc:clixon-stats\" xmlns:cc=\"http://clicon.org/config\"/></get></rpc>]]>]]>" | $clixon_netconf -qf $cfg)
new "netconf get stats"
res=$(echo '<rpc xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><stats xmlns="http://clicon.org/lib"/></rpc>]]>]]>' | $clixon_netconf -qf $cfg)
objects=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/global/xmlnr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}')
echo "Total"
echo -n " objects: "
echo $res | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/global/xmlnr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
echo " objects: $objects"
#
if [ -f /proc/$pid/statm ]; then # This ony works on Linux
# cat /proc/$pid/statm
echo -n " mem: "
@ -105,7 +110,7 @@ testrun(){
fi
for db in running candidate startup; do
echo "$db"
resdb=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/datastore[name=\"$db\"]")
resdb=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastore[name=\"$db\"]")
resdb=${resdb#"nodeset:0:"}
echo -n " objects: "
echo $resdb | $clixon_util_xpath -p "datastore/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
@ -134,4 +139,3 @@ rm -rf $dir
# unset conditional parameters
unset perfnr

46
test/test_perf_state.sh
View file

@ -3,7 +3,7 @@
# Config + state data, only get
# Restconf/Netconf/CLI
# Use mixed interfaces config+state
# ALso added two layers a/b to get extra depth (som caching can break)
# Also added two layers a/b to get extra depth (some caching can break)
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@ -50,35 +50,35 @@ EOF
cat <<EOF > $fyang
module $APPNAME{
yang-version 1.1;
prefix ex;
namespace "urn:example:clixon";
container interfaces {
yang-version 1.1;
prefix ex;
namespace "urn:example:clixon";
container interfaces {
list a{
key "name";
leaf name {
type string;
}
container b{
list interface {
key "name";
leaf name {
type string;
}
leaf type {
type string;
}
leaf enabled {
type boolean;
default true;
}
leaf status {
type string;
config false;
container b{
list interface {
key "name";
leaf name {
type string;
}
leaf type {
type string;
}
leaf enabled {
type boolean;
default true;
}
leaf status {
type string;
config false;
}
}
}
}
}
}
}
}
EOF

87
test/test_perf_state_only.sh
View file

@ -2,7 +2,7 @@
# Scaling/ performance tests
# State data only, in particular non-config lists (ie not state leafs on a config list)
# Restconf/Netconf/CLI
# ALso added two layers a/b to get extra depth (som caching can break)
# Also added two layers a/b to get extra depth (som caching can break)
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@ -42,42 +42,41 @@ cat <<EOF > $cfg
<CLICON_CLI_MODE>example</CLICON_CLI_MODE>
<CLICON_CLI_DIR>/usr/local/lib/example/cli</CLICON_CLI_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
<CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
<CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
</clixon-config>
EOF
cat <<EOF > $fyang
module $APPNAME{
yang-version 1.1;
prefix ex;
namespace "urn:example:clixon";
container interfaces {
yang-version 1.1;
prefix ex;
namespace "urn:example:clixon";
container interfaces {
config false;
list a{
key "name";
leaf name {
type string;
}
container b{
list interface {
key "name";
leaf name {
type string;
}
leaf type {
type string;
}
leaf enabled {
type boolean;
default true;
}
leaf status {
type string;
container b{
list interface {
key "name";
leaf name {
type string;
}
leaf type {
type string;
}
leaf enabled {
type boolean;
default true;
}
leaf status {
type string;
}
}
}
}
}
}
}
}
EOF
@ -113,22 +112,9 @@ start_restconf -f $cfg
new "waiting"
wait_restconf
if false; then
new "generate 'large' config with $perfnr list entries"
echo -n "<rpc><edit-config><target><candidate/></target><config><interfaces xmlns=\"urn:example:clixon\"><a><name>foo</name><b>" > $fconfig
for (( i=0; i<$perfnr; i++ )); do
echo -n "<interface><name>e$i</name><type>ex:eth</type></interface>" >> $fconfig
done
echo "</b></a></interfaces></config></edit-config></rpc>]]>]]>" >> $fconfig
# Now take large config file and write it via netconf to candidate
new "netconf write large config"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Now commit it from candidate to running
new "netconf commit large config"
expecteof "time -p $clixon_netconf -qf $cfg" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
fi
new "cli get large config"
echo "$clixon_cli -1f $cfg show state xml"
$TIMEFN $clixon_cli -1f $cfg show state xml 2>&1 | awk '/real/ {print $2}'
# START actual tests
# Having a large db, get single entries many times
@ -176,6 +162,7 @@ new "cli get $perfreq single reqs"
$clixon_cli -1 -f $cfg show state xml interfaces a b interface e$rnd > /dev/null
done } 2>&1 | awk '/real/ {print $2}'
fi
# Get config in one large get
new "netconf get large config"
{ time -p echo "<rpc><get> <filter type=\"xpath\" select=\"/ex:interfaces/ex:a[name='foo']/ex:b\" xmlns:ex=\"urn:example:clixon\"/></get></rpc>]]>]]>" | $clixon_netconf -qf $cfg > /tmp/netconf; } 2>&1 | awk '/real/ {print $2}'
@ -184,24 +171,22 @@ new "restconf get large config"
$TIMEFN curl -sG http://localhost/restconf/data/example:interfaces/a=foo/b 2>&1 | awk '/real/ {print $2}'
new "cli get large config"
$TIMEFN $clixon_cli -1f $cfg show state xml interfaces a foo b 2>&1 | awk '/real/ {print $2}'
$TIMEFN $clixon_cli -1f $cfg show state xml 2>&1 | awk '/real/ {print $2}'
new "Kill restconf daemon"
stop_restconf
if [ $BE -eq 0 ]; then
exit # BE
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
rm -rf $dir
# unset conditional parameters

2
test/test_perf_xml.sh
View file

@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Test: XML performance test
# Test: XML performance test (CDATA test only)
# See https://github.com/clicon/clixon/issues/96
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi

10
test/test_restconf.sh
View file

@ -193,16 +193,16 @@ new "restconf Add interfaces subtree eth/0/0 using POST"
expectpart "$(curl -si -X POST http://localhost/restconf/data/ietf-interfaces:interfaces -H "Content-Type: application/yang-data+json" -d '{"ietf-interfaces:interface":{"name":"eth/0/0","type":"clixon-example:eth","enabled":true}}')" 0 "HTTP/1.1 201 Created"
new "restconf Check eth/0/0 added config"
expectpart "$(curl -s -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces":{"interface":\[{"name":"eth/0/0","type":"clixon-example:eth","enabled":true,"oper-status":"up","clixon-example:my-status":{"int":42,"str":"foo"}}\]}}'
expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 'HTTP/1.1 200 OK' '{"ietf-interfaces:interfaces":{"interface":\[{"name":"eth/0/0","type":"clixon-example:eth","enabled":true,"oper-status":"up","clixon-example:my-status":{"int":42,"str":"foo"}}\]}}'
new "restconf Check eth/0/0 GET augmented state level 1"
expectpart "$(curl -s -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=eth%2f0%2f0)" 0 '{"ietf-interfaces:interface":\[{"name":"eth/0/0","type":"clixon-example:eth","enabled":true,"oper-status":"up","clixon-example:my-status":{"int":42,"str":"foo"}}\]}'
expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=eth%2f0%2f0)" 0 'HTTP/1.1 200 OK' '{"ietf-interfaces:interface":\[{"name":"eth/0/0","type":"clixon-example:eth","enabled":true,"oper-status":"up","clixon-example:my-status":{"int":42,"str":"foo"}}\]}'
new "restconf Check eth/0/0 GET augmented state level 2"
expectpart "$(curl -s -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=eth%2f0%2f0/clixon-example:my-status)" 0 '{"clixon-example:my-status":{"int":42,"str":"foo"}}'
expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=eth%2f0%2f0/clixon-example:my-status)" 0 'HTTP/1.1 200 OK' '{"clixon-example:my-status":{"int":42,"str":"foo"}}'
new "restconf Check eth/0/0 added state"
expectpart "$(curl -s -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/clixon-example:state)" 0 '{"clixon-example:state":{"op":\["41","42","43"\]}}'
new "restconf Check eth/0/0 added state XXXXXXX"
expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/clixon-example:state)" 0 'HTTP/1.1 200 OK' '{"clixon-example:state":{"op":\["41","42","43"\]}}'
new "restconf Re-post eth/0/0 which should generate error"
expectpart "$(curl -s -X POST -H "Content-Type: application/yang-data+json" -d '{"ietf-interfaces:interface":{"name":"eth/0/0","type":"clixon-example:eth","enabled":true}}' http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"data-exists","error-severity":"error","error-message":"Data already exists; cannot create new resource"}}} '

7
test/test_xml_trees.sh
View file

@ -89,12 +89,11 @@ testrun insert "$x0a<a><x>1</x></a>$x0b" "$x0a<d>42</d>$x0b" c 0 '<c xmlns="urn:
new "parent 1st element"
testrun parent "$x0a$x0b" "<a><x>1</x></a>" $p 0 '<c xmlns="urn:example:example"><a><x>1</x></a></c>'
new "patrse parent element"
testrun parent "$x0a<a><x>2</x></a>$x0b" '<a><x>1</x></a>' $p 0 '<c xmlns="urn:example:example"><a><x>1</x></a><a><x>2</x></a></c>'
new "parse parent element"
testrun parent "$x0a<a><x>2</x></a>$x0b" '<a><x>1</x></a>' $p 0 '<c xmlns="urn:example:example"><a><x>2</x></a><a><x>1</x></a></c>'
new "parse parent container"
testrun parent "$x0a<a><x>1</x></a>$x0b" '<d>42</d>' c 0 '<c xmlns="urn:example:example"><d>42</d><a><x>1</x></a></c>'
testrun parent "$x0a<a><x>1</x></a>$x0b" '<d>42</d>' c 0 '<c xmlns="urn:example:example"><a><x>1</x></a><d>42</d></c>'
# -------- merge