experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optimized get config xpath of large lists, such as a[x=1000] in a list of 100000s a:s.

Browse source
This commit is contained in:
Olof hagsand 2020-07-23 20:41:32 +02:00
parent 9e2bdb8c8e
commit 83203623cd
8 changed files with 163 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

6
test/test_perf_cli.sh
View file

@ -100,15 +100,13 @@ new "netconf write large config"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Here, there are $perfnr entries in candidate
new "netconf write large config again"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Now commit it from candidate to running
new "netconf commit large config"
expecteof "time -p $clixon_netconf -qf $cfg" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# CLI get (XXX why does this take so much time?)
# See: EXPAND_ONLY_INTERACTIVE in cligen. If set it is acceptable but there are some side-effects
# CLI get. This takes a lot of time because expand_dbvar gets perfnr (eg 100000) entries
# and loops over it.
new "cli get $perfreq small config 1 key index"
{ time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) ))

13
test/test_perf_netconf.sh
View file

@ -101,8 +101,6 @@ new "netconf write large config"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Here, there are $perfnr entries in candidate
new "netconf write large config again"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Now commit it from candidate to running
new "netconf commit large config"
@ -110,19 +108,26 @@ expecteof "time -p $clixon_netconf -qf $cfg" 0 "<rpc><commit/></rpc>]]>]]>" "^<r
# Now commit it again from candidate (validation takes time when
# comparing to existing)
new "netconf commit large config again"
expecteof "time -p $clixon_netconf -qf $cfg" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Having a large db, get and put single entries many times
# Note same entries in the range alreay there, db has same size
# NETCONF get 1 key index
# Note this is done by streaming input into one single netconf client. it is much
# slower if it is started and stopped for each request. (next)
new "netconf get $perfreq small config 1 key index"
{ time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) ))
echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/ex:x/ex:y[ex:a=$rnd]\" xmlns:ex=\"urn:example:clixon\"/></get-config></rpc>]]>]]>"
done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}'
new "netconf get $perfreq small config 1 key index start/stop"
{ time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) ))
echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/ex:x/ex:y[ex:a=$rnd]\" xmlns:ex=\"urn:example:clixon\"/></get-config></rpc>]]>]]>" | $clixon_netconf -qf $cfg > /dev/null;
done
} 2>&1 | awk '/real/ {print $2}'
# NETCONF get 1 key and one non-key index
new "netconf get $perfreq small config 1 key + 1 non-key index"
{ time -p for (( i=0; i<$perfreq; i++ )); do

2
test/test_perf_restconf.sh
View file

@ -112,8 +112,6 @@ new "netconf write large config"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Here, there are $perfnr entries in candidate
new "netconf write large config again"
expecteof_file "time -p $clixon_netconf -qf $cfg" 0 "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 2>&1 | awk '/real/ {print $2}'
# Now commit it from candidate to running
new "netconf commit large config"

12
test/test_rpc.sh
View file

@ -142,24 +142,24 @@ new "netconf edit-config ok"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target><candidate/></target><config/></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
if ! $YANG_UNKNOWN_ANYDATA ; then
new "netconf edit-config extra arg should fail"
new "netconf edit-config extra arg: should fail"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target><candidate/></target><extra/><config/></edit-config></rpc>]]>]]>' "^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown-element</error-tag><error-info><bad-element>extra</bad-element></error-info><error-severity>error</error-severity><error-message>Failed to find YANG spec of XML node: extra with parent: edit-config in namespace: urn:ietf:params:xml:ns:netconf:base:1.0</error-message></rpc-error></rpc-reply>]]>]]>$"
fi
new "netconf edit-config empty target should fail"
new "netconf edit-config empty target: should fail"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target/><config/></edit-config></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>data-missing</error-tag><error-app-tag>missing-choice</error-app-tag><error-info><missing-choice>config-target</missing-choice></error-info><error-severity>error</error-severity></rpc-error></rpc-reply>]]>]]>$'
new "netconf edit-config missing target should fail"
new "netconf edit-config missing target: should fail"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><config/></edit-config></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>target</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable</error-message></rpc-error></rpc-reply>]]>]]>$'
new "netconf edit-config missing config should fail"
new "netconf edit-config missing config: should fail"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target><candidate/></target></edit-config></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>data-missing</error-tag><error-app-tag>missing-choice</error-app-tag><error-info><missing-choice>edit-content</missing-choice></error-info><error-severity>error</error-severity></rpc-error></rpc-reply>]]>]]>$'
# Negative errors (namespace/module missing)
new "netconf wrong rpc namespace (should fail)"
new "netconf wrong rpc namespace: should fail"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc xmlns="urn:example:xxx"><get/></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown-element</error-tag><error-info><bad-element>get</bad-element></error-info><error-severity>error</error-severity></rpc-error></rpc-reply>]]>]]>$'
new "restconf wrong rpc (should fail"
new "restconf wrong rpc: should fail"
expectpart "$(curl $CURLOPTS -X POST -H "Content-Type: application/yang-data+json" u $RCPROTO://localhost/restconf/operations/clixon-foo:get)" 0 'HTTP/1.1 412 Precondition Failed' '{"ietf-restconf:errors":{"error":{"error-type":"protocol","error-tag":"operation-failed","error-severity":"error","error-message":"yang module not found"}}}'
if [ $RC -ne 0 ]; then

6
test/test_upgrade_module.sh
View file

@ -299,16 +299,16 @@ EOF
# There is some issue with having different payloads in the config file
# That is why there are tests with different payloads
new "b payload only---------"
new "b payload only"
testall '<dummy xmlns="urn:example:b"/>' '<dummy xmlns="urn:example:b"/>'
new "b payload and interfaces payload---------"
testall '<dummy xmlns="urn:example:b"/><dummy xmlns="urn:example:interfaces"/>' '<dummy xmlns="urn:example:b"/>'
new "a payload only---------"
new "a payload only"
testall '<dummy xmlns="urn:example:interfaces"/>' ''
new "empty payload---------"
new "empty payload"
testall '' ''
rm -rf $dir