experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

- Rewrite of netconf get/get-config code

Browse source 
- Unified get and get-config code to single function get_common
  - Integrated list-pagination code
  - Moved get code to new files backend_get.[ch]
This commit is contained in:
Olof hagsand 2021-08-22 12:41:45 +02:00
parent b03cf426a4
commit c9843b34a6
11 changed files with 1382 additions and 1246 deletions
Download patch file Download diff file Expand all files Collapse all files

1
apps/backend/Makefile.in
View file

@ -85,6 +85,7 @@ APPL = clixon_backend
APPSRC = backend_main.c
APPSRC += backend_socket.c
APPSRC += backend_client.c
APPSRC += backend_get.c
APPSRC += backend_plugin_restconf.c # Pseudo plugin for restconf daemon
APPSRC += backend_startup.c
APPOBJ = $(APPSRC:.c=.o)

1161
apps/backend/backend_client.c
View file

File diff suppressed because it is too large Load diff

1108
apps/backend/backend_get.c Normal file
View file

File diff suppressed because it is too large Load diff

47
apps/backend/backend_get.h Normal file
View file

@ -0,0 +1,47 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
Copyright (C) 2017-2019 Olof Hagsand
Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC (Netgate)
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*/
#ifndef _BACKEND_GET_H_
#define _BACKEND_GET_H_
/*
* Prototypes
*/
int from_client_get_config(clicon_handle h, cxobj *xe, cbuf *cbret, void *arg, void *regarg);
int from_client_get(clicon_handle h, cxobj *xe, cbuf *cbret, void *arg, void *regarg);
int from_client_get_pageable_list(clicon_handle h, cxobj *xe, cbuf *cbret, void *arg, void *regarg); /* XXX */
#endif /* _BACKEND_GET_H_ */

93
lib/src/clixon_xml_map.c
View file

@ -688,6 +688,7 @@ xml_diff(yang_stmt *yspec,
}
/*! Prune everything that does not pass test or have at least a child* does not
*
* @param[in] xt XML tree with some node marked
* @param[in] flag Which flag to test for
* @param[in] test 1: test that flag is set, 0: test that flag is not set
@ -701,7 +702,6 @@ xml_diff(yang_stmt *yspec,
* @note This function seems a little too complex semantics
* @see xml_tree_prune_flagged for a simpler variant
*/
#if 1
int
xml_tree_prune_flagged_sub(cxobj *xt,
int flag,
@ -773,97 +773,6 @@ xml_tree_prune_flagged_sub(cxobj *xt,
*upmark = mark;
return retval;
}
#else
/* This is optimized in the sense that xml_purge is replaced with xml_child_rm but it leaks memory,
* in poarticualr attributes and namespace caches
*/
int
xml_tree_prune_flagged_sub(cxobj *xt,
int flag,
int test,
int *upmark)
{
int retval = -1;
int submark;
int mark;
cxobj *x;
cxobj *xprev;
int iskey;
int anykey=0;
yang_stmt *yt;
int i;
mark = 0;
yt = xml_spec(xt); /* xan be null */
x = NULL;
xprev = x = NULL;
i = 0;
while ((x = xml_child_each(xt, x, -1)) != NULL) {
i++;
if (xml_type(x) != CX_ELMNT){
xprev = x;
continue;
}
if (xml_flag(x, flag) == test?flag:0){
/* Pass test */
mark++;
xprev = x;
continue; /* mark and stop here */
}
/* If it is key dont remove it yet (see second round) */
if (yt){
if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
goto done;
if (iskey){
anykey++;
xprev = x; /* skip if this is key */
continue;
}
}
if (xml_tree_prune_flagged_sub(x, flag, test, &submark) < 0)
goto done;
/* if xt is list and submark anywhere, then key subs are also marked
*/
if (submark)
mark++;
else{ /* Safe with xml_child_each if last */
if (xml_child_rm(xt, i-1) < 0)
goto done;
i--;
x = xprev;
}
xprev = x;
}
/* Second round: if any keys were found, and no marks detected, purge now */
if (anykey && !mark){
x = NULL;
xprev = x = NULL;
i = 0;
while ((x = xml_child_each(xt, x, -1)) != NULL) {
i++;
if (xml_type(x) != CX_ELMNT){
xprev = x;
continue;
}
/* If it is key remove it here */
if (yt){
if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
goto done;
if (xml_child_rm(xt, i-1) < 0)
goto done;
i--;
x = xprev;
}
xprev = x;
}
}
retval = 0;
done:
if (upmark)
*upmark = mark;
return retval;
}
#endif
/*! Prune everything that passes test
* @param[in] xt XML tree with some node marked

27
lib/src/clixon_xpath_yang.c
View file

@ -150,6 +150,7 @@ xp_yang_eval_step(xp_yang_ctx *xy0,
char *prefix;
yang_stmt *ys;
xp_yang_ctx *xy = NULL;
yang_stmt *ys1 = NULL;
/* Create new xy */
if ((xy = xy_dup(xy0)) == NULL)
@ -164,9 +165,12 @@ xp_yang_eval_step(xp_yang_ctx *xy0,
switch (nodetest->xs_type){
case XP_NODE:
if ((prefix = nodetest->xs_s0) != NULL){
if (yang_keyword_get(ys) == Y_MODULE){ /* This means top */
yang_stmt *ys1 = NULL;
/* XXX: Kludge with prefixes */
if (yang_keyword_get(ys) == Y_SPEC){ /* This means top */
if ((ys1 = yang_find_module_by_prefix_yspec(ys, prefix)) != NULL)
ys = ys1;
}
else if (yang_keyword_get(ys) == Y_MODULE){ /* This means top */
if ((ys1 = yang_find_module_by_prefix(ys, prefix)) == NULL)
ys1 = yang_find_module_by_prefix_yspec(ys_spec(ys), prefix);
if (ys1 != NULL)
@ -323,8 +327,14 @@ xp_yang_eval(xp_yang_ctx *xy,
}
}
break;
case XP_PRIME_STR:
if ((*xyr = xy_dup(xy)) == NULL)
goto done;
goto ok;
break;
case XP_ABSPATH:
/* Set context node to top node, and nodeset to that node only */
if (yang_keyword_get(xy->xy_node) != Y_SPEC)
xy->xy_node = ys_module(xy->xy_node);
break;
case XP_PRED:
@ -424,10 +434,19 @@ xp_yang_eval(xp_yang_ctx *xy,
* key nodes for list entries. Each predicate consists of exactly one
* equality test per key, and multiple adjacent predicates MAY be
* present if a list has multiple keys.
* @param[in] ys YANG referring leaf node
* @param[in] path_arg Leafref path-arg
* @param[in] ys YANG referring node
* @param[in] path_arg path-arg
* @param[out] yref YANG referred node
* @note this function uses XPATH parser, which is (much too) general
* @code
* yang_stmt *ys; // source / referring node
* yang_stmt *yref = NULL; // target / referred node
* char *path_arg="../config/name";
*
* if (yang_path_arg(ys, path_arg, &yref) < 0)
* err;
* @endcode
* @see rfc7950 Sec 9.9.2
* @see rfc7950 Sec 14 (leafref path)
*/

1
test/test_augment_state.sh
View file

@ -189,6 +189,7 @@ cat <<EOF > $fstate
</global-state>
EOF
# Note Expect gbds(default) + gbos(optional), the latter given by file above
EXPSTATE=$(cat <<EOF
<global-state xmlns="urn:example:lib"><gbds>gbds</gbds><gbos>gbos</gbos><aug:gads xmlns:aug="urn:example:augment">gads</aug:gads><aug:gaos xmlns:aug="urn:example:augment">gaos</aug:gaos></global-state>
EOF

146
test/test_leafref_state.sh
View file

@ -10,7 +10,10 @@
# with different paths.
# Using the -sS <file> state capability of the main example, that is why CLICON_BACKEND_DIR is
# /usr/local/lib/$APPNAME/backend so that the main backend plugins is included.
# These tests require VALIDATE_STATE_XML to be set
# Note: Three runs:
# 1. with state data validation and with require-instance (Invalid)
# 2. with state data validation and without require-instance (OK)
# 3. without state data validation and with require-instance (Wrong state data no detected)
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@ -20,6 +23,7 @@ APPNAME=example
cfg=$dir/conf_yang.xml
fstate=$dir/state.xml
fyang=$dir/leafref.yang
fyangno=$dir/leafrefno.yang # No require-instance
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
@ -66,6 +70,32 @@ module leafref{
}
EOF
# No require-instance in leafref
cat <<EOF > $fyangno
module leafref{
yang-version 1.1;
namespace "urn:example:example";
prefix ex;
list sender-config{
description "Main config of senders";
key name;
leaf name{
type string;
}
}
list sender-state{
description "State referencing configured senders";
config false;
key ref;
leaf ref{
type leafref {
path "/ex:sender-config/ex:name";
}
}
}
}
EOF
# This is state data written to file that backend reads from (on request)
cat <<EOF > $fstate
<sender-state xmlns="urn:example:example">
@ -73,6 +103,7 @@ cat <<EOF > $fstate
</sender-state>
EOF
# First run: With validation of state callbacks
new "test params: -f $cfg -- -sS $fstate"
if [ $BE -ne 0 ]; then
@ -167,7 +198,7 @@ expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-confi
new "netconf commit"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
# Leafref wrong
# Leafref wrong internal: state references x but config contains only y
new "netconf get / config+state should fail"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get content=\"all\"><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>operation-failed</error-tag><error-info><bad-element>x</bad-element></error-info><error-severity>error</error-severity><error-message>Leafref validation failed: No leaf x matching path /ex:sender-config/ex:name in leafref.yang:[0-9]*. Internal error, state callback returned invalid XML</error-message></rpc-error></rpc-reply>]]>]]>$"
@ -188,6 +219,117 @@ if [ $BE -ne 0 ]; then
stop_backend -f $cfg
fi
# Second run: Validation and no require-instance
new "Second run: -f $cfg -o CLICON_YANG_MAIN_FILE=$fyangno -- -sS $fstate"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s init -f $cfg -o CLICON_VALIDATE_STATE_XML=false -- -sS $fstate"
start_backend -s init -f $cfg -o CLICON_YANG_MAIN_FILE=$fyangno -- -sS $fstate
fi
new "wait backend"
wait_backend
# Add y
XML=$(cat <<EOF
<sender-config xmlns="urn:example:example">
<name>y</name>
</sender-config>
EOF
)
# Reference (non-existing) x
cat <<EOF > $fstate
<sender-state xmlns="urn:example:example">
<ref>x</ref>
</sender-state>
EOF
new "leafref config sender x"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><target><candidate/></target><config>$XML</config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
new "netconf commit"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
# Leafref wrong internal: state references x but config contains only y
new "netconf get / config+state wrong state xml but no validation"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get content=\"all\"><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><sender-config xmlns=\"urn:example:example\"><name>y</name></sender-config><sender-state xmlns=\"urn:example:example\"><ref>x</ref></sender-state></data></rpc-reply>]]>]]>$"
new "netconf get / state-only wrong state xml but no validation"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><data><sender-state xmlns=\"urn:example:example\"><ref>x</ref></sender-state></data></rpc-reply>]]>]]>$"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
# Third run: No validation of state callbacks
new "Third run: -f $cfg -o CLICON_VALIDATE_STATE_XML=true -- -sS $fstate"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s init -f $cfg -o CLICON_VALIDATE_STATE_XML=false -- -sS $fstate"
start_backend -s init -f $cfg -o CLICON_VALIDATE_STATE_XML=false -- -sS $fstate
fi
new "wait backend"
wait_backend
# Add y
XML=$(cat <<EOF
<sender-config xmlns="urn:example:example">
<name>y</name>
</sender-config>
EOF
)
# Reference (non-existing) x
cat <<EOF > $fstate
<sender-state xmlns="urn:example:example">
<ref>x</ref>
</sender-state>
EOF
new "leafref config sender x"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><edit-config><target><candidate/></target><config>$XML</config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
new "netconf commit"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><commit/></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>$"
# Leafref wrong internal: state references x but config contains only y
new "netconf get / config+state wrong state xml but no validation"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get content=\"all\"><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><sender-config xmlns=\"urn:example:example\"><name>y</name></sender-config><sender-state xmlns=\"urn:example:example\"><ref>x</ref></sender-state></data></rpc-reply>]]>]]>$"
new "netconf get / state-only wrong state xml but no validation"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><data><sender-state xmlns=\"urn:example:example\"><ref>x</ref></sender-state></data></rpc-reply>]]>]]>$"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
rm -rf $dir
new "endtest"

10
test/test_pagination.sh
View file

@ -20,6 +20,9 @@ fstate=$dir/mystate.xml
# Define default restconfig config: RESTCONFIG
RESTCONFIG=$(restconf_config none false)
# Validate internal state xml
: ${validatexml:=false}
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
@ -38,7 +41,7 @@ cat <<EOF > $cfg
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
<CLICON_VALIDATE_STATE_XML>false</CLICON_VALIDATE_STATE_XML>
<CLICON_VALIDATE_STATE_XML>$validatexml</CLICON_VALIDATE_STATE_XML>
$RESTCONFIG
</clixon-config>
EOF
@ -269,10 +272,10 @@ function testlimit()
# "clixon get"
new "clixon limit=$limit NETCONF get-config"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get-config></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get-config></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
new "clixon limit=$limit NETCONF get"
# expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
# "old: ietf"
new "ietf limit=$limit NETCONF"
@ -346,6 +349,7 @@ if [ $BE -ne 0 ]; then
fi
unset RESTCONFIG
unset validatexml
rm -rf $dir

2
test/test_upgrade_failsafe.sh
View file

@ -361,7 +361,7 @@ if [ $valgrindtest -ne 2 ]; then
new "8. Load non-compat startup. Syntax fail, enter failsafe, startup invalid"
(cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
(cd $dir; cp compat-err.xml startup_db)
runtest true startup '<data><a1 xmlns="urn:example:a">always work</a1></data>' '<rpc-error><error-type>application</error-type><error-tag>operation-failed</error-tag><error-severity>error</error-severity><error-message>read registry</error-message></rpc-error>'
runtest true startup '<data><a1 xmlns="urn:example:a">always work</a1></data>' '<rpc-error><error-type>application</error-type><error-tag>operation-failed</error-tag><error-severity>error</error-severity><error-message>Get startup datastore: xml_parse: line 14: syntax error: at or before: &lt;</error-message></rpc-error>'
fi # valgrindtest
rm -rf $dir

14
yang/clixon/clixon-netconf-list-pagination@2021-08-27.yang
View file

@ -81,6 +81,20 @@ module clixon-netconf-list-pagination {
elements.";
}
grouping pageing-parameters {
leaf list-pagination {
type boolean;
default false;
description
"NETCONF get / get-config needs some way to know that this is a pagination
request, in which case the target is a list/leaf-list and the elements below
(limit/offset/...) are valid.
RESTCONF list pagination has a specific media-type for this purpose.
This is an experimental proposal to make this property explicit.
Possibly there is a better way (annotation?) to signal that this is in fact a
list pagination request.
It is also possible to determine this using heurestics (ie a 'limit' property exixts),
but it seems not 100% deterministic.";
}
leaf limit {
type union {
type uint32;