List pagination, uniform config/state handling with new attributes

New `clixon-lib@2024-04-01.yang` revision and list_pagination_partial_state extension
This commit is contained in:
Olof hagsand 2024-07-17 14:39:48 +02:00
parent 05c881dc39
commit 07a1fa164f
14 changed files with 357 additions and 135 deletions

View file

@ -17,6 +17,16 @@ Expected: October 2024
### Features
* List pagination: Added where, sort-by and direction parameter for configured data
* New `clixon-lib@2024-04-01.yang` revision
- Added: list_pagination_partial_state
### API changes on existing protocol/config features
Users may have to change how they access the system
* List pagination of large lists
* For backward-compatibility, mark the list with extension cl:list_pagination_partial_state extension
* New default is to use regular state read mechanism, which could have poorer performance but more functionality
### C/CLI-API changes on existing features

View file

@ -495,6 +495,87 @@ list_pagination_hdr(clixon_handle h,
return retval;
}
/*! Special handling of state data for partial reading
*
* Only if extension list-pagination-partial-state is enabled on the list
* @param[in] h Clixon handle
* @param[in] ce Client entry, for locking
* @param[in] yspec (Top-level) yang spec
* @param[in] xpath XPath point to object to get
* @param[in] offset Start of pagination interval
* @param[in] limit Number of elements (limit)
* @param[out] xret Returned xml state tree
* @param[out] cbret Return xml tree, eg <rpc-reply>..., <rpc-error..
* @retval 1 OK
* @retval 0 Fail, cbret contains error message
* @retval -1 Error
*/
static int
get_pagination_partial(clixon_handle h,
struct client_entry *ce,
yang_stmt *yspec,
char *xpath,
uint32_t offset,
uint32_t limit,
cxobj *xret,
cbuf *cbret)
{
int retval = -1;
int locked;
cbuf *cberr = NULL;
uint32_t iddb; /* DBs lock, if any */
cxobj *xerr = NULL;
int ret;
if ((iddb = xmldb_islocked(h, "running")) != 0 &&
iddb == ce->ce_id)
locked = 1;
else
locked = 0;
if ((ret = clixon_pagination_cb_call(h, xpath, locked,
offset, limit,
xret)) < 0)
goto done;
if (ret == 0){
if ((cberr = cbuf_new()) == NULL){
clixon_err(OE_UNIX, errno, "cbuf_new");
goto done;
}
/* error reason should be in clixon_err_reason */
cprintf(cberr, "Internal error, pagination state callback invalid return : %s",
clixon_err_reason());
if (netconf_operation_failed_xml(&xerr, "application", cbuf_get(cberr)) < 0)
goto done;
if (clixon_xml2cbuf(cbret, xerr, 0, 0, NULL, -1, 0) < 0)
goto done;
goto fail;
}
/* System makes the binding */
if ((ret = xml_bind_yang(h, xret, YB_MODULE, yspec, &xerr)) < 0)
goto done;
if (ret == 0){
clixon_debug_xml(CLIXON_DBG_BACKEND, xret, "Yang bind pagination state");
if (clixon_netconf_internal_error(xerr,
". Internal error, state callback returned invalid XML",
NULL) < 0)
goto done;
if (clixon_xml2cbuf(cbret, xerr, 0, 0, NULL, -1, 0) < 0)
goto done;
goto fail;
}
retval = 1;
done:
if (cberr)
cbuf_free(cberr);
if (xerr)
xml_free(xerr);
return retval;
fail:
retval = 0;
goto done;
}
/*! Specialized get for list-pagination
*
* It is specialized enough to have its own function. Specifically, extra attributes as well
@ -543,14 +624,11 @@ get_list_pagination(clixon_handle h,
uint32_t offset = 0;
uint32_t limit = 0;
uint32_t upper;
int list_config;
int partial_pagination_cb = 0; /* use state partial reads callback */
yang_stmt *ylist;
cxobj *xerr = NULL;
cbuf *cbmsg = NULL; /* For error msg */
cxobj *xret = NULL;
uint32_t iddb; /* DBs lock, if any */
int locked;
cbuf *cberr = NULL;
cxobj **xvec = NULL;
size_t xlen;
cxobj *x;
@ -559,6 +637,7 @@ get_list_pagination(clixon_handle h,
char *sort_by = NULL;
char *direction = NULL;
char *where = NULL;
int extflag = 0;
int i;
int j;
int ret;
@ -588,7 +667,7 @@ get_list_pagination(clixon_handle h,
goto ok;
}
/* Sanity checks on state/config */
if ((list_config = yang_config_ancestor(ylist)) != 0){ /* config list */
if (yang_config_ancestor(ylist) != 0){ /* config list */
if (content == CONTENT_NONCONFIG){
if (netconf_invalid_value(cbret, "application", "list-pagination targets a config list but content request is nonconfig") < 0)
goto done;
@ -601,7 +680,13 @@ get_list_pagination(clixon_handle h,
goto done;
goto ok;
}
if (yang_extension_value(ylist, "list-pagination-partial-state", CLIXON_LIB_NS, &extflag, NULL) < 0)
goto done;
if (extflag){ /* pagination_cb / partial state API */
partial_pagination_cb = 1;
}
}
/* first processes the "where" parameter (see Section 3.1.1) */
if ((x = xml_find_type(xe, NULL, "where", CX_ELMNT)) != NULL &&
(where = xml_body(x)) != NULL){
@ -664,7 +749,7 @@ get_list_pagination(clixon_handle h,
break;
case CONTENT_ALL: /* both config and state */
case CONTENT_NONCONFIG: /* state data only */
if (list_config == 0) /* Special handling */
if (partial_pagination_cb) /* Partial reads, special handling */
break;
if ((ret = get_statedata(h, xpath?xpath:"/", nsc, &xret)) < 0)
goto done;
@ -681,7 +766,13 @@ get_list_pagination(clixon_handle h,
if (xml_default_recurse(xret, 1, 0) < 0)
goto done;
}
if (list_config){
if (partial_pagination_cb) {
if ((ret = get_pagination_partial(h, ce, yspec, xpath, offset, limit, xret, cbret)) < 0)
goto done;
if (ret == 0)
goto ok;
}
else {
/* first processes the "where" parameter (see Section 3.1.1) */
if (where){
if (xpath_vec(xret, nsc, "%s[%s]", &xvec, &xlen, xpath?xpath:"/", where) < 0)
@ -762,45 +853,6 @@ get_list_pagination(clixon_handle h,
}
#endif
}
else {/* Check if running locked (by this session) */
if ((iddb = xmldb_islocked(h, "running")) != 0 &&
iddb == ce->ce_id)
locked = 1;
else
locked = 0;
if ((ret = clixon_pagination_cb_call(h, xpath, locked,
offset, limit,
xret)) < 0)
goto done;
if (ret == 0){
if ((cberr = cbuf_new()) == NULL){
clixon_err(OE_UNIX, errno, "cbuf_new");
goto done;
}
/* error reason should be in clixon_err_reason */
cprintf(cberr, "Internal error, pagination state callback invalid return : %s",
clixon_err_reason());
if (netconf_operation_failed_xml(&xerr, "application", cbuf_get(cberr)) < 0)
goto done;
if (clixon_xml2cbuf(cbret, xerr, 0, 0, NULL, -1, 0) < 0)
goto done;
goto ok;
}
/* System makes the binding */
if ((ret = xml_bind_yang(h, xret, YB_MODULE, yspec, &xerr)) < 0)
goto done;
if (ret == 0){
clixon_debug_xml(CLIXON_DBG_BACKEND, xret, "Yang bind pagination state");
if (clixon_netconf_internal_error(xerr,
". Internal error, state callback returned invalid XML",
NULL) < 0)
goto done;
if (clixon_xml2cbuf(cbret, xerr, 0, 0, NULL, -1, 0) < 0)
goto done;
goto ok;
}
}
if (xpath_vec(xret, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
goto done;
/* Help function to filter out anything that is outside of xpath */
@ -837,8 +889,6 @@ get_list_pagination(clixon_handle h,
cbuf_free(cbmsg);
if (xerr)
xml_free(xerr);
if (cberr)
cbuf_free(cberr);
if (xret)
xml_free(xret);
return retval;

View file

@ -485,10 +485,15 @@ clixon_plugin_lockdb_all(clixon_handle h,
return retval;
}
/*! Traverse state data callbacks
/*! Traverse state data callbacks for partial pagination state callbacks
*
* Only if list-pagination-partial-state extension is set
* @param[in] h Clixon handle
* @param[in] xpath Registered XPath using canonical prefixes
* @param[in] locked Running datastore is locked by this caller
* @param[in] offset Start of pagination interval
* @param[in] limit Number of elements (limit)
* @param[out] xstate Returned xml state tree
* @retval 1 OK
* @retval -1 Error
*/

View file

@ -97,7 +97,7 @@ show("Show a particular state of the system"){
xml("Show comparison in xml"), compare_dbs("running", "candidate", "xml");
text("Show comparison in text"), compare_dbs("running", "candidate", "text");
}
pagination("Show list pagination") xpath("Show configuration") <xpath:string>("XPATH expression"), cli_pagination("use xpath var", "es", "http://example.com/ns/example-social", "default", "10");
pagination("Show list pagination") xpath("Show configuration") <xpath:string>("XPATH expression"), cli_pagination("use xpath var", "es", "https://example.com/ns/example-social", "default", "10");
configuration("Show configuration"), cli_show_auto_mode("candidate", "default", true, false, "explicit", "set ");{
@|example_pipe, cli_show_auto_mode("candidate", "xml", true, false, "explicit");
default("With-default mode"){

View file

@ -101,7 +101,7 @@
* and thus an xpath on the form "../PARENT" may not be evaluated as they should. x0 is eventually
* added to its parent but then it is more difficult to check the when condition.
* This fix add the parent x0p as a "candidate" so that the xpath-eval function can use it as
* an alernative if it exists.
* an alternative if it exists.
* Note although this solves many usecases involving parents and absolute paths, it still does not
* solve all usecases, such as absolute usecases where the added node is looked for
*/

View file

@ -73,7 +73,7 @@ DATASTORE_TOP="config"
# clixon yang revisions occuring in tests (see eg yang/clixon/Makefile.in)
CLIXON_AUTOCLI_REV="2023-09-01"
CLIXON_LIB_REV="2024-04-01"
CLIXON_LIB_REV="2024-08-01"
CLIXON_CONFIG_REV="2024-04-01"
CLIXON_RESTCONF_REV="2022-08-01"
CLIXON_EXAMPLE_REV="2022-11-01"

View file

@ -1,13 +1,13 @@
#!/usr/bin/env bash
# Example-social from draft-netconf-list-pagination-00.txt appendix A.1
# Assumes variable fexample is set to name of yang file
# Note audit-logs/audit-log/outcome is changed from mandatory to default
# Also: leaf-list member/state/numbers is added
# Mark deviation from original with "Clixon"
cat <<EOF > $fexample
module example-social {
yang-version 1.1;
namespace "http://example.com/ns/example-social";
namespace "https://example.com/ns/example-social";
prefix es;
import ietf-yang-types {
@ -20,18 +20,21 @@ cat <<EOF > $fexample
reference
"RFC 6991: Common YANG Data Types";
}
import iana-crypt-hash {
prefix ianach;
reference
"RFC 7317: A YANG Data Model for System Management";
}
import clixon-lib { // Clixon
prefix cl;
}
organization "Example, Inc.";
contact "support@example.com";
description "Example Social Data Model.";
revision 2021-07-21 { /* clixon edit */
// revision YYYY-MM-DD {
revision 2021-07-21 { // Clixon
description
"Initial version.";
reference
@ -58,7 +61,8 @@ cat <<EOF > $fexample
}
leaf email-address {
type string;
// type inet:email-address;
type string; // Clixon
mandatory true;
description
"The member's email address.";
@ -119,7 +123,7 @@ cat <<EOF > $fexample
leaf-list following {
type leafref {
path "/members/member/member-id";
require-instance false;
require-instance false; // Clixon
}
description
"Other members this members is following.";
@ -216,7 +220,7 @@ cat <<EOF > $fexample
config false;
description
"Operational state members values.";
leaf-list numbers {
leaf-list numbers { // Clixon
description "config false extension";
type int32;
}
@ -261,6 +265,7 @@ cat <<EOF > $fexample
list audit-log {
description
"List of audit logs.";
cl:list_pagination_partial_state; // Clixon
leaf timestamp {
type yang:date-and-time;
mandatory true;
@ -287,7 +292,7 @@ cat <<EOF > $fexample
}
leaf outcome {
type boolean;
default true; /* Note changed from mandatory in original */
mandatory true;
description
"Indicate if request was permitted.";
}

View file

@ -30,7 +30,7 @@ fexample=$dir/example-social.yang
# Validate internal state xml
: ${validatexml:=false}
# Number of audit-log entries
# Number of leaf-list entries
# Note mem.sh sets it
: ${perfnr:=20000}
@ -60,7 +60,7 @@ EOF
# start file
cat <<'EOF' > $dir/startup_db
<config>
<members xmlns="http://example.com/ns/example-social">
<members xmlns="https://example.com/ns/example-social">
<member>
<member-id>alice</member-id>
<email-address>alice@example.com</email-address>
@ -163,7 +163,7 @@ xpath="/es:members/es:member[es:member-id=\'bob\']/es:favorites/es:uint64-number
new "cli show pagination config using expect"
sudo="sudo -g ${CLICON_GROUP}" ## cheat
clixon_cli_="${clixon_cli##$sudo }"
clixon_cli="$clixon_cli_" $sudo --preserve-env=clixon_cli expect ./test_pagination_expect.exp "$cfg" "$xpath" "uint64-numbers 18" "uint64-numbers 19"
clixon_cli="$clixon_cli_" $sudo --preserve-env=clixon_cli expect ./test_pagination_expect.exp "$cfg" "$xpath" "uint64-numbers 20" "uint64-numbers 21"
if [ $? -ne 0 ]; then
err1 "Failed: CLI show paginate config scroll using expect"
fi

View file

@ -166,7 +166,7 @@ EOF
# See draft-wwlh-netconf-list-pagination-00 A.2 (only stats and audit-log)
cat<<EOF > $fstate
<members xmlns="http://example.com/ns/example-social">
<members xmlns="https://example.com/ns/example-social">
<member>
<member-id>alice</member-id>
<stats>
@ -208,7 +208,7 @@ cat<<EOF > $fstate
</stats>
</member>
</members>
<audit-logs xmlns="http://example.com/ns/example-social">
<audit-logs xmlns="https://example.com/ns/example-social">
<audit-log>
<timestamp>": "2020-10-11T06:47:59Z",</timestamp>
<member-id>alice</member-id>
@ -287,16 +287,16 @@ function testlimit()
# if [ $limit == 0 ]; then
if true; then
el="<uint8-numbers>$li</uint8-numbers>"
el2="<uint8-numbers xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"
el2="<uint8-numbers xmlns=\"https://example.com/ns/example-social\">$li</uint8-numbers>"
else
el="<uint8-numbers lp:remaining=\"$remaining\" xmlns:lp=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination\">$li</uint8-numbers>"
el2="<uint8-numbers lp:remaining=\"$remaining\" xmlns:lp=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination\" xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"
el2="<uint8-numbers lp:remaining=\"$remaining\" xmlns:lp=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination\" xmlns=\"https://example.com/ns/example-social\">$li</uint8-numbers>"
jsonmeta=",\"@example-social:uint8-numbers\":\[{\"ietf-list-pagination:remaining\":$remaining}\]"
fi
jsonlist="$li"
else
el="<uint8-numbers>$li</uint8-numbers>"
el2="<uint8-numbers xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>" jsonlist="$jsonlist,$li"
el2="<uint8-numbers xmlns=\"https://example.com/ns/example-social\">$li</uint8-numbers>" jsonlist="$jsonlist,$li"
fi
xmllist="$xmllist$el"
xmllist2="$xmllist2$el2"
@ -324,18 +324,18 @@ function testlimit()
if [ -z "$list" ]; then
reply="<rpc-reply $DEFAULTNS><data/></rpc-reply>"
else
reply="<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites>$xmllist</favorites></member></members></data></rpc-reply>"
reply="<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites>$xmllist</favorites></member></members></data></rpc-reply>"
fi
new "limit=$limit offset=$offset NETCONF get-config"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\">$limitxmlstr$offsetxmlstr</list-pagination></get-config></rpc>" "" "$reply"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\">$limitxmlstr$offsetxmlstr</list-pagination></get-config></rpc>" "" "$reply"
if [ -z "$list" ]; then
reply="<rpc-reply $DEFAULTNS><data/></rpc-reply>"
else
reply="<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites>$xmllist</favorites></member></members></data></rpc-reply>"
reply="<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites>$xmllist</favorites></member></members></data></rpc-reply>"
fi
new "limit=$limit offset=$offset NETCONF get"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\">$limitxmlstr$offsetxmlstr</list-pagination></get></rpc>" "" "$reply"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\">$limitxmlstr$offsetxmlstr</list-pagination></get></rpc>" "" "$reply"
if [ -z "$list" ]; then
reply="<xml-list xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination\"/>"
@ -383,7 +383,7 @@ new "wait restconf"
wait_restconf
new "Baseline: no pagination"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><favorite"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"https://example.com/ns/example-social\"/></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><favorite"
new "A.3.1.1. limit=1"
testlimit 0 1 5 "17"

View file

@ -1,4 +1,5 @@
#!/usr/bin/env expect -f
# Use of expect to emulate a terminal with given size, difficult in shell-based test-scripts
# Tests of paginated state scrolling using expect. Simply that --More-- is shown and
# that first two pages scroll OK. More tests could be done.
# Arguments:
@ -10,7 +11,7 @@
set timeout 1
#log_user 0
set stty_init "rows 20 cols 128"
set stty_init "rows 22 cols 128"
send_user "\nTest State paginate cli scrolling\n"

View file

@ -157,7 +157,7 @@ EOF
# See draft-wwlh-netconf-list-pagination-00 A.2 (only stats and audit-log)
cat<<EOF > $fstate
<members xmlns="http://example.com/ns/example-social">
<members xmlns="https://example.com/ns/example-social">
<member>
<member-id>bob</member-id>
<stats>
@ -221,38 +221,38 @@ wait_backend
new "A.3.4.1. direction=forwards"
# 17, 13, 11, 7, 5, 3]
# Confusing: forwards means dont change order
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><direction>forwards</direction></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites><uint8-numbers>17</uint8-numbers><uint8-numbers>13</uint8-numbers><uint8-numbers>11</uint8-numbers><uint8-numbers>7</uint8-numbers><uint8-numbers>5</uint8-numbers><uint8-numbers>3</uint8-numbers></favorites></member></members></data></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><direction>forwards</direction></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites><uint8-numbers>17</uint8-numbers><uint8-numbers>13</uint8-numbers><uint8-numbers>11</uint8-numbers><uint8-numbers>7</uint8-numbers><uint8-numbers>5</uint8-numbers><uint8-numbers>3</uint8-numbers></favorites></member></members></data></rpc-reply>"
new "A.3.4.2. direction=backwards"
# 3, 5, 7, 11, 13, 17]
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><direction>backwards</direction></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites><uint8-numbers>3</uint8-numbers><uint8-numbers>5</uint8-numbers><uint8-numbers>7</uint8-numbers><uint8-numbers>11</uint8-numbers><uint8-numbers>13</uint8-numbers><uint8-numbers>17</uint8-numbers></favorites></member></members></data></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><direction>backwards</direction></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites><uint8-numbers>3</uint8-numbers><uint8-numbers>5</uint8-numbers><uint8-numbers>7</uint8-numbers><uint8-numbers>11</uint8-numbers><uint8-numbers>13</uint8-numbers><uint8-numbers>17</uint8-numbers></favorites></member></members></data></rpc-reply>"
new "A.3.5.1.1. sort-by type is a leaf-list"
# 3,5,7,11,13,17
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><sort-by>.</sort-by></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites><uint8-numbers>3</uint8-numbers><uint8-numbers>5</uint8-numbers><uint8-numbers>7</uint8-numbers><uint8-numbers>11</uint8-numbers><uint8-numbers>13</uint8-numbers><uint8-numbers>17</uint8-numbers></favorites></member></members></data></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><sort-by>.</sort-by></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><favorites><uint8-numbers>3</uint8-numbers><uint8-numbers>5</uint8-numbers><uint8-numbers>7</uint8-numbers><uint8-numbers>11</uint8-numbers><uint8-numbers>13</uint8-numbers><uint8-numbers>17</uint8-numbers></favorites></member></members></data></rpc-reply>"
new "A.3.5.1.2. sort-by type is a list and sort-by node is a direct descendent"
# alice, bob, eric, joe, lin
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><sort-by>member-id</sort-by></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id>.*<member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>joe</member-id>.*<member-id>lin</member-id>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><sort-by>member-id</sort-by></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id>.*<member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>joe</member-id>.*<member-id>lin</member-id>"
new "A.3.5.1.3. sort-by type is a list and sort-by node is an indirect descendent"
# alice, lin, bob, eric, joe
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><sort-by>stats/joined</sort-by></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id>.*<member-id>lin</member-id>.*<member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>joe</member-id>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><sort-by>stats/joined</sort-by></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id>.*<member-id>lin</member-id>.*<member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>joe</member-id>"
new "A.3.6.2. where, match on descendent string containing a substring"
# bob, eric, alice, lin, joe
# Confusing: all match
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><where>.[contains (email-address,'@example.com')]</where></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>alice</member-id>.*<member-id>lin</member-id>.*<member-id>joe</member-id>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><where>.[contains (email-address,'@example.com')]</where></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>alice</member-id>.*<member-id>lin</member-id>.*<member-id>joe</member-id>"
new "A.3.6.3. where, match on decendent timestamp starting with a substring"
# bob, eric, alice, joe,
# starts-with NYI, replaced with contains
# posts//post[starts-with(timestamp,'2020')]
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><where>posts/post[contains(timestamp,'2020')]</where></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>alice</member-id>.*<member-id>joe</member-id>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><where>posts/post[contains(timestamp,'2020')]</where></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>bob</member-id>.*<member-id>eric</member-id>.*<member-id>alice</member-id>.*<member-id>joe</member-id>"
new "A.3.9.1. All six parameters at once"
# eric, bob
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><where>//post[contains(timestamp,'2020')]</where><sort-by>member-id</sort-by><direction>backwards</direction><offset>2</offset><limit>2</limit></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>eric</member-id>.*<member-id>bob</member-id>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><where>//post[contains(timestamp,'2020')]</where><sort-by>member-id</sort-by><direction>backwards</direction><offset>2</offset><limit>2</limit></list-pagination></get></rpc>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>eric</member-id>.*<member-id>bob</member-id>"
if [ $BE -ne 0 ]; then
new "Kill backend"

View file

@ -7,6 +7,7 @@
# 3. CLI get audit logs (only interactive)
# This tests contains a large state list: audit-logs from the example
# Only CLI is used
# Test also of list_pagination_partial_state extension
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@ -53,7 +54,7 @@ EOF
# See draft-wwlh-netconf-list-pagination-00 A.2 (only stats and audit-log)
cat<<EOF > $fstate
<members xmlns="http://example.com/ns/example-social">
<members xmlns="https://example.com/ns/example-social">
<member>
<member-id>alice</member-id>
<stats>
@ -83,13 +84,14 @@ EOF
# Generation of random timestamps (not used)
# and succesive bob$i member-ids
new "generate state with $perfnr list entries"
echo "<audit-logs xmlns=\"http://example.com/ns/example-social\">" >> $fstate
echo "<audit-logs xmlns=\"https://example.com/ns/example-social\">" >> $fstate
for (( i=0; i<$perfnr; i++ )); do
echo " <audit-log>" >> $fstate
echo " <timestamp>2021-09-05T018:48:11Z</timestamp>" >> $fstate
echo " <member-id>bob$i</member-id>" >> $fstate
echo " <source-ip>192.168.1.32</source-ip>" >> $fstate
echo " <request>POST</request>" >> $fstate
echo " <outcome>true</outcome>" >> $fstate
echo " </audit-log>" >> $fstate
done
@ -136,15 +138,15 @@ xpath="$xpath0/es:numbers"
testrun_start $xpath
new "NETCONF get leaf-list member/numbers 0-10 alice"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><stats><numbers>3</numbers><numbers>4</numbers><numbers>5</numbers><numbers>6</numbers><numbers>7</numbers><numbers>8</numbers></stats></member></members></data></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><stats><numbers>3</numbers><numbers>4</numbers><numbers>5</numbers><numbers>6</numbers><numbers>7</numbers><numbers>8</numbers></stats></member></members></data></rpc-reply>"
# negative
new "NETCONF get container, expect fail"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath0\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>invalid-value</error-tag><error-severity>error</error-severity><error-message>list-pagination is enabled but target is not list or leaf-list</error-message></rpc-error></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath0\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>invalid-value</error-tag><error-severity>error</error-severity><error-message>list-pagination is enabled but target is not list or leaf-list</error-message></rpc-error></rpc-reply>"
xpath="/es:members/es:member[es:member-id='bob']/es:stats/es:numbers"
new "NETCONF get leaf-list member/numbers 0-10 bob"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>bob</member-id><stats><numbers>13</numbers><numbers>14</numbers><numbers>15</numbers><numbers>16</numbers><numbers>17</numbers><numbers>18</numbers></stats></member></members></data></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>bob</member-id><stats><numbers>13</numbers><numbers>14</numbers><numbers>15</numbers><numbers>16</numbers><numbers>17</numbers><numbers>18</numbers></stats></member></members></data></rpc-reply>"
testrun_stop
@ -153,7 +155,7 @@ xpath="/es:members/es:member/es:stats/es:numbers"
testrun_start $xpath
new "NETCONF get leaf-list member/numbers all"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><stats><numbers>3</numbers><numbers>4</numbers><numbers>5</numbers><numbers>6</numbers><numbers>7</numbers><numbers>8</numbers></stats></member><member><member-id>bob</member-id><stats><numbers>13</numbers><numbers>14</numbers><numbers>15</numbers><numbers>16</numbers></stats></member></members></data></rpc-reply>"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get content=\"nonconfig\"><filter type=\"xpath\" select=\"$xpath\" xmlns:es=\"https://example.com/ns/example-social\"/><list-pagination xmlns=\"urn:ietf:params:xml:ns:yang:ietf-list-pagination-nc\"><offset>0</offset><limit>10</limit></list-pagination></get></rpc>" "" "<rpc-reply $DEFAULTNS><data><members xmlns=\"https://example.com/ns/example-social\"><member><member-id>alice</member-id><stats><numbers>3</numbers><numbers>4</numbers><numbers>5</numbers><numbers>6</numbers><numbers>7</numbers><numbers>8</numbers></stats></member><member><member-id>bob</member-id><stats><numbers>13</numbers><numbers>14</numbers><numbers>15</numbers><numbers>16</numbers></stats></member></members></data></rpc-reply>"
testrun_stop
@ -165,8 +167,10 @@ if [ -n "$(type expect 2> /dev/null)" ]; then
testrun_start "/es:audit-logs/es:audit-log"
new "CLI scroll test using expect"
xpath="/es:audit-logs/es:audit-log"
sudo="sudo -g ${CLICON_GROUP}" ## cheat
clixon_cli_="${clixon_cli##$sudo }"
# echo "$sudo --preserve-env=clixon_cli expect ./test_pagination_expect.exp $cfg $xpath bob3 bob4"
clixon_cli="$clixon_cli_" $sudo --preserve-env=clixon_cli expect ./test_pagination_expect.exp "$cfg" "$xpath" bob3 bob4
if [ $? -ne 0 ]; then
err1 "Failed: CLI show paginate state scroll using expect"

View file

@ -43,7 +43,7 @@ YANG_INSTALLDIR = @YANG_INSTALLDIR@
# Note: mirror these to test/config.sh.in
YANGSPECS = clixon-config@2024-04-01.yang # 7.1
YANGSPECS += clixon-lib@2024-04-01.yang # 7.1
YANGSPECS += clixon-lib@2024-08-01.yang # 7.2
YANGSPECS += clixon-rfc5277@2008-07-01.yang
YANGSPECS += clixon-xml-changelog@2019-03-21.yang
YANGSPECS += clixon-restconf@2022-08-01.yang # 5.9

View file

@ -66,8 +66,21 @@ module clixon-lib {
- source-host (see RFC6022)
- objectcreate
- objectexisted
- link # For split multiple XML files
";
revision 2024-08-01 {
description
"Added: list-pagination-partial-state
Released in Clixon 7.2";
}
revision 2024-04-01 {
description
"Added: debug bits type
Added: xmldb-split extension
Added: Default format
Released in Clixon 7.1";
}
revision 2024-01-01 {
description
"Removed container creators from 6.5
@ -175,6 +188,116 @@ module clixon-lib {
enum cli{
description "CLI format";
}
enum default{
description "Default format";
}
}
}
typedef clixon_debug_t {
description
"Debug flags.
Flags are seperated into subject areas and detail
Can also be given directly as -D <flag> to clixon commands
Note there are also constants in the code that need to be in sync with these values";
type bits {
/* Subjects: */
bit default {
description "Default logs";
position 0;
}
bit msg {
description "In/out messages";
position 1;
}
bit init {
description "Initialization";
position 2;
}
bit xml {
description "XML processing";
position 3;
}
bit xpath {
description "XPath processing";
position 4;
}
bit yang {
description "YANG processing";
position 5;
}
bit backend {
description "Backend-specific";
position 6;
}
bit cli {
description "CLI frontend";
position 7;
}
bit netconf {
description "NETCONF frontend";
position 8;
}
bit restconf {
description "RESTCONF frontend";
position 9;
}
bit snmp {
description "SNMP frontend";
position 10;
}
bit nacm {
description "NACM processing";
position 11;
}
bit proc {
description "Process handling";
position 12;
}
bit datastore {
description "Datastore xmldb management";
position 13;
}
bit event {
description "Event processing";
position 14;
}
bit rpc {
description "RPC handling";
position 15;
}
bit stream {
description "Notification streams";
position 16;
}
bit parse {
description "Parser: XML,YANG, etc";
position 17;
}
bit app {
description "External applications";
position 20;
}
bit app2 {
description "External application";
position 21;
}
bit app3 {
description "External application 2";
position 22;
}
/* Detail level: */
bit detail {
description "Details: traces, parse trees, etc";
position 24;
}
bit detail2 {
description "Extra details";
position 25;
}
bit detail3 {
description "Probably more detail than you want";
position 26;
}
}
}
identity snmp {
@ -199,12 +322,34 @@ module clixon-lib {
"A CLI session";
base ncm:transport;
}
extension list-pagination-partial-state {
description
"List should be partially read according to the clixon_pagination_cb_register API.
This is a performance enhancement of pagination state data.
This means that a special callback is used for retreiving list state which is aware of
offset/limit attributes.
In this way the non-config data can be partially read by the server, instead of reading
the whole state on every pagination request.
It affects only the server/backend-side
It only handles the offset and limit attributes, all other attributes,
such as where, sort-by, direction, etc, are ignored";
}
extension ignore-compare {
description
"The object should be ignored when comparing device configs for equality.
The object should never be added, modified, or deleted on target.
Essentially a read-only object
One example is auto-created objects by the , such as uid.";
One example is auto-created objects by the controller, such as uid.";
}
extension xmldb-split {
description
"When split configuration stores are used, ie CLICON_XMLDB_MULTI is set,
This extension marks where in the configuration tree, one file terminates
and a new sub-file is written.
A designer adds the 'xmldb-split' extension to a YANG node which should be split.
For example, a split could be made at mountpoints.
See also the 'link 'attribute.
";
}
md:annotation creator {
type string;
@ -217,7 +362,9 @@ module clixon-lib {
A sub-object will not be noted";
}
rpc debug {
description "Set debug level of backend.";
description
"Set debug flags of backend.
Note only numerical values";
input {
leaf level {
type uint32;