- pageing offset working

This commit is contained in:
Olof hagsand 2021-08-27 14:22:47 +02:00
parent fb0b9409f3
commit 390b0886ed
5 changed files with 186 additions and 66 deletions

View file

@ -336,6 +336,8 @@ element2value(clicon_handle h,
* @retval -1 Error
* @see from_client_get
* @see from_client_get_config
* @note pagination uses appending xpath with predicate, eg [position()<limit], this may not work
* if there is an existing predicate
*/
static int
get_common(clicon_handle h,
@ -348,6 +350,7 @@ get_common(clicon_handle h,
int retval = -1;
cxobj *xfilter;
char *xpath = NULL;
char *xpath2; /* With optional pageing predicate */
cxobj *xret = NULL;
cxobj **xvec = NULL;
size_t xlen;
@ -517,14 +520,10 @@ get_common(clicon_handle h,
/* XXX remaining of state list??*/
}
/* Append predicate to original xpath and replace it */
if (xpath)
free(xpath);
if ((xpath = strdup(cbuf_get(cbpath))) == NULL){
clicon_err(OE_UNIX, errno, "strdup");
goto done;
}
xpath2 = cbuf_get(cbpath);
} /* list_pagination */
else
xpath2 = xpath;
#endif /* LIST_PAGINATION */
/* Read config
* XXX This seems unnecessary complex
@ -532,7 +531,7 @@ get_common(clicon_handle h,
switch (content){
case CONTENT_CONFIG: /* config data only */
/* specific xpath */
if (xmldb_get0(h, db, YB_MODULE, nsc, xpath?xpath:"/", 1, &xret, NULL, NULL) < 0) {
if (xmldb_get0(h, db, YB_MODULE, nsc, xpath2?xpath2:"/", 1, &xret, NULL, NULL) < 0) {
if ((cbmsg = cbuf_new()) == NULL){
clicon_err(OE_UNIX, errno, "cbuf_new");
goto done;
@ -560,7 +559,7 @@ get_common(clicon_handle h,
}
else if (content == CONTENT_ALL){
/* specific xpath */
if (xmldb_get0(h, db, YB_MODULE, nsc, xpath?xpath:"/", 1, &xret, NULL, NULL) < 0) {
if (xmldb_get0(h, db, YB_MODULE, nsc, xpath2?xpath2:"/", 1, &xret, NULL, NULL) < 0) {
if ((cbmsg = cbuf_new()) == NULL){
clicon_err(OE_UNIX, errno, "cbuf_new");
goto done;
@ -585,7 +584,7 @@ get_common(clicon_handle h,
break;
case CONTENT_ALL: /* both config and state */
case CONTENT_NONCONFIG: /* state data only */
if ((ret = client_statedata(h, xpath?xpath:"/", nsc, &xret)) < 0)
if ((ret = client_statedata(h, xpath2?xpath2:"/", nsc, &xret)) < 0)
goto done;
if (ret == 0){ /* Error from callback (error in xret) */
if (clicon_xml2cbuf(cbret, xret, 0, 0, -1) < 0)
@ -635,6 +634,7 @@ get_common(clicon_handle h,
* and modules_state functions.
* But it is problematic, because defaults, at least of config data, is in place
* and we need to re-add it.
* Note original xpath
*/
if (xpath_vec(xret, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
goto done;
@ -662,8 +662,10 @@ get_common(clicon_handle h,
goto done;
#ifdef LIST_PAGINATION
/* Add remaining attribute */
if (list_pagination && remaining && xlen){
/* Add remaining attribute Sec 3.1.5:
Any list or leaf-list that is limited includes, on the first element in the result set,
a metadata value [RFC7952] called "remaining"*/
if (list_pagination && limit && xlen){
cxobj *xa;
cbuf *cba = NULL;

View file

@ -338,6 +338,7 @@ api_data_collection(clicon_handle h,
cxobj *xerr = NULL; /* malloced */
cxobj *xe = NULL; /* not malloced */
cxobj **xvec = NULL;
size_t xlen = 0;
int i;
int ret;
cvec *nsc = NULL;
@ -355,6 +356,7 @@ api_data_collection(clicon_handle h,
char *direction;
char *sort;
char *where;
char *ns;
clicon_debug(1, "%s", __FUNCTION__);
if ((yspec = clicon_dbspec_yang(h)) == NULL){
@ -479,35 +481,8 @@ api_data_collection(clicon_handle h,
goto done;
if (xmlns_set(xpr, NULL, RESTCONF_PAGINATON_NAMESPACE) < 0)
goto done;
if ((xp = xpath_first(xret, nsc, "%s", xpath)) != NULL){
char *ns=NULL;
if (xml2ns(xp, NULL, &ns) < 0)
goto done;
if (ns != NULL){
if (xmlns_set(xp, NULL, ns) < 0)
goto done;
}
if (xml_rm(xp) < 0)
goto done;
if (xml_insert(xpr, xp, INS_LAST, NULL, NULL) < 0)
goto done;
}
/* Normal return, no error */
if ((cbx = cbuf_new()) == NULL)
if (xpath_vec(xret, nsc, "%s", &xvec, &xlen, xpath) < 0)
goto done;
switch (media_out){
case YANG_COLLECTION_XML:
if (clicon_xml2cbuf(cbx, xpr, 0, pretty, -1) < 0) /* Dont print top object? */
goto done;
break;
case YANG_COLLECTION_JSON:
if (xml2json_cbuf(cbx, xpr, pretty) < 0)
goto done;
break;
default:
break;
}
#if 0
/* Check if not exists */
if (xlen == 0){
@ -525,6 +500,35 @@ api_data_collection(clicon_handle h,
goto ok;
}
#endif
for (i=0; i<xlen; i++){
xp = xvec[i];
ns = NULL;
if (xml2ns(xp, NULL, &ns) < 0)
goto done;
if (ns != NULL){
if (xmlns_set(xp, NULL, ns) < 0)
goto done;
}
if (xml_rm(xp) < 0)
goto done;
if (xml_insert(xpr, xp, INS_LAST, NULL, NULL) < 0)
goto done;
}
/* Normal return, no error */
if ((cbx = cbuf_new()) == NULL)
goto done;
switch (media_out){
case YANG_COLLECTION_XML:
if (clicon_xml2cbuf(cbx, xpr, 0, pretty, -1) < 0) /* Dont print top object? */
goto done;
break;
case YANG_COLLECTION_JSON:
if (xml2json_cbuf(cbx, xpr, pretty) < 0)
goto done;
break;
default:
break;
}
clicon_debug(1, "%s cbuf:%s", __FUNCTION__, cbuf_get(cbx));
if (restconf_reply_header(req, "Content-Type", "%s", restconf_media_int2str(media_out)) < 0)
goto done;

View file

@ -711,9 +711,9 @@ json_metadata_encoding(cbuf *cb,
{
int retval = -1;
cprintf(cb, ",\"@%s:%s\": [", modname, name);
cprintf(cb, ",\"@%s:%s\":[", modname, name);
cprintf(cb, "%*s", pretty?((level+1)*JSON_INDENT):0, "{");
cprintf(cb, "\"%s:%s\": %s", modname2, name2, val);
cprintf(cb, "\"%s:%s\":%s", modname2, name2, val);
cprintf(cb, "%*s", pretty?((level+1)*JSON_INDENT):0, "}");
cprintf(cb, "%*s", pretty?(level*JSON_INDENT):0, "]");
retval = 0;
@ -766,7 +766,8 @@ xml2json1_cbuf(cbuf *cb,
int level,
int pretty,
int flat,
char *modname0)
char *modname0,
cbuf **metacbp)
{
int retval = -1;
int i;
@ -778,6 +779,7 @@ xml2json1_cbuf(cbuf *cb,
yang_stmt *ymod = NULL; /* yang module */
int commas;
char *modname = NULL;
cbuf *metacbc = NULL;
if ((ys = xml_spec(x)) != NULL){
if (ys_real_module(ys, &ymod) < 0)
@ -873,22 +875,62 @@ xml2json1_cbuf(cbuf *cb,
commas = xml_child_nr_notype(x, CX_ATTR) - 1;
for (i=0; i<xml_child_nr(x); i++){
xc = xml_child_i(x, i);
if (xml_type(xc) == CX_ATTR)
if (xml_type(xc) == CX_ATTR){
int ismeta = 0;
char *namespace = NULL;
yang_stmt *ymod;
cbuf *metacb = NULL;
if (xml2ns(xc, xml_prefix(xc), &namespace) < 0)
goto done;
if (namespace == NULL)
continue;
if ((ymod = yang_find_module_by_namespace(ys_spec(ys), namespace)) == NULL)
continue;
if (xml2ns(xc, xml_prefix(xc), &namespace) < 0)
goto done;
if (yang_metadata_annotation_check(xc, ymod, &ismeta) < 0)
goto done;
if (!ismeta)
continue;
if ((metacb = cbuf_new()) == NULL){
clicon_err(OE_UNIX, errno, "cbuf_new");
goto done;
}
if (json_metadata_encoding(metacb, x, level, pretty,
modname, xml_name(x),
yang_argument_get(ymod),
xml_name(xc),
xml_value(xc)) < 0)
goto done;
if (metacbp)
*metacbp = metacb;
else
cbuf_free(metacb);
continue; /* XXX Only xmlns attributes mapped */
}
xc_arraytype = array_eval(i?xml_child_i(x,i-1):NULL,
xc,
xml_child_i(x, i+1));
if (xml2json1_cbuf(cb,
xc,
xc_arraytype,
level+1, pretty, 0, modname0) < 0)
level+1, pretty, 0, modname0,
&metacbc) < 0)
goto done;
if (commas > 0) {
cprintf(cb, ",%s", pretty?"\n":"");
--commas;
}
}
#ifdef LIST_PAGINATION /* identify md:annotations as RFC 7952 Sec 5.2.1*/
if (metacbc){
cprintf(cb, "%s", cbuf_get(metacbc));
}
#endif
#if 0 /* identify md:annotations as RFC 7952 Sec 5.2.1*/
for (i=0; i<xml_child_nr(x); i++){
xc = xml_child_i(x, i);
if (xml_type(xc) == CX_ATTR){
@ -976,6 +1018,8 @@ xml2json1_cbuf(cbuf *cb,
}
retval = 0;
done:
if (metacbc)
cbuf_free(metacbc);
return retval;
}
@ -1017,8 +1061,8 @@ xml2json_cbuf(cbuf *cb,
level+1,
pretty,
0,
NULL /* ancestor modname / namespace */
) < 0)
NULL, /* ancestor modname / namespace */
NULL) < 0)
goto done;
cprintf(cb, "%s%*s}%s",
pretty?"\n":"",
@ -1077,7 +1121,7 @@ xml2json_cbuf_vec(cbuf *cb,
xp,
NO_ARRAY,
level+1, pretty,
1, NULL) < 0)
1, NULL, NULL) < 0)
goto done;
if (0){

View file

@ -261,28 +261,73 @@ EOF
# Run limit-only test with netconf, restconf+xml and restconf+json
# Args:
# 1. limit
# 2. remaining
# 3. list XXX remaining
# 1. offset
# 2. limit
# 3. remaining
# 4. list
function testlimit()
{
limit=$1
remaining=$2
offset=$1
limit=$2
remaining=$3
list=$4
# "clixon get"
new "clixon limit=$limit NETCONF get-config"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get-config></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
xmllist="" # for netconf
xmllist2="" # for restconf xml
jsonlist="" # for restconf json
jsonmeta=""
let i=0
for li in $list; do
if [ $i = 0 ]; then
if [ $limit == 0 ]; then
el="<uint8-numbers>$li</uint8-numbers>"
el2="<uint8-numbers xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"
else
el="<uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">$li</uint8-numbers>"
el2="<uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\" xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"
jsonmeta=",\"@example-social:uint8-numbers\":\[{\"clixon-netconf-list-pagination:remaining\":$remaining}\]"
fi
jsonlist="$li"
else
el="<uint8-numbers>$li</uint8-numbers>"
el2="<uint8-numbers xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>" jsonlist="$jsonlist,$li"
fi
xmllist="$xmllist$el"
xmllist2="$xmllist2$el2"
let i++
done
new "clixon limit=$limit NETCONF get"
expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
jsonstr=""
if [ $limit -eq 0 ]; then
limitxmlstr=""
else
limitxmlstr="<limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit>"
jsonstr="?limit=$limit"
fi
if [ $offset -eq 0 ]; then
offsetxmlstr=""
else
offsetxmlstr="<offset xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$offset</offset>"
if [ -z "$jsonstr" ]; then
jsonstr="?offset=$offset"
else
jsonstr="${jsonstr}&offset=$offset"
fi
fi
new "limit=$limit NETCONF get-config"
# expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination>$limitxmlstr$offsetxmlstr</get-config></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites>$xmllist</favorites></member></members></data></rpc-reply>]]>]]>$"
new "limit=$limit NETCONF get"
# expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination>$limitxmlstr$offsetxmlstr</get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites>$xmllist</favorites></member></members></data></rpc-reply>]]>]]>$"
new "limit=$limit Parameter RESTCONF xml"
expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+xml" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers?limit=$limit)" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+xml" "<yang-collection xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf-list-pagination\"><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\" xmlns=\"http://example.com/ns/example-social\">17</uint8-numbers></yang-collection>"
expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+xml" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers${jsonstr})" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+xml" "<yang-collection xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf-list-pagination\">$xmllist2</yang-collection>"
# XXX [17]
new "limit=$limit Parameter RESTCONF json"
expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+json" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers?limit=$limit)" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+json" '{"yang-collection":{"example-social:uint8-numbers":17,"@example-social:uint8-numbers": \[{"clixon-netconf-list-pagination:remaining": 5}\]}}'
}
expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+json" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers${jsonstr})" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+json" "{\"yang-collection\":{\"example-social:uint8-numbers\":\[$jsonlist\]$jsonmeta}"
} # testrunf
new "test params: -f $cfg -s startup -- -sS $fstate"
@ -313,10 +358,35 @@ new "wait restconf"
wait_restconf
new "A.3.1.1. limit=1"
testlimit 1 5 "[17]"
testlimit 0 1 5 "17"
#new "A.3.1.2. limit=2"
#testlimit 2 4 "[17 13]"
new "A.3.1.2. limit=2"
testlimit 0 2 4 "17 13"
new "A.3.1.3. limit=5"
testlimit 0 5 1 "17 13 11 7 5"
new "A.3.1.4. limit=6"
testlimit 0 6 0 "17 13 11 7 5 3"
new "A.3.1.5. limit=7"
testlimit 0 7 0 "17 13 11 7 5 3"
new "A.3.2.1. offset=1"
testlimit 1 0 0 "13 11 7 5 3"
new "A.3.2.2. offset=2"
testlimit 2 0 0 "11 7 5 3"
new "A.3.2.3. offset=5"
testlimit 5 0 0 "3"
#new "A.3.2.4. offset=6"
#testlimit 6 0 0 ""
# This is incomplete wrt the draft
new "A.3.7. limit=2 offset=2"
testlimit 2 2 2 "11 7"
# CLI
# XXX This relies on a very specific clispec command: need a more generic test

View file

@ -105,7 +105,7 @@ module clixon-netconf-list-pagination {
default "unbounded";
description
"The maximum number of list entries to return. The
value of the 'count' parameter is either an integer
value of the 'limit' parameter is either an integer
greater than or equal to 1, or the string 'unbounded'.
The string 'unbounded' is the default value.";
}
@ -119,7 +119,7 @@ module clixon-netconf-list-pagination {
default "none";
description
"The first list item to return.
the 'skip' parameter is either an integer greater than
the 'offset' parameter is either an integer greater than
or equal to 1, or the string 'unbounded'. The string
'unbounded' is the default value.";
}