- pageing offset working

apps/backend/backend_get.c

@@ -336,6 +336,8 @@ element2value(clicon_handle  h,
  * @retval    -1       Error
  * @see from_client_get
  * @see from_client_get_config 
+ * @note pagination uses appending xpath with predicate, eg [position()<limit], this may not work 
+ *       if there is an existing predicate
  */
 static int
 get_common(clicon_handle   h,
@@ -348,6 +350,7 @@ get_common(clicon_handle   h,
     int             retval = -1;
     cxobj          *xfilter;
     char           *xpath = NULL;
+    char           *xpath2; /* With optional pageing predicate */
     cxobj          *xret = NULL;
     cxobj         **xvec = NULL;
     size_t          xlen;
@@ -517,14 +520,10 @@ get_common(clicon_handle   h,
 	    /* XXX remaining of state list??*/
 	}
 	/* Append predicate to original xpath and replace it */
-	if (xpath)
+	xpath2 = cbuf_get(cbpath);
-	    free(xpath);
-	if ((xpath = strdup(cbuf_get(cbpath))) == NULL){
-	    clicon_err(OE_UNIX, errno, "strdup");
-	    goto done;
-	}
-
     } /* list_pagination */
+    else
+	xpath2 = xpath;
 #endif /* LIST_PAGINATION */
     /* Read config 
      * XXX This seems unnecessary complex
@@ -532,7 +531,7 @@ get_common(clicon_handle   h,
     switch (content){
     case CONTENT_CONFIG:    /* config data only */
 	/* specific xpath */
-	if (xmldb_get0(h, db, YB_MODULE, nsc, xpath?xpath:"/", 1, &xret, NULL, NULL) < 0) {
+	if (xmldb_get0(h, db, YB_MODULE, nsc, xpath2?xpath2:"/", 1, &xret, NULL, NULL) < 0) {
 	    if ((cbmsg = cbuf_new()) == NULL){
 		clicon_err(OE_UNIX, errno, "cbuf_new");
 		goto done;
@@ -560,7 +559,7 @@ get_common(clicon_handle   h,
 	}
 	else if (content == CONTENT_ALL){
 	    /* specific xpath */
-	    if (xmldb_get0(h, db, YB_MODULE, nsc, xpath?xpath:"/", 1, &xret, NULL, NULL) < 0) {
+	    if (xmldb_get0(h, db, YB_MODULE, nsc, xpath2?xpath2:"/", 1, &xret, NULL, NULL) < 0) {
 		if ((cbmsg = cbuf_new()) == NULL){
 		    clicon_err(OE_UNIX, errno, "cbuf_new");
 		    goto done;
@@ -585,7 +584,7 @@ get_common(clicon_handle   h,
 	break;
     case CONTENT_ALL:       /* both config and state */
     case CONTENT_NONCONFIG: /* state data only */
-	if ((ret = client_statedata(h, xpath?xpath:"/", nsc, &xret)) < 0)
+	if ((ret = client_statedata(h, xpath2?xpath2:"/", nsc, &xret)) < 0)
 	    goto done;
 	if (ret == 0){ /* Error from callback (error in xret) */
 	    if (clicon_xml2cbuf(cbret, xret, 0, 0, -1) < 0)
@@ -635,6 +634,7 @@ get_common(clicon_handle   h,
      * and modules_state functions.
      * But it is problematic, because defaults, at least of config data, is in place
      * and we need to re-add it.
+     * Note original xpath
      */
     if (xpath_vec(xret, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
 	goto done;
@@ -662,8 +662,10 @@ get_common(clicon_handle   h,
 	goto done;
 
 #ifdef LIST_PAGINATION
-    /* Add remaining attribute */
+    /* Add remaining attribute Sec 3.1.5: 
-    if (list_pagination && remaining && xlen){ 
+       Any list or leaf-list that is limited includes, on the first element in the result set, 
+       a metadata value [RFC7952] called "remaining"*/
+    if (list_pagination && limit && xlen){ 
 	cxobj *xa;
 	cbuf  *cba = NULL;
 

apps/restconf/restconf_methods_get.c

@@ -338,6 +338,7 @@ api_data_collection(clicon_handle  h,
     cxobj     *xerr = NULL; /* malloced */
     cxobj     *xe = NULL;   /* not malloced */
     cxobj    **xvec = NULL;
+    size_t     xlen = 0;
     int        i;
     int        ret;
     cvec      *nsc = NULL;
@@ -355,6 +356,7 @@ api_data_collection(clicon_handle  h,
     char      *direction;
     char      *sort;
     char      *where;
+    char      *ns;
     
     clicon_debug(1, "%s", __FUNCTION__);
     if ((yspec = clicon_dbspec_yang(h)) == NULL){
@@ -479,35 +481,8 @@ api_data_collection(clicon_handle  h,
 	goto done;
     if (xmlns_set(xpr, NULL, RESTCONF_PAGINATON_NAMESPACE) < 0)
 	goto done;
-    if ((xp = xpath_first(xret, nsc, "%s", xpath)) != NULL){
+    if (xpath_vec(xret, nsc, "%s", &xvec, &xlen, xpath) < 0)
-	char *ns=NULL;
-	if (xml2ns(xp, NULL, &ns) < 0)
 	goto done;
-	if (ns != NULL){
-	    if (xmlns_set(xp, NULL, ns) < 0)
-		goto done;
-	}
-	if (xml_rm(xp) < 0)
-	    goto done;
-	if (xml_insert(xpr, xp, INS_LAST, NULL, NULL) < 0) 
-	    goto done;
-    }
-    
-    /* Normal return, no error */
-    if ((cbx = cbuf_new()) == NULL)
-	goto done;
-    switch (media_out){
-    case YANG_COLLECTION_XML:
-	if (clicon_xml2cbuf(cbx, xpr, 0, pretty, -1) < 0) /* Dont print top object?  */
-	    goto done;
-	break;
-    case YANG_COLLECTION_JSON:
-	if (xml2json_cbuf(cbx, xpr, pretty) < 0)
-	    goto done;
-	break;
-    default:
-	break;
-    }
 #if 0
     /* Check if not exists */
     if (xlen == 0){
@@ -525,6 +500,35 @@ api_data_collection(clicon_handle  h,
 	goto ok;
     }
 #endif
+    for (i=0; i<xlen; i++){
+	xp = xvec[i];
+	ns = NULL;
+	if (xml2ns(xp, NULL, &ns) < 0)
+	    goto done;
+	if (ns != NULL){
+	    if (xmlns_set(xp, NULL, ns) < 0)
+		goto done;
+	}
+	if (xml_rm(xp) < 0)
+	    goto done;
+	if (xml_insert(xpr, xp, INS_LAST, NULL, NULL) < 0) 
+	    goto done;
+    }
+    /* Normal return, no error */
+    if ((cbx = cbuf_new()) == NULL)
+	goto done;
+    switch (media_out){
+    case YANG_COLLECTION_XML:
+	if (clicon_xml2cbuf(cbx, xpr, 0, pretty, -1) < 0) /* Dont print top object?  */
+	    goto done;
+	break;
+    case YANG_COLLECTION_JSON:
+	if (xml2json_cbuf(cbx, xpr, pretty) < 0)
+	    goto done;
+	break;
+    default:
+	break;
+    }
     clicon_debug(1, "%s cbuf:%s", __FUNCTION__, cbuf_get(cbx));
     if (restconf_reply_header(req, "Content-Type", "%s", restconf_media_int2str(media_out)) < 0)
 	goto done;

lib/src/clixon_json.c

@@ -766,7 +766,8 @@ xml2json1_cbuf(cbuf                   *cb,
 	       int                     level,
 	       int                     pretty,
 	       int                     flat,
-	       char                   *modname0)
+	       char                   *modname0,
+	       cbuf                  **metacbp)
 {
     int              retval = -1;
     int              i;
@@ -778,6 +779,7 @@ xml2json1_cbuf(cbuf                   *cb,
     yang_stmt       *ymod = NULL; /* yang module */
     int              commas;
     char            *modname = NULL;
+    cbuf            *metacbc = NULL;
 
     if ((ys = xml_spec(x)) != NULL){
 	if (ys_real_module(ys, &ymod) < 0)
@@ -873,22 +875,62 @@ xml2json1_cbuf(cbuf                   *cb,
     commas = xml_child_nr_notype(x, CX_ATTR) - 1;
     for (i=0; i<xml_child_nr(x); i++){
 	xc = xml_child_i(x, i);
-	if (xml_type(xc) == CX_ATTR)
+	if (xml_type(xc) == CX_ATTR){
+	    int        ismeta = 0;
+	    char      *namespace = NULL;
+	    yang_stmt *ymod;
+	    cbuf      *metacb = NULL;
+	    
+	    if (xml2ns(xc, xml_prefix(xc), &namespace) < 0)
+		goto done;
+	    if (namespace == NULL)
+		continue;
+	    if ((ymod = yang_find_module_by_namespace(ys_spec(ys), namespace)) == NULL)
+		continue;
+	    if (xml2ns(xc, xml_prefix(xc), &namespace) < 0)
+		goto done;
+	    if (yang_metadata_annotation_check(xc, ymod, &ismeta) < 0)
+		goto done;
+	    if (!ismeta)
+		continue;
+	    if ((metacb = cbuf_new()) == NULL){
+		clicon_err(OE_UNIX, errno, "cbuf_new");
+		goto done;
+	    }
+	    if (json_metadata_encoding(metacb, x, level, pretty,
+				       modname, xml_name(x),
+				       yang_argument_get(ymod),
+				       xml_name(xc),
+				       xml_value(xc)) < 0)
+		goto done;
+	    if (metacbp)
+		*metacbp = metacb;
+	    else
+		cbuf_free(metacb);
 	    continue; /* XXX Only xmlns attributes mapped */
+	}
 	xc_arraytype = array_eval(i?xml_child_i(x,i-1):NULL, 
 				xc, 
 				xml_child_i(x, i+1));
 	if (xml2json1_cbuf(cb, 
 			   xc, 
 			   xc_arraytype,
-			   level+1, pretty, 0, modname0) < 0)
+			   level+1, pretty, 0, modname0,
+			   &metacbc) < 0)
 	    goto done;
 	if (commas > 0) {
 	    cprintf(cb, ",%s", pretty?"\n":"");
 	    --commas;
 	}
     }
+
 #ifdef LIST_PAGINATION /* identify md:annotations as RFC 7952 Sec 5.2.1*/
+    if (metacbc){
+	cprintf(cb, "%s", cbuf_get(metacbc));
+    }
+#endif
+
+#if 0 /* identify md:annotations as RFC 7952 Sec 5.2.1*/
     for (i=0; i<xml_child_nr(x); i++){
 	xc = xml_child_i(x, i);
 	if (xml_type(xc) == CX_ATTR){
@@ -976,6 +1018,8 @@ xml2json1_cbuf(cbuf                   *cb,
     }
     retval = 0;
  done:
+    if (metacbc)
+	cbuf_free(metacbc);
     return retval;
 }
 
@@ -1017,8 +1061,8 @@ xml2json_cbuf(cbuf      *cb,
 		       level+1,
 		       pretty,
 		       0,
-		       NULL /* ancestor modname / namespace */
+		       NULL, /* ancestor modname / namespace */
-		       ) < 0)
+		       NULL) < 0)
 	goto done;
     cprintf(cb, "%s%*s}%s", 
 	    pretty?"\n":"",
@@ -1077,7 +1121,7 @@ xml2json_cbuf_vec(cbuf      *cb,
 		       xp, 
 		       NO_ARRAY,
 		       level+1, pretty,
-		       1, NULL) < 0)
+		       1, NULL, NULL) < 0)
 	goto done;
 
     if (0){

test/test_pagination.sh

@@ -261,28 +261,73 @@ EOF
 
 # Run limit-only test with netconf, restconf+xml and restconf+json
 # Args:
-# 1. limit
+# 1. offset
-# 2. remaining
+# 2. limit
-# 3. list XXX remaining
+# 3. remaining
+# 4. list
 function testlimit()
 {
-    limit=$1
+    offset=$1
-    remaining=$2
+    limit=$2
+    remaining=$3
+    list=$4
 
     # "clixon get"
-    new "clixon limit=$limit NETCONF get-config"
+    xmllist=""  # for netconf
-    expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get-config></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
+    xmllist2="" # for restconf xml
+    jsonlist="" # for restconf json
+    jsonmeta=""
+    let i=0
+    for li in $list; do
+	if [ $i = 0 ]; then
+	    if [ $limit == 0 ]; then
+		el="<uint8-numbers>$li</uint8-numbers>"
+		el2="<uint8-numbers xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"
+	    else
+		el="<uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">$li</uint8-numbers>"
+		el2="<uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\" xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"
+		jsonmeta=",\"@example-social:uint8-numbers\":\[{\"clixon-netconf-list-pagination:remaining\":$remaining}\]"
+	    fi
+	    jsonlist="$li"
+	else
+	    el="<uint8-numbers>$li</uint8-numbers>"
+	    el2="<uint8-numbers xmlns=\"http://example.com/ns/example-social\">$li</uint8-numbers>"	       jsonlist="$jsonlist,$li"
+	fi
+	xmllist="$xmllist$el"
+	xmllist2="$xmllist2$el2"
+	let i++
+    done
 
-    new "clixon limit=$limit NETCONF get"
+    jsonstr=""
-    expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination><limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit></get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\">17</uint8-numbers></favorites></member></members></data></rpc-reply>]]>]]>$"
+    if [ $limit -eq 0 ]; then
+	limitxmlstr=""
+    else
+	limitxmlstr="<limit xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$limit</limit>"
+	jsonstr="?limit=$limit"
+    fi
+    if [ $offset -eq 0 ]; then
+	offsetxmlstr=""
+    else
+	offsetxmlstr="<offset xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">$offset</offset>"
+	if [ -z "$jsonstr" ]; then
+	    jsonstr="?offset=$offset"
+	else
+	    jsonstr="${jsonstr}&offset=$offset"
+	fi
+    fi
+    new "limit=$limit NETCONF get-config"
+#    expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get-config><source><running/></source><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination>$limitxmlstr$offsetxmlstr</get-config></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites>$xmllist</favorites></member></members></data></rpc-reply>]]>]]>$"
+
+    new "limit=$limit NETCONF get"
+#    expecteof "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO<rpc $DEFAULTNS><get><filter type=\"xpath\" select=\"/es:members/es:member[es:member-id='alice']/es:favorites/es:uint8-numbers\" xmlns:es=\"http://example.com/ns/example-social\"/><list-pagination xmlns=\"http://clicon.org/clixon-netconf-list-pagination\">true</list-pagination>$limitxmlstr$offsetxmlstr</get></rpc>]]>]]>" "<rpc-reply $DEFAULTNS><data><members xmlns=\"http://example.com/ns/example-social\"><member><member-id>alice</member-id><privacy-settings><post-visibility>public</post-visibility></privacy-settings><favorites>$xmllist</favorites></member></members></data></rpc-reply>]]>]]>$"
 
     new "limit=$limit Parameter RESTCONF xml"
-    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+xml" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers?limit=$limit)" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+xml" "<yang-collection xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf-list-pagination\"><uint8-numbers cp:remaining=\"$remaining\" xmlns:cp=\"http://clicon.org/clixon-netconf-list-pagination\" xmlns=\"http://example.com/ns/example-social\">17</uint8-numbers></yang-collection>"
+    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+xml" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers${jsonstr})" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+xml" "<yang-collection xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf-list-pagination\">$xmllist2</yang-collection>"
 
-    # XXX [17]
     new "limit=$limit Parameter RESTCONF json"
-    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+json" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers?limit=$limit)" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+json" '{"yang-collection":{"example-social:uint8-numbers":17,"@example-social:uint8-numbers": \[{"clixon-netconf-list-pagination:remaining": 5}\]}}'
+    expectpart "$(curl $CURLOPTS -X GET -H "Accept: application/yang-collection+json" $RCPROTO://localhost/restconf/data/example-social:members/member=alice/favorites/uint8-numbers${jsonstr})" 0 "HTTP/$HVER 200" "Content-Type: application/yang-collection+json" "{\"yang-collection\":{\"example-social:uint8-numbers\":\[$jsonlist\]$jsonmeta}"
-}
+
+} # testrunf
 
 new "test params: -f $cfg -s startup -- -sS $fstate"
 
@@ -313,10 +358,35 @@ new "wait restconf"
 wait_restconf
 
 new "A.3.1.1. limit=1"
-testlimit 1 5 "[17]"
+testlimit 0 1 5 "17"
 
-#new "A.3.1.2. limit=2"
+new "A.3.1.2. limit=2"
-#testlimit 2 4 "[17 13]"
+testlimit 0 2 4 "17 13"
+
+new "A.3.1.3. limit=5"
+testlimit 0 5 1 "17 13 11 7 5"
+
+new "A.3.1.4. limit=6"
+testlimit 0 6 0 "17 13 11 7 5 3"
+
+new "A.3.1.5. limit=7"
+testlimit 0 7 0 "17 13 11 7 5 3"
+
+new "A.3.2.1. offset=1"
+testlimit 1 0 0 "13 11 7 5 3"
+
+new "A.3.2.2. offset=2"
+testlimit 2 0 0 "11 7 5 3"
+
+new "A.3.2.3. offset=5"
+testlimit 5 0 0 "3"
+
+#new "A.3.2.4. offset=6"
+#testlimit 6 0 0 ""
+
+# This is incomplete wrt the draft
+new "A.3.7. limit=2 offset=2"
+testlimit 2 2 2 "11 7"
 
 # CLI
 # XXX This relies on a very specific clispec command: need a more generic test

yang/clixon/clixon-netconf-list-pagination@2021-08-27.yang

@@ -105,7 +105,7 @@ module clixon-netconf-list-pagination {
 	    default "unbounded";
 	    description
 		"The maximum number of list entries to return. The
-           value of the 'count' parameter is either an integer
+           value of the 'limit' parameter is either an integer
            greater than or equal to 1, or the string 'unbounded'.
            The string 'unbounded' is the default value.";
 	}
@@ -119,7 +119,7 @@ module clixon-netconf-list-pagination {
 	    default "none";
 	    description
 		"The first list item to return.
-           the 'skip' parameter is either an integer greater than
+           the 'offset' parameter is either an integer greater than
            or equal to 1, or the string 'unbounded'.  The string
            'unbounded' is the default value.";
 	}