Empty values in JSON has changed to comply to RFC 7951

* empty values of yang type `empty` are encoded as: `{"x":[null]}`
   * empty string values are encoded as: `{"x":""}` (changed from `null` in 4.0 and `[null]` in 4.3)
   * empty containers are encoded as: `{"x":{}}`
   * empty elements in unknown/anydata/anyxml encoded as: `{"x":{}}` (changed from `{"x":null}`)

CHANGELOG.md

@@ -39,6 +39,11 @@ Expected: Early March 2020
 [search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml)
 	
 ### API changes on existing features (you may need to change your code)
+* Empty values in JSON has changed to comply to RFC 7951
+   * empty values of yang type `empty` are encoded as: `{"x":[null]}`
+   * empty string values are encoded as: `{"x":""}` (changed from `null` in 4.0 and `[null]` in 4.3)
+   * empty containers are encoded as: `{"x":{}}`
+   * empty elements in unknown/anydata/anyxml encoded as: `{"x":{}}` (changed from `{"x":null}`)
 * Bugfix of config false statement may cause change of sorting of lists in GET opertions (lists that were sorted should not have been sorted)
 * New clixon-config@2020-02-22.yang revision
   * Search index extension `search_index` for declaring which non-key variables are search indexes

example/hello/Makefile.in

@@ -125,7 +125,7 @@ install: $(YANGSPECS) $(CLISPECS) $(PLUGINS) $(APPNAME).xml
 	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/clispec
 	install -m 0644 $(CLISPECS) $(DESTDIR)$(libdir)/$(APPNAME)/clispec
 	install -d -m 0755 $(DESTDIR)$(datarootdir)/$(APPNAME)/yang
-	install -m 0644 $(YANGSPECS) $(DESTDIR)$(DESTDIR)$(YANG_INSTALLDIR)
+	install -m 0644 $(YANGSPECS) $(DESTDIR)$(YANG_INSTALLDIR)
 	install -d -m 0755 $(DESTDIR)$(localstatedir)/$(APPNAME)
 
 # Uncomment for installing config file in /usr/local/etc instead

lib/src/clixon_json.c

@@ -519,12 +519,12 @@ xml2json_encode_identityref(cxobj     *xb,
  * @param[out]    cb0  Encoded string
  */
 static int
-xml2json_encode(cxobj     *xb,
+xml2json_encode_leafs(cxobj     *xb,
-		cbuf      *cb0)
+		      cxobj     *xp,
+		      yang_stmt *yp,
+		      cbuf      *cb0)
 {
     int           retval = -1;
-    cxobj        *xp;
-    yang_stmt    *yp;
     enum rfc_6020 keyword;
     yang_stmt    *ytype;
     char         *restype;  /* resolved type */
@@ -538,10 +538,9 @@ xml2json_encode(cxobj     *xb,
 	clicon_err(OE_XML, errno, "cbuf_new");
 	goto done;
     }
-    body = xml_value(xb);
+    body = xb?xml_value(xb):NULL;
-    if ((xp = xml_parent(xb)) == NULL ||
+    if (yp == NULL){
-	(yp = xml_spec(xp)) == NULL){
+	cprintf(cb, "%s", body?body:"null"); 
-	cprintf(cb, "%s", body); 
 	goto ok; /* unknown */
     }
     keyword = yang_keyword_get(yp);
@@ -554,7 +553,10 @@ xml2json_encode(cxobj     *xb,
 	cvtype = yang_type2cv(yp);
 	switch (cvtype){ 
 	case CGV_STRING:
-	    if (ytype){
+	case CGV_REST:
+	    if (body==NULL)
+		; /* empty: "" */
+	    else if (ytype){
 		if (strcmp(restype, "identityref")==0){
 		    if (xml2json_encode_identityref(xb, body, yp, cb) < 0)
 			goto done;
@@ -578,8 +580,18 @@ xml2json_encode(cxobj     *xb,
 	    cprintf(cb, "%s", body);
 	    quote = 0;
 	    break;
+	case CGV_VOID:
+	    /* special case YANG empty type */
+	    if (body == NULL && strcmp(restype, "empty")==0){
+		quote = 0;
+		cprintf(cb, "[null]");
+	    }
+	    break;
 	default:
-	    cprintf(cb, "%s", body);
+	    if (body)
+		cprintf(cb, "{}"); /* dont know */
+	    else
+		cprintf(cb, "%s", body);
 	}
 	break;
     default:
@@ -607,6 +619,49 @@ xml2json_encode(cxobj     *xb,
     return retval;
 }
 
+/* X has no XML child - no body.
+ * If x is a container, use {} instead of null 
+ * if leaf or leaf-list then assume EMPTY type, then [null]
+ * else null
+ */
+static int
+nullchild(cbuf      *cb,
+	  cxobj     *x,
+	  yang_stmt *y)
+{
+    int           retval = -1;
+
+    if (y == NULL){
+	/* This is very problematic.
+	 * RFC 7951 explicitly forbids "null" to be used unless for empty types in [null]
+	 */
+	cprintf(cb, "{}");
+    }
+    else{
+	switch (yang_keyword_get(y)){
+	case Y_ANYXML:
+	case Y_ANYDATA:
+	case Y_CONTAINER:
+	    cprintf(cb, "{}");
+	    break;
+	case Y_LEAF:
+	case Y_LEAF_LIST:
+	    if (xml2json_encode_leafs(NULL, x, y, cb) < 0)
+		goto done;
+	    break;
+	default:
+	    /* This is very problematic.
+	     * RFC 7951 explicitly forbids "null" to be used unless for empty types in [null]
+	     */
+	    cprintf(cb, "{}");
+	    break;
+	}
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Do the actual work of translating XML to JSON 
  * @param[out]   cb        Cligen text buffer containing json on exit
  * @param[in]    x         XML tree structure containing XML to translate
@@ -657,6 +712,7 @@ xml2json1_cbuf(cbuf                   *cb,
     int              retval = -1;
     int              i;
     cxobj           *xc;
+    cxobj           *xp;
     enum childtype   childt;
     enum array_element_type xc_arraytype;
     yang_stmt       *ys;
@@ -678,8 +734,9 @@ xml2json1_cbuf(cbuf                   *cb,
 		arraytype2str(arraytype),
 		childtype2str(childt));
     switch(arraytype){
-    case BODY_ARRAY: /* Only place in fn where body is printed */
+    case BODY_ARRAY: /* Only place in fn where body is printed (except nullchild) */
-	if (xml2json_encode(x, cb) < 0)
+	xp = xml_parent(x);
+	if (xml2json_encode_leafs(x, xp, xml_spec(xp), cb) < 0)
 	    goto done;
 	break;
     case NO_ARRAY:
@@ -691,19 +748,8 @@ xml2json1_cbuf(cbuf                   *cb,
 	}
 	switch (childt){
 	case NULL_CHILD:
-	    /* If x is a container, use {} instead of null 
+	    if (nullchild(cb, x, ys) < 0)
-	     * if leaf or leaf-list then assume EMPTY type, then [null]
+		goto done;
-	     * else null
-	     */
-	    if (ys && yang_keyword_get(ys) == Y_CONTAINER)
-		cprintf(cb, "{}");
-	    else{
-		if (ys &&
-		    (yang_keyword_get(ys) == Y_LEAF || yang_keyword_get(ys) == Y_LEAF_LIST))
-		    cprintf(cb, "[null]");
-		else
-		    cprintf(cb, "null");
-	    }
 	    break;
 	case BODY_CHILD:
 	    break;
@@ -726,7 +772,8 @@ xml2json1_cbuf(cbuf                   *cb,
 		pretty?(level*JSON_INDENT):0, "");
 	switch (childt){
 	case NULL_CHILD:
-	    cprintf(cb, "null");
+	    if (nullchild(cb, x, ys) < 0)
+		goto done;
 	    break;
 	case BODY_CHILD:
 	    break;
@@ -744,7 +791,8 @@ xml2json1_cbuf(cbuf                   *cb,
 		pretty?(level*JSON_INDENT):0, "");
 	switch (childt){
 	case NULL_CHILD:
-	    cprintf(cb, "null");
+	    if (nullchild(cb, x, ys) < 0)
+		goto done;
 	    break;
 	case BODY_CHILD:
 	    break;

lib/src/clixon_xml_map.c

@@ -1296,6 +1296,27 @@ populate_self_top(cxobj     *xt,
     goto done;
 }
 
+/*! After yang binding, bodies of containers and lists are stripped from XML bodies
+ * May apply to other nodes?
+ */
+static int
+strip_whitespace(cxobj *xt)
+{
+    yang_stmt    *yt;
+    enum rfc_6020 keyword;
+    cxobj        *xc;
+    
+    if ((yt = xml_spec(xt)) != NULL){
+	keyword = yang_keyword_get(yt);
+	if (keyword == Y_LIST || keyword == Y_CONTAINER){
+	    xc = NULL;
+	    while ((xc = xml_find_type(xt, NULL, "body", CX_BODY)) != NULL)
+		xml_purge(xc);
+	}
+    }
+    return 0;
+}
+
 /*! Find yang spec association of tree of XML nodes
  *
  * Populate xt:s children as top-level symbols
@@ -1329,6 +1350,7 @@ xml_spec_populate(cxobj     *xt,
     int    ret;
     int    failed = 0; /* we continue loop after failure, should we stop at fail?`*/
 
+    strip_whitespace(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	if ((ret = xml_spec_populate0(xc, yspec, xerr)) < 0)
@@ -1354,6 +1376,7 @@ xml_spec_populate_parent(cxobj  *xt,
     int    ret;
     int    failed = 0; /* we continue loop after failure, should we stop at fail?`*/
     
+    strip_whitespace(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	if ((ret = xml_spec_populate0_parent(xc, xerr)) < 0)
@@ -1390,6 +1413,7 @@ xml_spec_populate0(cxobj     *xt,
 	goto done;
     if (ret == 0)
 	goto fail;
+    strip_whitespace(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	if ((ret = xml_spec_populate0_parent(xc, xerr)) < 0)
@@ -1424,6 +1448,7 @@ xml_spec_populate0_parent(cxobj  *xt,
 	goto done;
     if (ret == 0)
 	goto fail;
+    strip_whitespace(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	if ((ret = xml_spec_populate0_parent(xc, xerr)) < 0)

lib/src/clixon_xml_parse.y

@@ -136,6 +136,9 @@ xml_parse_whitespace(clixon_xml_yacc *xy,
     xy->xy_xelement = NULL; /* init */
     /* If there is an element already, only add one whitespace child 
      * otherwise, keep all whitespace. See code in xml_parse_bslash
+     * Note that this xml element is not aware of YANG yet. 
+     * For example, if xp is LEAF then a body child is OK, but if xp is CONTAINER
+     * then the whitespace body is pretty-prints and should be stripped (later)
      */
     for (i=0; i<xml_child_nr(xp); i++){
 	if (xml_type(xml_child_i(xp, i)) == CX_ELMNT)
@@ -397,7 +400,7 @@ endtag      : BSLASH NAME '>'
             ;
 
 elist       : elist content { clicon_debug(2, "elist -> elist content"); }
-            | content      { clicon_debug(2, "elist -> content"); }
+            | content       { clicon_debug(2, "elist -> content"); }
             ;
 
 /* Rule 43 */

test/test_restconf_jukebox.sh

@@ -101,7 +101,7 @@ expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+xml' http://loca
 # This just catches the header and the jukebox module, the RFC has foo and bar which
 # seems wrong to recreate
 new "B.1.2.  Retrieve the Server Module Information"
-expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-yang-library:modules-state)" 0 "HTTP/1.1 200 OK" 'Cache-Control: no-cache' "Content-Type: application/yang-data+json" '{"ietf-yang-library:modules-state":{"module-set-id":"0","module":\[{"name":"example-events","revision":\[null\],"namespace":"urn:example:events","conformance-type":"implement"},{"name":"example-jukebox","revision":"2016-08-15","namespace":"http://example.com/ns/example-jukebox","conformance-type":"implement"},{"name":"example-system","revision":\[null\],"namespace":"http://example.com/ns/example-system","conformance-type":"implement"},{"name":"clixon-lib","revision":"2019-08-13","namespace":"http://clicon.org/lib","conformance-type":"implement"},{"name":"ietf-yang-library","revision":"2016-06-21","namespace":"urn:ietf:params:xml:ns:yang:ietf-yang-library"'
+expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+json' http://localhost/restconf/data/ietf-yang-library:modules-state)" 0 "HTTP/1.1 200 OK" 'Cache-Control: no-cache' "Content-Type: application/yang-data+json" '{"ietf-yang-library:modules-state":{"module-set-id":"0","module":\[{"name":"example-events","revision":"","namespace":"urn:example:events","conformance-type":"implement"},{"name":"example-jukebox","revision":"2016-08-15","namespace":"http://example.com/ns/example-jukebox","conformance-type":"implement"},{"name":"example-system","revision":"","namespace":"http://example.com/ns/example-system","conformance-type":"implement"},{"name":"clixon-lib","revision":"2019-08-13","namespace":"http://clicon.org/lib","conformance-type":"implement"},{"name":"ietf-yang-library","revision":"2016-06-21","namespace":"urn:ietf:params:xml:ns:yang:ietf-yang-library"'
 
 new "B.1.3.  Retrieve the Server Capability Information"
 expectpart "$(curl -si -X GET -H 'Accept: application/yang-data+xml' http://localhost/restconf/data/ietf-restconf-monitoring:restconf-state/capabilities)" 0 "HTTP/1.1 200 OK" "Content-Type: application/yang-data+xml" 'Cache-Control: no-cache' '<capabilities xmlns="urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"><capability>urn:ietf:params:restconf:capability:defaults:1.0?basic-mode=explicit</capability><capability>urn:ietf:params:restconf:capability:depth</capability>

test/test_xml.sh

@@ -11,8 +11,9 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 new "xml parse"
 expecteof "$clixon_util_xml -o" 0 "<a><b/></a>" "^<a><b/></a>$"
 
+# Note dont know what b is.
 new "xml parse to json"
-expecteof "$clixon_util_xml -oj" 0 "<a><b/></a>" '{"a":{"b":null}}'
+expecteof "$clixon_util_xml -oj" 0 "<a><b/></a>" '{"a":{"b":{}}}'
 
 new "xml parse strange names"
 expecteof "$clixon_util_xml -o" 0 "<_-><b0.><c-.-._/></b0.></_->" "<_-><b0.><c-.-._/></b0.></_->"