Added option to treat unknown XML (wrt YANG) as anydata.

CHANGELOG.md

@@ -50,6 +50,8 @@ Expected: May 2020
 
 ### Minor changes
 
+* Added option `CLICON_YANG_UNKNOWN_ANYDATA` to treat unknown XML (wrt YANG) as anydata.
+  * This is to be (very) forgiving but you need to accept eg unsynchronized YANG and XML
 * Compile-time option: `USE_CLIGEN44` for running clixon-45 with cligen-44.
   * Temporary fix since cligen-45 have some non-backward compatible behaviour.
 * Optimizations

apps/backend/backend_main.c

@@ -523,7 +523,6 @@ main(int    argc,
 	    usage(h, argv[0]);
 	return -1;
     }
-    
     /* External NACM file? */
     nacm_mode = clicon_option_str(h, "CLICON_NACM_MODE");
     if (nacm_mode && strcmp(nacm_mode, "external") == 0)
@@ -692,6 +691,9 @@ main(int    argc,
 		   clicon_configfile(h));
 	goto done;
     }
+    /* Treat unknown XML as anydata */
+    if (clicon_option_bool(h, "CLICON_YANG_UNKNOWN_ANYDATA") == 1)
+	xml_bind_yang_unknown_anydata(1);
 
     /* Publish stream on pubsub channels.
      * CLICON_STREAM_PUB should be set to URL to where streams are published

apps/cli/cli_main.c

@@ -515,6 +515,10 @@ main(int argc, char **argv)
     if (netconf_module_features(h) < 0)
 	goto done;
 
+    /* Treat unknwon XML as anydata */
+    if (clicon_option_bool(h, "CLICON_YANG_UNKNOWN_ANYDATA") == 1)
+	xml_bind_yang_unknown_anydata(1);
+
     /* Create top-level and store as option */
     if ((yspec = yspec_new()) == NULL)
 	goto done;

apps/restconf/restconf_main.c

@@ -715,6 +715,9 @@ main(int    argc,
     if ((yspec = yspec_new()) == NULL)
 	goto done;
     clicon_dbspec_yang_set(h, yspec);
+    /* Treat unknown XML as anydata */
+    if (clicon_option_bool(h, "CLICON_YANG_UNKNOWN_ANYDATA") == 1)
+	xml_bind_yang_unknown_anydata(1);
     
     /* Load restconf plugins before yangs are loaded (eg extension callbacks) */
     if ((dir = clicon_restconf_dir(h)) != NULL)

lib/clixon/clixon_xml_bind.h

@@ -43,6 +43,7 @@
 /*
  * Prototypes
  */
+int xml_bind_yang_unknown_anydata(int bool);
 int xml_bind_yang_rpc(cxobj *xrpc, yang_stmt *yspec, cxobj **xerr);
 int xml_bind_yang_rpc_reply(cxobj *xrpc, char *name, yang_stmt *yspec, cxobj **xerr);
 int xml_bind_yang0(cxobj *xt, yang_bind yb, yang_stmt *yspec, cxobj **xerr);

lib/src/clixon_options.c

@@ -563,7 +563,9 @@ clicon_option_bool(clicon_handle h,
 	return 0;
     if (strcmp(s,"true")==0)
 	return 1;
-    return 0; /* Hopefully false, but anything else than "true" */
+    if (strcmp(s,"1")==0)
+	return 1;
+    return 0; /* Hopefully false, but anything else than "true" or "one" */
 }
 
 /*! Set option given as bool
@@ -578,8 +580,14 @@ clicon_option_bool_set(clicon_handle h,
 {
     char s[64];
     
-    if (snprintf(s, sizeof(s)-1, "%u", val) < 0)
+    if (val != 0 && val != 1){
+	clicon_err(OE_CFG, EINVAL, "val is %d, 0 or 1 expected", val);
 	return -1;
+    }
+    if (snprintf(s, sizeof(s)-1, "%s", val?"true":"false") < 0){
+	clicon_err(OE_CFG, errno, "snprintf");
+	return -1;
+    }
     return clicon_option_str_set(h, name, s);
 }
 

lib/src/clixon_validate.c

@@ -1091,15 +1091,17 @@ xml_yang_validate_all(clicon_handle h,
        and !Node has a config sub-statement and it is false */
     ys=xml_spec(xt);
     if (ys==NULL){
+	if (clicon_option_bool(h, "CLICON_YANG_UNKNOWN_ANYDATA") == 1)
+	    goto ok;
 	if ((cb = cbuf_new()) == NULL){
 	    clicon_err(OE_UNIX, errno, "cbuf_new");
 	    goto done;
 	}
-	if (xml2ns(xt, xml_prefix(xt), &namespace) < 0)
-	    goto done;
 	cprintf(cb, "Failed to find YANG spec of XML node: %s", xml_name(xt));
 	if ((xp = xml_parent(xt)) != NULL)
 	    cprintf(cb, " with parent: %s", xml_name(xp));
+	if (xml2ns(xt, xml_prefix(xt), &namespace) < 0)
+	    goto done;
 	if (namespace)
 	    cprintf(cb, " in namespace: %s", namespace);
 	if (netconf_unknown_element_xml(xret, "application", xml_name(xt), cbuf_get(cb)) < 0)

lib/src/clixon_xml_bind.c

@@ -79,6 +79,21 @@
 #include "clixon_yang_type.h"
 #include "clixon_xml_bind.h"
 
+/*
+ * Local variables
+ */
+static int _yang_unknown_anydata = 0;
+
+/*! Kludge to equate unknown XML with anydata
+ * The problem with this is that its global and shuld be bound to a handle
+ */
+int
+xml_bind_yang_unknown_anydata(int bool)
+{
+    _yang_unknown_anydata = bool;
+    return 0;
+}
+
 /*! After yang binding, bodies of containers and lists are stripped from XML bodies
  * May apply to other nodes?
  */
@@ -156,6 +171,10 @@ populate_self_parent(cxobj  *xt,
     if (xml2ns(xt, xml_prefix(xt), &ns) < 0)
 	goto done;
     if ((y = yang_find_datanode(yparent, name)) == NULL){
+	if (_yang_unknown_anydata){
+	    retval = 2; /* treat as anydata */
+	    goto done;
+	}
 	if ((cb = cbuf_new()) == NULL){
 	    clicon_err(OE_UNIX, errno, "cbuf_new");
 	    goto done;
@@ -244,6 +263,10 @@ populate_self_top(cxobj     *xt,
     if (xml2ns(xt, xml_prefix(xt), &ns) < 0)
 	goto done;
     if ((y = yang_find_schemanode(ymod, name)) == NULL){ /* also rpc */
+	if (_yang_unknown_anydata){
+	    retval = 2; /* treat as anydata */
+	    goto done;
+	}
 	if ((cb = cbuf_new()) == NULL){
 	    clicon_err(OE_UNIX, errno, "cbuf_new");
 	    goto done;

test/lib.sh

@@ -106,6 +106,9 @@ fi
 # Backend user
 BUSER=clicon
 
+# If set, unknown XML is treated as ANYDATA
+: ${YANG_UNKNOWN_ANYDATA:=false}
+
 # Follow the binary programs that can be parametrized (eg with valgrind)
 
 : ${clixon_cli:=clixon_cli}

test/test_copy_config.sh

@@ -150,7 +150,9 @@ new "copy startup->candidate"
 expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><copy-config><target><candidate/></target><source><startup/></source></copy-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "copy startup->running not allowed"
+if ! $YANG_UNKNOWN_ANYDATA ; then
 expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><copy-config><target><running/></target><source><startup/></source></copy-config></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown-element</error-tag><error-info><bad-element>running</bad-element></error-info><error-severity>error</error-severity><error-message>Failed to find YANG spec of XML node: running with parent: target in namespace: urn:ietf:params:xml:ns:netconf:base:1.0</error-message></rpc-error></rpc-reply>]]>]]>$"
+fi
 
 # Here running is empty
 new "Check running empty"

test/test_identity.sh

@@ -282,8 +282,10 @@ new "restconf delete identity"
 expectpart "$(curl -s -i -X DELETE  http://localhost/restconf/data/example:crypto)" 0 "HTTP/1.1 204 No Content"
 
 # 2. set identity in other module with restconf , read it with restconf and netconf
+if ! $YANG_UNKNOWN_ANYDATA ; then
 new "restconf add POST instead of PUT (should fail)"
 expectpart "$(curl -s -i -X POST -H "Content-Type: application/yang-data+json" http://localhost/restconf/data/example:crypto -d '{"example:crypto":"example-des:des3"}')" 0 'HTTP/1.1 400 Bad Request' '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"bad-element","error-info":{"bad-element":"crypto"},"error-severity":"error","error-message":"Failed to find YANG spec of XML node: crypto with parent: crypto in namespace: urn:example:my-crypto"}}}'
+fi
 
 # Alternative error:
 #'{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"unknown-element","error-info":{"bad-element":"crypto"},"error-severity":"error","error-message":"Leaf contains sub-element"}}}'

test/test_restconf.sh

@@ -237,8 +237,10 @@ new "restconf rpc using POST json"
 expecteq "$(curl -s -X POST -H "Content-Type: application/yang-data+json" -d '{"clixon-example:input":{"x":42}}' http://localhost/restconf/operations/clixon-example:example)" 0 '{"clixon-example:output":{"x":"42","y":"42"}}
 
'
 
+if ! $YANG_UNKNOWN_ANYDATA ; then
 new "restconf rpc using POST json wrong"
 expectpart "$(curl -si -X POST -H "Content-Type: application/yang-data+json" -d '{"clixon-example:input":{"wrongelement":"ipv4"}}' http://localhost/restconf/operations/clixon-example:example)" 0 'HTTP/1.1 400 Bad Request' '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"bad-element","error-info":{"bad-element":"wrongelement"},"error-severity":"error","error-message":"Failed to find YANG spec of XML node: wrongelement with parent: example in namespace: urn:example:clixon"}}}'
+fi
 
 new "restconf rpc non-existing rpc without namespace"
 expectpart "$(curl -is -X POST -H "Content-Type: application/yang-data+json" -d '{}' http://localhost/restconf/operations/kalle)" 0 'HTTP/1.1 400 Bad Request' '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"missing-element","error-info":{"bad-element":"kalle"},"error-severity":"error","error-message":"RPC not defined"}}'

test/test_rpc.sh

@@ -122,8 +122,10 @@ fi
 new "restconf omit mandatory"
 expectpart "$(curl -is -X POST -H "Content-Type: application/yang-data+json" -d '{"clixon-example:input":null}' http://localhost/restconf/operations/clixon-example:example)" 0 'HTTP/1.1 400 Bad Request' '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"missing-element","error-info":{"bad-element":"x"},"error-severity":"error","error-message":"Mandatory variable"}}}
'
 
-new "restconf add extra"
+new "restconf add extra w/o yang: should fail"
+if ! $YANG_UNKNOWN_ANYDATA ; then
 expectpart "$(curl -is -X POST -H "Content-Type: application/yang-data+json" -d '{"clixon-example:input":{"x":"0","extra":"0"}}' http://localhost/restconf/operations/clixon-example:example)" 0 'HTTP/1.1 400 Bad Request' '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"bad-element","error-info":{"bad-element":"extra"},"error-severity":"error","error-message":"Failed to find YANG spec of XML node: extra with parent: example in namespace: urn:example:clixon"}}}'
+fi
 
 new "restconf wrong method"
 expectpart "$(curl -is -X POST -H "Content-Type: application/yang-data+json" -d '{"clixon-example:input":{"x":"0"}}' http://localhost/restconf/operations/clixon-example:wrong)" 0 'HTTP/1.1 400 Bad Request' '{"ietf-restconf:errors":{"error":{"error-type":"application","error-tag":"missing-element","error-info":{"bad-element":"wrong"},"error-severity":"error","error-message":"RPC not defined"}}}
'
@@ -137,8 +139,10 @@ expecteof "$clixon_netconf -qf $cfg" 0 '<rpc xmlns="urn:ietf:params:xml:ns:netco
 new "netconf edit-config ok"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target><candidate/></target><config/></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
+if ! $YANG_UNKNOWN_ANYDATA ; then
 new "netconf edit-config extra arg should fail"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target><candidate/></target><extra/><config/></edit-config></rpc>]]>]]>' "^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>unknown-element</error-tag><error-info><bad-element>extra</bad-element></error-info><error-severity>error</error-severity><error-message>Failed to find YANG spec of XML node: extra with parent: edit-config in namespace: urn:ietf:params:xml:ns:netconf:base:1.0</error-message></rpc-error></rpc-reply>]]>]]>$"
+fi
 
 new "netconf edit-config empty target should fail"
 expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config xmlns="urn:ietf:params:xml:ns:netconf:base:1.0"><target/><config/></edit-config></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>data-missing</error-tag><error-app-tag>missing-choice</error-app-tag><error-info><missing-choice>config-target</missing-choice></error-info><error-severity>error</error-severity></rpc-error></rpc-reply>]]>]]>$'

util/clixon_util_xml.c

@@ -66,7 +66,7 @@
 #include "clixon/clixon.h"
 
 /* Command line options passed to getopt(3) */
-#define UTIL_XML_OPTS "hD:f:Jjl:pvoy:Y:t:T:"
+#define UTIL_XML_OPTS "hD:f:Jjl:pvoy:Y:t:T:u"
 
 static int
 validate_tree(clicon_handle h,
@@ -125,6 +125,7 @@ usage(char *argv0)
     	    "\t-Y <dir> \tYang dirs (can be several)\n"
    	    "\t-t <file>\tXML top input file (where base tree is pasted to)\n"
 	    "\t-T <path>\tXPath to where in top input file base should be pasted\n"
+	    "\t-u \t\tTreat unknown XML as anydata\n"
 	    ,
 	    argv0);
     exit(0);
@@ -221,6 +222,11 @@ main(int    argc,
 	case 'T': /* top file xpath */
 	    top_path = optarg;
 	    break;
+	case 'u':
+	    if (clicon_option_bool_set(h, "CLICON_YANG_UNKNOWN_ANYDATA", 1) < 0)
+		goto done;
+	    xml_bind_yang_unknown_anydata(1);
+	    break;
 	default:
 	    usage(argv[0]);
 	    break;

yang/clixon/clixon-config@2020-02-22.yang

@@ -314,6 +314,14 @@ module clixon-config {
                  Some yang specs seem not to fulfil this. However, if you reset this, there may
                  be follow-up errors due to code that assumes a configuration list has keys";
 	}
+	leaf CLICON_YANG_UNKNOWN_ANYDATA{
+	    type boolean;
+	    default false;
+	    description
+		"Treat unknown XML/JSON nodes as anydata. 
+                 This does not apply to namespaces, which means a top-level node: xxx:yyy
+                 is accepted only if yyy is unknown, not xxx";
+	}
 	leaf CLICON_BACKEND_DIR {
 	    type string;
 	    description