* JSON parse and print improvements

* Integrated parsing with namespace translation and yang spec lookup

README.md

@@ -30,6 +30,7 @@ support.
   * [Scaling: large lists](doc/scaling/large-lists.md)
   * [Containers](docker/README.md)
   * [Roadmap](doc/ROADMAP.md)
+  * [Standard compliance](#standard-compliance)
   * [Reference manual](#reference) 
   
 ## Background
@@ -270,6 +271,24 @@ The functionality is as follows (references to sections in [RFC8341](https://too
 
 The figure shows the SDK runtime of Clixon.
 
+## Standard Compliance
+
+This is work-in-progress on which standards Clixon supports:
+- [RFC 6020](https://www.rfc-editor.org/rfc/rfc6020.txt) YANG - A Data Modeling Language for the Network Configuration Protocol (NETCONF)
+- [RFC7895](http://www.rfc-base.org/txt/rfc-7895.txt) YANG Module Library
+* [RFC7950](http://www.rfc-base.org/txt/rfc-7950.txt) The YANG 1.1 Data Modeling Language
+* [RFC7951](http://www.rfc-base.org/txt/rfc-7951.txt) JSON Encoding of Data Modeled with YANG
+- [RFC 6241: NETCONF Configuration Protocol](http://www.rfc-base.org/txt/rfc-6241.txt)
+- [RFC 6242: Using the NETCONF Configuration Protocol over Secure Shell (SSH)](http://www.rfc-base.org/txt/rfc-6242.txt)
+- [RFC 5277: NETCONF Event Notifications](http://www.rfc-base.org/txt/rfc-5277.txt)
+- [RFC 8341: Network Configuration Access Control Model](http://www.rfc-base.org/txt/rfc-8341.txt)
+- [RFC 8040: RESTCONF Protocol](https://tools.ietf.org/html/rfc8040).
+- [RFC8341(NACM)](https://tools.ietf.org/html/rfc8341).
+- [XML 1.0](https://www.w3.org/TR/2008/REC-xml-20081126)
+- [Namespaces in XML 1.0](https://www.w3.org/TR/2009/REC-xml-names-20091208)
+- [XPATH 1.0](https://www.w3.org/TR/xpath-10)
+- [W3C XML XSD](http://www.w3.org/TR/2004/REC-xmlschema-2-20041028)
+
 ## Reference
 
 Clixon uses [Doxygen](http://www.doxygen.nl/index.html) for reference documentation.

apps/restconf/restconf_methods.c

@@ -498,7 +498,8 @@ api_data_post(clicon_handle h,
 	    goto ok;
 	}
     }
-    else if (json_parse_str(data, &xdata) < 0){
+    else {
+	if ((ret = json_parse_str(data, yspec, &xdata, &xerr)) < 0){ 
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
@@ -509,6 +510,16 @@ api_data_post(clicon_handle h,
 		goto done;
 	    goto ok;
 	}
+	if (ret == 0){
+	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
+		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
+		goto done;
+	    }
+	    if (api_return_err(h, r, xe, pretty, use_xml) < 0)
+		goto done;
+	    goto ok;
+	}
+    }
     /* 4.4.1: The message-body MUST contain exactly one instance of the
      * expected data resource. 
      */
@@ -533,19 +544,6 @@ api_data_post(clicon_handle h,
     /* Replace xbot with x, ie bottom of api-path with data */
     if (xml_addsub(xbot, x) < 0)
 	goto done;
-    if (!parse_xml){ /* If JSON, translate namespace from module:name to xmlns=uri */
-	if (json2xml_ns(yspec, x, &xerr) < 0)
-	    goto done;
-	if (xerr){
-	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
-		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
-		goto done;
-	    }
-	    if (api_return_err(h, r, xe, pretty, use_xml) < 0)
-		goto done;
-	    goto ok;
-	}
-    }
     /* Create text buffer for transfer to backend */
     if ((cbx = cbuf_new()) == NULL)
 	goto done;
@@ -805,7 +803,7 @@ api_data_put(clicon_handle h,
 	}
     }
     else{
-	if (json_parse_str(data, &xdata) < 0){
+	if ((ret = json_parse_str(data, yspec, &xdata, &xerr)) < 0){
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
@@ -816,6 +814,15 @@ api_data_put(clicon_handle h,
 		goto done;
 	    goto ok;
 	}
+	if (ret == 0){
+	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
+		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
+		goto done;
+	    }
+	    if (api_return_err(h, r, xe, pretty, use_xml) < 0)
+		goto done;
+	    goto ok;
+	}
     }
     /* The message-body MUST contain exactly one instance of the
      * expected data resource. 
@@ -832,20 +839,6 @@ api_data_put(clicon_handle h,
 	goto ok;
     }
     x = xml_child_i(xdata,0);
-    if (!parse_xml){ /* If JSON, translate namespace from module:name to xmlns=uri */
-	if (json2xml_ns(yspec, x, &xerr) < 0)
-	    goto done;
-	if (xerr){
-	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
-		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
-		goto done;
-	    }
-	    if (api_return_err(h, r, xe, pretty, use_xml) < 0)
-		goto done;
-	    goto ok;
-	}
-    }
-    
     /* Add operation (create/replace) as attribute */
     if ((xa = xml_new("operation", x, NULL)) == NULL)
 	goto done;
@@ -1329,6 +1322,7 @@ api_operations_post_input(clicon_handle h,
     cxobj     *xinput;
     cxobj     *x;
     cbuf      *cbret = NULL;
+    int        ret;
 
     clicon_debug(1, "%s %s", __FUNCTION__, data);
     if ((cbret = cbuf_new()) == NULL){
@@ -1350,7 +1344,7 @@ api_operations_post_input(clicon_handle h,
 	}
     }
     else { /* JSON */
-	if (json_parse_str(data, &xdata) < 0){
+	if ((ret = json_parse_str(data, yspec, &xdata, &xerr)) < 0){
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
@@ -1361,12 +1355,7 @@ api_operations_post_input(clicon_handle h,
 		goto done;
 	    goto fail;
 	}
-	/* Special case for JSON: It looks like: <top><module:input>
+	if (ret == 0){
-	 * Need to translate to <top><input xmlns="">
-	 */
-	if (json2xml_ns(yspec, xdata, &xerr) < 0)
-	    goto done;
-	if (xerr){
 	    if ((xe = xpath_first(xerr, "rpc-error")) == NULL){
 		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
 		goto done;

lib/clixon/clixon_json.h

@@ -42,9 +42,9 @@
 int xml2json_cbuf(cbuf *cb, cxobj *x, int pretty);
 int xml2json_cbuf_vec(cbuf *cb, cxobj **vec, size_t veclen, int pretty);
 int xml2json(FILE *f, cxobj *x, int pretty);
+int json_print(FILE *f, cxobj *x);
 int xml2json_vec(FILE *f, cxobj **vec, size_t veclen, int pretty);
-int json2xml_ns(yang_stmt *yspec, cxobj *x, cxobj **xerr);
+int json_parse_str(char *str, yang_stmt *yspec, cxobj **xt, cxobj **xret);
-int json_parse_str(char *str, cxobj **xt);
+int json_parse_file(int fd, yang_stmt *yspec, cxobj **xt, cxobj **xret);
-int json_parse_file(int fd, yang_stmt *yspec, cxobj **xt);
 
 #endif /* _CLIXON_JSON_H */

lib/clixon/clixon_yang.h

@@ -186,6 +186,7 @@ int        yang_print_cbuf(cbuf *cb, yang_stmt *yn, int marginal);
 int        if_feature(yang_stmt *yspec, char *module, char *feature);
 int        ys_populate(yang_stmt *ys, void *arg);
 yang_stmt *yang_parse_file(int fd, const char *name, yang_stmt *ysp);
+yang_stmt *yang_parse_filename(const char *filename, yang_stmt  *ysp);
 int        yang_apply(yang_stmt *yn, enum rfc_6020 key, yang_applyfn_t fn, 
 		      void *arg);
 int        yang_datanode(yang_stmt *ys);

lib/clixon/clixon_yang_type.h

@@ -71,6 +71,7 @@ int        yang_type_resolve(yang_stmt *yorig, yang_stmt *ys,
 			     yang_stmt **restype, int *options, 
 			     cvec **cvv, cvec *patterns, cvec *regexps,
 			     uint8_t *fraction);
+enum cv_type yang_type2cv(yang_stmt *ys);
 
 
 #endif  /* _CLIXON_YANG_TYPE_H_ */

lib/src/clixon_datastore_read.c

@@ -316,6 +316,7 @@ xmldb_readfile(clicon_handle      h,
     char  *dbfile = NULL;
     int    fd = -1;
     char  *format;
+    int    ret;
     
     if (xmldb_db2file(h, db, &dbfile) < 0)
 	goto done;
@@ -338,7 +339,7 @@ xmldb_readfile(clicon_handle      h,
 	    goto done;
 	}    
 	if (strcmp(format, "json")==0){
-	    if ((json_parse_file(fd, yspec, &x0)) < 0)
+	    if ((ret = json_parse_file(fd, yspec, &x0, NULL)) < 0) /* XXX: ret == 0*/
 		goto done;
 	}
 	else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)

lib/src/clixon_json.c

@@ -59,7 +59,11 @@
 #include "clixon_hash.h"
 #include "clixon_handle.h"
 #include "clixon_yang.h"
+#include "clixon_yang_type.h"
+#include "clixon_options.h"
 #include "clixon_xml.h"
+#include "clixon_xml_sort.h"
+#include "clixon_xml_map.h"
 #include "clixon_netconf_lib.h"
 #include "clixon_json.h"
 #include "clixon_json_parse.h"
@@ -265,6 +269,44 @@ json_str_escape_cdata(cbuf *cb,
     return retval;
 }
 
+/*! If set, quoute the json value with double quotes 
+ * @þaram[in]  xb  XML body object
+ @ @retval     0   Value should not be quouted, XML value is int, boolean,..
+ @ @retval     1   Value should be quouted, XML value is string,..
+ */
+static int
+jsonvaluestr(cxobj *xb)
+{
+    int            retval = 1;
+    cxobj         *xp;
+    yang_stmt     *yp;
+    enum rfc_6020  keyword;
+
+    if ((xp = xml_parent(xb)) == NULL ||
+	(yp = xml_spec(xp)) == NULL)
+	goto done; /* unknown */
+    keyword = yang_keyword_get(yp);
+    if ((keyword == Y_LEAF || keyword == Y_LEAF_LIST))
+	switch (yang_type2cv(yp)){
+	case CGV_INT8:
+	case CGV_INT16:
+	case CGV_INT32:
+	case CGV_INT64:
+	case CGV_UINT8:
+	case CGV_UINT16:
+	case CGV_UINT32:
+	case CGV_UINT64:
+	case CGV_DEC64:
+	case CGV_BOOL:
+	    retval = 0;
+	    break;
+	default:
+	    break;
+	}
+ done:
+    return retval;
+}
+
 /*! Do the actual work of translating XML to JSON 
  * @param[out]   cb        Cligen text buffer containing json on exit
  * @param[in]    x         XML tree structure containing XML to translate
@@ -309,7 +351,7 @@ xml2json1_cbuf(cbuf                   *cb,
 	       int                     level,
 	       int                     pretty,
 	       int                     flat,
-	       int                     bodystr)
+	       char                   *modname0)
 {
     int              retval = -1;
     int              i;
@@ -318,27 +360,16 @@ xml2json1_cbuf(cbuf                   *cb,
     enum array_element_type xc_arraytype;
     yang_stmt       *ys;
     yang_stmt       *ymod; /* yang module */
-    yang_stmt       *yspec = NULL; /* yang spec */
-    int              bodystr0=1;
-    char            *prefix=NULL;    /* prefix / local namespace name */
-    char            *namespace=NULL; /* namespace uri */
-    char            *modname=NULL;   /* Module name */
     int              commas;
+    char            *modname = NULL;
 
-    /* If x is labelled with a default namespace, it should be translated
+    if ((ys = xml_spec(x)) != NULL){
-     * to a module name. 
+	ymod = ys_real_module(ys);
-     * Harder if x has a prefix, then that should also be translated to associated
-     * module name
-     */
-    prefix = xml_prefix(x);
-    namespace = xml_find_type_value(x, prefix, "xmlns", CX_ATTR);
-
-    if ((ys = xml_spec(x)) != NULL) /* yang spec associated with x */
-	yspec = ys_spec(ys);
-    /* Find module name associated with namspace URI */
-    if (namespace && yspec &&
-	(ymod = yang_find_module_by_namespace(yspec, namespace)) != NULL){
 	modname = yang_argument_get(ymod);
+	if (modname0 && strcmp(modname, modname0) == 0)
+	    modname=NULL;
+	else
+	    modname0 = modname; /* modname0 is ancestor ns passed to child */
     }
     childt = child_type(x);
     if (pretty==2)
@@ -347,21 +378,20 @@ xml2json1_cbuf(cbuf                   *cb,
 		childtype2str(childt));
     switch(arraytype){
     case BODY_ARRAY:{
-	if (bodystr){
+	if (jsonvaluestr(x)) { /* Only print quotation if string-type */
-	    /* XXX String if right type */
 	    cprintf(cb, "\""); 	    
 	    if (json_str_escape_cdata(cb, xml_value(x)) < 0)
 		goto done;
 	    cprintf(cb, "\"");
 	}
-	else
+	else /* No quotation marks */
 	    cprintf(cb, "%s", xml_value(x));
 	break;
     }
     case NO_ARRAY:
 	if (!flat){
 	    cprintf(cb, "%*s\"", pretty?(level*JSON_INDENT):0, "");
-	    if (modname) /* XXX should remove this? */
+	    if (modname) 
 		cprintf(cb, "%s:", modname);
 	    cprintf(cb, "%s\": ", xml_name(x));
 	}
@@ -426,26 +456,6 @@ xml2json1_cbuf(cbuf                   *cb,
      * arraytype=* but child-type is BODY_CHILD 
      * This is code for writing <a>42</a> as "a":42 and not "a":"42"
      */
-    if (childt == BODY_CHILD && ys!=NULL &&
-	(yang_keyword_get(ys) == Y_LEAF || yang_keyword_get(ys) == Y_LEAF_LIST))
-	switch (cv_type_get(yang_cv_get(ys))){
-	case CGV_INT8:
-	case CGV_INT16:
-	case CGV_INT32:
-	case CGV_INT64:
-	case CGV_UINT8:
-	case CGV_UINT16:
-	case CGV_UINT32:
-	case CGV_UINT64:
-	case CGV_DEC64:
-	case CGV_BOOL:
-	    bodystr0 = 0;
-	    break;
-	default:
-	    bodystr0 = 1;
-	    break;
-	}
-
     commas = xml_child_nr_notype(x, CX_ATTR) - 1;
     for (i=0; i<xml_child_nr(x); i++){
 	xc = xml_child_i(x, i);
@@ -457,7 +467,7 @@ xml2json1_cbuf(cbuf                   *cb,
 	if (xml2json1_cbuf(cb, 
 			   xc, 
 			   xc_arraytype,
-			   level+1, pretty, 0, bodystr0) < 0)
+			   level+1, pretty, 0, modname0) < 0)
 	    goto done;
 	if (commas > 0) {
 	    cprintf(cb, ",%s", pretty?"\n":"");
@@ -527,6 +537,9 @@ xml2json1_cbuf(cbuf                   *cb,
 }
 
 /*! Translate an XML tree to JSON in a CLIgen buffer
+ *
+ * XML-style namespace notation in tree, but RFC7951 in output assume yang 
+ * populated 
  *
  * @param[in,out] cb     Cligen buffer to write to
  * @param[in]     x      XML tree to translate from
@@ -551,28 +564,18 @@ xml2json_cbuf(cbuf      *cb,
 {
     int    retval = 1;
     int    level = 0;
-    char  *prefix;
-    char  *namespace;
 
     cprintf(cb, "%*s{%s", 
 	    pretty?level*JSON_INDENT:0,"", 
 	    pretty?"\n":"");
-    /* If x is labelled with a default namespace, it should be translated
-     * to a module name. 
-     * Harder if x has a prefix, then that should also be translated to associated
-     * module name
-     */
-    prefix = xml_prefix(x);
-    if (xml2ns(x, prefix, &namespace) < 0)
-	goto done;
-    /* Some complexities in grafting namespace in existing trees to new */
-    if (xml_find_type_value(x, prefix, "xmlns", CX_ATTR) == NULL && namespace)
-	if (xmlns_set(x, prefix, namespace) < 0)
-		goto done;
     if (xml2json1_cbuf(cb, 
 		       x, 
 		       NO_ARRAY,
-		       level+1, pretty,0,1) < 0)
+		       level+1,
+		       pretty,
+		       0,
+		       NULL /* ancestor modname / namespace */
+		       ) < 0)
 	goto done;
     cprintf(cb, "%s%*s}%s", 
 	    pretty?"\n":"",
@@ -605,24 +608,16 @@ xml2json_cbuf_vec(cbuf      *cb,
 {
     int    retval = -1;
     int    level = 0;
-    int    i;
     cxobj *xp = NULL;
+    int    i;
     cxobj *xc;
-    char  *prefix;
-    char  *namespace;
 
     if ((xp = xml_new("xml2json", NULL, NULL)) == NULL)
 	goto done;
     /* Some complexities in grafting namespace in existing trees to new */
     for (i=0; i<veclen; i++){
-	prefix = xml_prefix(vec[i]);
-	if (xml2ns(vec[i], prefix, &namespace) < 0)
-	    goto done;
 	xc = xml_dup(vec[i]);
 	xml_addsub(xp, xc);
-	if (xml_find_type_value(xc, prefix, "xmlns", CX_ATTR) == NULL && namespace)
-	    if (xmlns_set(xc, prefix, namespace) < 0)
-		goto done;
     }
     if (0){
 	cprintf(cb, "[%s", pretty?"\n":" ");
@@ -631,7 +626,8 @@ xml2json_cbuf_vec(cbuf      *cb,
     if (xml2json1_cbuf(cb, 
 		       xp, 
 		       NO_ARRAY,
-		       level+1, pretty,1, 1) < 0)
+		       level+1, pretty,
+		       1, NULL) < 0)
 	goto done;
 
     if (0){
@@ -683,6 +679,18 @@ xml2json(FILE      *f,
     return retval;
 }
 
+/*! Print an XML tree structure to an output stream as JSON
+ *
+ * @param[in]   f           UNIX output stream
+ * @param[in]   xn          clicon xml tree
+ */
+int
+json_print(FILE  *f, 
+	   cxobj *xn)
+{
+    return xml2json(f, xn, 1);
+}
+
 /*! Translate a vector of xml objects to JSON File.
  * This is done by adding a top pseudo-object, and add the vector as subs,
  * and then not pritning the top pseudo-.object using the 'flat' option.
@@ -719,16 +727,21 @@ xml2json_vec(FILE      *f,
     return retval;
 }
 
-/*! Translate from JSON module:name to XML name xmlns="uri" recursively
+/*! Translate from JSON module:name to XML default ns: xmlns="uri" recursively
+ * Assume an xml tree where prefix:name have been split into "module":"name"
+ * In other words, from JSON RFC7951 to XML namespace trees
+ * 
  * @param[in]     yspec Yang spec
  * @param[in,out] x     XML tree. Translate it in-line
- * @param[out]    xerr  If namespace not set, create xml error tree
+ * @param[out]    xerr  Reason for invalid tree returned as netconf err msg or NULL
- * @retval        0     OK (if xerr set see above)
+ * @retval        1     OK 
+ * @retval        0     Invalid, wrt namespace.  xerr set
  * @retval       -1     Error
  * @note the opposite - xml2ns is made inline in xml2json1_cbuf
+ * Example: <top><module:input> --> <top><input xmlns="">
  */
 int
-json2xml_ns(yang_stmt *yspec,
+json_xmlns_translate(yang_stmt *yspec,
 		     cxobj     *x,
 		     cxobj    **xerr)
 {
@@ -736,63 +749,77 @@ json2xml_ns(yang_stmt *yspec,
     yang_stmt *ymod;
     char      *namespace0;
     char      *namespace;
-    char      *name = NULL;
     char      *prefix = NULL;
     cxobj     *xc;
+    int        ret;
     
-    if (nodeid_split(xml_name(x), &prefix, &name) < 0)
+    prefix = xml_prefix(x); /* prefix is here module name */
-	goto done;
     if (prefix != NULL){
 	if ((ymod = yang_find_module_by_name(yspec, prefix)) == NULL){
-	    if (netconf_unknown_namespace_xml(xerr, "application",
+	    if (xerr &&
+		netconf_unknown_namespace_xml(xerr, "application",
 					      prefix,
 					      "No yang module found corresponding to prefix") < 0)
 		goto done;
-	    goto ok;
+	    goto fail;
 	}
 	namespace = yang_find_mynamespace(ymod);
 	/* Get existing default namespace in tree */
 	if (xml2ns(x, NULL, &namespace0) < 0)
 	    goto done;
 	/* Set xmlns="" default namespace attribute (if diff from default) */
-	if (namespace0==NULL || strcmp(namespace0, namespace))
+	if (namespace0==NULL || strcmp(namespace0, namespace)){
 	    if (xmlns_set(x, NULL, namespace) < 0)
 		goto done;
-	/* Remove prefix from name */
+	    /* and remove prefix */
-	if (xml_name_set(x, name) < 0)
+	    xml_prefix_set(x, NULL);
-	    goto done;
+	}
     }
     xc = NULL;
     while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL){
-	if (json2xml_ns(yspec, xc, xerr) < 0)
+	if ((ret = json_xmlns_translate(yspec, xc, xerr)) < 0)
 	    goto done;
-	if (*xerr != NULL)
+	if (ret == 0)
-	    break;
+	    goto fail;
     }
- ok:
+    retval = 1;
-    retval = 0;
  done:
-    if (prefix)
-	free(prefix);
-    if (name)
-	free(name);
     return retval;
+ fail:
+    retval = 0;
+    goto done;
 }
  
 /*! Parse a string containing JSON and return an XML tree
+ *
+ * Parsing using yacc according to JSON syntax. Names with <prefix>:<id>
+ * are split and interpreted as in RFC7951
+ *
  * @param[in]  str    Input string containing JSON
+ * @param[in]  yspec  If set, also do yang validation
  * @param[in]  name   Log string, typically filename
  * @param[out] xt     XML top of tree typically w/o children on entry (but created)
+ * @param[out] xerr  Reason for invalid returned as netconf err msg 
+ * 
+ * @see _xml_parse  for XML variant
+ * @retval        1     OK and valid
+ * @retval        0     Invalid (only if yang spec)
+ * @retval       -1     Error with clicon_err called
+ * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+ * @see RFC 7951
  */
 static int 
 json_parse(char       *str, 
+	    yang_stmt  *yspec,
 	    const char *name, 
-	   cxobj      *xt)
+	    cxobj      *xt,
+	    cxobj     **xerr)
 {
     int                         retval = -1;
     struct clicon_json_yacc_arg jy = {0,};
+    int                          ret;
 
-    //    clicon_debug(1, "%s", __FUNCTION__);
+    clicon_debug(1, "%s", __FUNCTION__);
     jy.jy_parse_string = str;
     jy.jy_name = name;
     jy.jy_linenum = 1;
@@ -807,20 +834,35 @@ json_parse(char       *str,
 	    clicon_err(OE_XML, 0, "JSON parser error with no error code (should not happen)");
 	goto done;
     }
-    retval = 0;
+    if (yspec){
+	/* Names are split into name/prefix, but now add namespace info */
+	if ((ret = json_xmlns_translate(yspec, xt, xerr)) < 0)
+	    goto done;
+	if (ret == 0)
+	    goto fail;
+	/* Populate, ie associate xml nodes with yang specs  */
+	if (xml_apply0(xt, CX_ELMNT, xml_spec_populate, yspec) < 0)
+	    goto done;
+	if (xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0)
+	    goto done;
+    }
+    retval = 1;
  done:
-    //    clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     json_parse_exit(&jy);
     json_scan_exit(&jy);
+    clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     return retval; 
+ fail: /* invalid */
+    retval = 0;
+    goto done;
 }
 
 /*! Parse string containing JSON and return an XML tree
  *
  * @param[in]     str   String containing JSON
- * @param[out] xt    On success a top of XML parse tree is created with name 'top'
+ * @param[in]     yspec Yang specification, or NULL
- * @retval  0  OK
+ * @param[in,out] xt    On success a top of XML parse tree is created with name 'top'
- * @retval -1  Error with clicon_err called. Includes parse errors
+ * @param[out]    xerr  Reason for invalid returned as netconf err msg 
  *
  * @code
  *  cxobj *cx = NULL;
@@ -829,39 +871,64 @@ json_parse(char       *str,
  *  xml_free(cx);
  * @endcode
  * @note  you need to free the xml parse tree after use, using xml_free()
+ * @see json_parse_file
+ * @retval        1     OK and valid
+ * @retval        0     Invalid (only if yang spec) w xerr set
+ * @retval       -1     Error with clicon_err called
+ * @see json_parse_file with a file descriptor (and more description)
  */
 int 
 json_parse_str(char      *str, 
-	       cxobj **xt)
+	       yang_stmt *yspec,
+	       cxobj    **xt,
+	       cxobj    **xerr)
 {
+    clicon_debug(1, "%s", __FUNCTION__);
     if (*xt == NULL)
 	if ((*xt = xml_new("top", NULL, NULL)) == NULL)
 	    return -1;
-    return json_parse(str, "", *xt);
+    return json_parse(str, yspec, "", *xt, xerr);
 }
 
 /*! Read a JSON definition from file and parse it into a parse-tree. 
  *
- * @param[in]  fd  A file descriptor containing the JSON file (as ASCII characters)
+ * File will be parsed as follows:
- * @param[in]  yspec   Yang specification, or NULL XXX Not yet used
+ *   (1) parsed according to JSON; # Only this check if yspec is NULL
+ *   (2) sanity checked wrt yang  
+ *   (3) namespaces check (using <ns>:<name> notation
+ *   (4) an xml parse tree will be returned
+ * Note, only (1) and (4) will be done if yspec is NULL.
+ * Part of (3) is to split json names if they contain colon, 
+ *   eg: name="a:b" -> prefix="a", name="b"
+ * But this is not done if yspec=NULL, and is not part of the JSON spec
+ * 
+ * @param[in]     fd    File descriptor to the JSON file (ASCII string)
+ * @param[in]     yspec Yang specification, or NULL
  * @param[in,out] xt    Pointer to (XML) parse tree. If empty, create.
- * @retval        0  OK
+ * @param[out]    xerr  Reason for invalid returned as netconf err msg 
- * @retval       -1  Error with clicon_err called
  *
  * @code
  *  cxobj *xt = NULL;
- *  if (json_parse_file(0, NULL, &xt) < 0)
+ *  if (json_parse_file(0, yspec, &xt) < 0)
  *    err;
  *  xml_free(xt);
  * @endcode
  * @note  you need to free the xml parse tree after use, using xml_free()
  * @note, If xt empty, a top-level symbol will be added so that <tree../> will be:  <top><tree.../></tree></top>
  * @note May block on file I/O
+ *
+ * @retval        1     OK and valid
+ * @retval        0     Invalid (only if yang spec) w xerr set
+ * @retval       -1     Error with clicon_err called
+ *
+ * @see json_parse_str
+ * @see RFC7951
  */
 int
 json_parse_file(int        fd,
 		yang_stmt *yspec,
-		cxobj    **xt)
+		cxobj    **xt,
+		cxobj    **xerr)
 {
     int   retval = -1;
     int   ret;
@@ -889,8 +956,12 @@ json_parse_file(int        fd,
 	    if (*xt == NULL)
 		if ((*xt = xml_new(JSON_TOP_SYMBOL, NULL, NULL)) == NULL)
 		    goto done;
-	    if (len && json_parse(ptr, "", *xt) < 0)
+	    if (len){
+		if ((ret = json_parse(ptr, yspec, "", *xt, xerr)) < 0)
 		    goto done;
+		if (ret == 0)
+		    goto fail;
+	    }
 	    break;
 	}
 	if (len>=jsonbuflen-1){ /* Space: one for the null character */
@@ -904,7 +975,7 @@ json_parse_file(int        fd,
 	    ptr = jsonbuf;
 	}
     }
-    retval = 0;
+    retval = 1;
  done:
     if (retval < 0 && *xt){
 	free(*xt);
@@ -913,6 +984,9 @@ json_parse_file(int        fd,
     if (jsonbuf)
 	free(jsonbuf);
     return retval;    
+ fail:
+    retval = 0;
+    goto done;
 }
 
 

lib/src/clixon_json_parse.y

@@ -33,6 +33,7 @@
 
  * JSON Parser
  * From http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf
+ * And RFC7951 JSON Encoding of Data Modeled with YANG
 
 Structural tokens: 
  [   left square bracket 
@@ -72,7 +73,6 @@ object.
 
  */
 
-
 %start json
 
 %union {
@@ -125,6 +125,7 @@ object.
 #include "clixon_err.h"
 #include "clixon_log.h"
 #include "clixon_queue.h"
+#include "clixon_string.h"
 #include "clixon_hash.h"
 #include "clixon_handle.h"
 #include "clixon_yang.h"
@@ -157,26 +158,39 @@ json_parse_init(struct clicon_json_yacc_arg *jy)
     return 0;
 }
 
-
 int
 json_parse_exit(struct clicon_json_yacc_arg *jy)
 {
     return 0;
 }
  
+/*! Create xml object from json object name (eg "string") 
+ *  Split name into prefix:name (extended JSON RFC7951)
+ */
 static int
 json_current_new(struct clicon_json_yacc_arg *jy,
 		 char                        *name)
 {
     int        retval = -1;
-    cxobj *xn;
+    cxobj     *x;
+    char      *prefix = NULL;
+    char      *id = NULL;
 
     clicon_debug(2, "%s", __FUNCTION__);
-    if ((xn = xml_new(name, jy->jy_current, NULL)) == NULL)
+    /* Find colon separator and if found split into prefix:name */
+    if (nodeid_split(name, &prefix, &id) < 0)
 	goto done;
-    jy->jy_current = xn;
+    if ((x = xml_new(id, jy->jy_current, NULL)) == NULL)
+	goto done; 
+    if (prefix && xml_prefix_set(x, prefix) < 0)
+	goto done;
+    jy->jy_current = x;
     retval = 0;
  done:
+    if (prefix)
+	free(prefix);
+    if (id)
+	free(id);
     return retval;
 }
 

lib/src/clixon_xml.c

@@ -238,7 +238,8 @@ xml_prefix_set(cxobj *xn,
  * @retval     0          OK
  * @retval    -1          Error
  * @see xmlns_check XXX can these be merged?
- * @note, this function uses a cache. Any case where cache should be cleared?
+ * @see xml2ns_set cache is set
+ * @note, this function uses a cache. 
  */
 int
 xml2ns(cxobj *x,
@@ -291,7 +292,7 @@ xml2ns(cxobj *x,
 int
 xmlns_set(cxobj *x,
 	  char  *prefix,
-	  char  *namespace)
+	  char  *ns)
 {
     int    retval = -1;
     cxobj *xa;
@@ -307,8 +308,15 @@ xmlns_set(cxobj *x,
 	    goto done;
 	xml_type_set(xa, CX_ATTR);
     }
-    if (xml_value_set(xa, namespace) < 0)
+    if (xml_value_set(xa, ns) < 0)
 	goto done;
+    /* (re)set namespace cache (as used in xml2ns) */
+    if (x->x_ns_cache)
+	free(x->x_ns_cache);
+    if ((x->x_ns_cache = strdup(ns)) == NULL){
+	clicon_err(OE_XML, errno, "strdup");
+	goto done;
+    }
     retval = 0;
  done:
     return retval;
@@ -868,6 +876,9 @@ xml_addsub(cxobj *xp,
 {
     cxobj *oldp;
     int    i;
+    char  *pns = NULL; /* parent namespace */
+    char  *cns = NULL; /* child namespace */
+    cxobj *xa;
 
     if ((oldp = xml_parent(xc)) != NULL){
 	/* Find child order i in old parent*/
@@ -884,6 +895,18 @@ xml_addsub(cxobj *xp,
 	    return -1;
 	/* Set new parent in child */
 	xml_parent_set(xc, xp);
+	/* Ensure default namespace is not duplicated 
+	 * here only remove duplicate default namespace, there may be more */
+	/* 1. Get parent default namespace */
+	xml2ns(xp, NULL, &pns);
+	/* 2. Get child default namespace */
+	if (pns &&
+	    (xa = xml_find_type(xc, NULL, "xmlns", CX_ATTR)) != NULL &&
+	    (cns = xml_value(xa)) != NULL){
+	    /* 3. check if same, if so remove child's */
+	    if (strcmp(pns, cns) == 0)
+		xml_purge(xa); 
+	}
     }
     return 0;
 }
@@ -1704,8 +1727,10 @@ _xml_parse(const char  *str,
     	goto done;
     /* Sort the complete tree after parsing */
     if (yspec){
+	/* Populate, ie associate xml nodes with yang specs */
 	if (xml_apply0(xt, CX_ELMNT, xml_spec_populate, yspec) < 0)
 	    goto done;
+	/* Sort according to yang */
 	if (xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0)
 	    goto done;
     }

lib/src/clixon_xml_map.c

@@ -3022,70 +3022,5 @@ done:
 }
 
 
-/*
- * Turn this on for uni-test programs
- * Usage: clixon_string join
- * Example compile:
- gcc -g -o clixon_xml_map -I. -I../clixon ./clixon_xml_map.c -lclixon -lcligen
- * Example run:
-/interfaces/interface=%s/name --> interfaces/interface/name
-/interfaces/interface=%s/ipv4/address=%s e --> /interfaces/interface=e/ipv4/address
-/interfaces/interface=%s,%s/ipv4/address=%s e f --> /interfaces/interface=e,f/ipv4/address
-/interfaces/interface=%s/ipv4/address=%s,%s e f --> /interfaces/interface=e/ipv4/address=f
-
-/interfaces/interface=%s/ipv4/address=%s/prefix-length eth 1.2.3.4 -->
-/interfaces/interface=eth/ipv4/address=1.2.3.4/prefix-length
-
-*/
-#if 0 /* Test program */
-
-static int
-usage(char *argv0)
-{
-    fprintf(stderr, "usage:%s <api_path_fmt> <cv0>, <cv1>,...\n", argv0);
-    exit(0);
-}
-
-int
-main(int argc, char **argv)
-{
-    int nvec;
-    char **vec;
-    char *str0;
-    char *str1;
-    int   i;
-    char  *api_path_fmt;
-    cg_var *cv;
-    cvec  *cvv;
-    char  *api_path=NULL;
-
-    clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR); 
-    if (argc < 2){
-	usage(argv[0]);
-	return 0;
-    }
-    api_path_fmt = argv[1];
-    if ((cvv = cvec_new(0)) == NULL){
-	perror("cvec_new");
-	return -1;
-    }
-    cv = cv_new(CGV_STRING);
-    cv_string_set(cv, "CLI base command");
-    cvec_append_var(cvv, cv);
-    for (i=2; i<argc; i++){
-	cv = cv_new(CGV_STRING);
-	if (cv_parse(argv[i], cv) < 0){
-	    perror("cv_parse");
-	    return -1;
-	}
-	cvec_append_var(cvv, cv);
-    }
-    if (api_path_fmt2api_path(api_path_fmt, cvv, &api_path) < 0)
-	return -1;
-    printf("%s\n", api_path);
-    return 0;
-}
-
-#endif /* Test program */
 
 

lib/src/clixon_xml_parse.y

@@ -166,7 +166,6 @@ xml_parse_version(struct xml_parse_yacc_arg *ya,
 
 /*! Parse Qualified name --> Unprefixed name
  * @param[in] ya        XML parser yacc handler struct 
- * @param[in] prefix    Prefix, namespace, or NULL
  * @param[in] localpart Name
  * @note the call to xml_child_spec() may not have xmlns attribute read yet XXX
  */

lib/src/clixon_yang.c

@@ -2292,7 +2292,7 @@ yang_parse_find_match(clicon_handle h,
  * (cloned from cligen)
  * @param[in] h        CLICON handle
  * @param[in] filename Name of file
- * @param[in] ysp      Yang specification. Should ave been created by caller using yspec_new
+ * @param[in] ysp      Yang specification. Should have been created by caller using yspec_new
  * @retval ymod        Top-level yang (sub)module
  * @retval NULL        Error encountered
 
@@ -2305,7 +2305,7 @@ yang_parse_find_match(clicon_handle h,
  *   yang_parse_str         # Set up yacc parser and call it given a string
  *   clixon_yang_parseparse # Actual yang parsing using yacc
  */
-static yang_stmt *
+yang_stmt *
 yang_parse_filename(const char *filename, 
 		    yang_stmt  *ysp)
 {

lib/src/clixon_yang_type.c

@@ -48,6 +48,7 @@
  * |                  \  /    yang_type_cache_regex_set
  * ys_populate_leaf,   +--> compile_pattern2regexp (compile regexps)
  * xml_cv_cache (NULL) +--> cv_validate1 --> cv_validate_pattern (exec regexps)
+ * yang_type2cv (simplified)
  *
  * NOTE
  * 1) ys_cv_validate/ys_cv_validate_union_one and 
@@ -1387,3 +1388,24 @@ yang_type_get(yang_stmt    *ys,
     return retval;
 }
 
+/*! Utility function to translate a leaf/leaf-list to its base CV-type only
+ * @see yang_type_get  Full leaf/list type api
+ */
+enum cv_type
+yang_type2cv(yang_stmt  *ys)
+{
+    yang_stmt      *yrestype;  /* resolved type */
+    char           *restype;  /* resolved type */
+    char           *type;   /* original type */
+    enum cv_type    cvtype = CGV_ERR;
+    
+    /* Find type specification */
+    if (yang_type_get(ys, &type, &yrestype, NULL, NULL, NULL, NULL, NULL)
+ < 0)
+	goto done;
+    restype = yrestype?yrestype->ys_argument:NULL;
+    if (clicon_type2cv(type, restype, ys, &cvtype) < 0) /* This handles non-resolved also */
+	goto done;
+ done:
+    return cvtype;
+}

test/test_json.sh

@@ -19,4 +19,40 @@ expecteofx "$clixon_util_json" 0 '{"a":[0,1,2,3]}' "<a>0</a><a>1</a><a>2</a><a>3
 new "json parse list json" # should be {"a":[0,1,2,3]}
 expecteofx "$clixon_util_json -j" 0 '{"a":[0,1,2,3]}' '{"a": "0"}{"a": "1"}{"a": "2"}{"a": "3"}'
 
+fyang=$dir/json.yang
+fjson=$dir/json.json
+cat <<EOF > $fyang
+module json{
+   prefix ex;
+   namespace "urn:example:clixon";
+   leaf a{
+     type int32;
+   }
+   container c{
+     leaf a{
+       type int32;
+     }
+     leaf s{
+       type string;
+     }
+   }
+}
+EOF
+
+JSON='{"json:a": -23}'
+
+new "json leaf back to json"
+expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON"
+
+JSON='{"json:c": {"a": 937}}'
+new "json parse container back to json"
+expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON"
+
+# This is wrong
+if false; then
+JSON='{"json:c": {"s": "<![CDATA[  z > x  & x < y ]]>"}}'
+new "json parse cdata xml"
+expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON"
+fi
+
 rm -rf $dir

util/clixon_util_json.c

@@ -70,12 +70,13 @@
 static int
 usage(char *argv0)
 {
-    fprintf(stderr, "usage:%s [options]\n"
+    fprintf(stderr, "usage:%s [options] JSON as input on stdin\n"
 	    "where options are\n"
             "\t-h \t\tHelp\n"
     	    "\t-D <level> \tDebug\n"
 	    "\t-j \t\tOutput as JSON\n"
-	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n",
+	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n"
+	    "\t-y <filename> \tyang filename to parse (must be stand-alone)\n"	    ,
 	    argv0);
     exit(0);
 }
@@ -91,10 +92,14 @@ main(int    argc,
     int        c;
     int        logdst = CLICON_LOG_STDERR;
     int        json = 0;
+    char      *yang_filename = NULL;
+    yang_stmt *yspec = NULL;
+    cxobj     *xerr = NULL; /* malloced must be freed */
+    int        ret;
     
     optind = 1;
     opterr = 0;
-    while ((c = getopt(argc, argv, "hD:jl:")) != -1)
+    while ((c = getopt(argc, argv, "hD:jl:y:")) != -1)
 	switch (c) {
 	case 'h':
 	    usage(argv[0]);
@@ -110,13 +115,28 @@ main(int    argc,
 	    if ((logdst = clicon_log_opt(optarg[0])) < 0)
 		usage(argv[0]);
 	    break;
+	case 'y':
+	    yang_filename = optarg;
+	    break;
 	default:
 	    usage(argv[0]);
 	    break;
 	}
     clicon_log_init(__FILE__, debug?LOG_DEBUG:LOG_INFO, logdst);
-    if (json_parse_file(0, NULL, &xt) < 0)
+    if (yang_filename){
+	if ((yspec = yspec_new()) == NULL)
 	    goto done;
+	if (yang_parse_filename(yang_filename, yspec) == NULL){
+	    fprintf(stderr, "yang parse error %s\n", clicon_err_reason);
+	    return -1;
+	}
+    }
+    if ((ret = json_parse_file(0, yspec, &xt, &xerr)) < 0)
+	goto done;
+    if (ret == 0){
+	xml_print(stderr, xerr);
+	goto done;
+    }
     xc = NULL;
     while ((xc = xml_child_each(xt, xc, -1)) != NULL) 
 	if (json)
@@ -127,6 +147,8 @@ main(int    argc,
     fflush(stdout);
     retval = 0;
  done:
+    if (yspec)
+	yspec_free(yspec);
     if (xt)
 	xml_free(xt);
     if (cb)