From 2aeb92552176768cc23647bb668a72e713a19b77 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 3 Jun 2019 16:40:54 +0200 Subject: [PATCH] * JSON parse and print improvements * Integrated parsing with namespace translation and yang spec lookup --- README.md | 19 ++ apps/restconf/restconf_methods.c | 77 ++++---- lib/clixon/clixon_json.h | 6 +- lib/clixon/clixon_yang.h | 1 + lib/clixon/clixon_yang_type.h | 1 + lib/src/clixon_datastore_read.c | 3 +- lib/src/clixon_json.c | 310 +++++++++++++++++++------------ lib/src/clixon_json_parse.y | 28 ++- lib/src/clixon_xml.c | 33 +++- lib/src/clixon_xml_map.c | 65 ------- lib/src/clixon_xml_parse.y | 1 - lib/src/clixon_yang.c | 8 +- lib/src/clixon_yang_type.c | 22 +++ test/test_json.sh | 36 ++++ util/clixon_util_json.c | 44 +++-- 15 files changed, 396 insertions(+), 258 deletions(-) diff --git a/README.md b/README.md index c5509c03..0a6a90db 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ support. * [Scaling: large lists](doc/scaling/large-lists.md) * [Containers](docker/README.md) * [Roadmap](doc/ROADMAP.md) + * [Standard compliance](#standard-compliance) * [Reference manual](#reference) ## Background @@ -270,6 +271,24 @@ The functionality is as follows (references to sections in [RFC8341](https://too The figure shows the SDK runtime of Clixon. +## Standard Compliance + +This is work-in-progress on which standards Clixon supports: +- [RFC 6020](https://www.rfc-editor.org/rfc/rfc6020.txt) YANG - A Data Modeling Language for the Network Configuration Protocol (NETCONF) +- [RFC7895](http://www.rfc-base.org/txt/rfc-7895.txt) YANG Module Library +* [RFC7950](http://www.rfc-base.org/txt/rfc-7950.txt) The YANG 1.1 Data Modeling Language +* [RFC7951](http://www.rfc-base.org/txt/rfc-7951.txt) JSON Encoding of Data Modeled with YANG +- [RFC 6241: NETCONF Configuration Protocol](http://www.rfc-base.org/txt/rfc-6241.txt) +- [RFC 6242: Using the NETCONF Configuration Protocol over Secure Shell (SSH)](http://www.rfc-base.org/txt/rfc-6242.txt) +- [RFC 5277: NETCONF Event Notifications](http://www.rfc-base.org/txt/rfc-5277.txt) +- [RFC 8341: Network Configuration Access Control Model](http://www.rfc-base.org/txt/rfc-8341.txt) +- [RFC 8040: RESTCONF Protocol](https://tools.ietf.org/html/rfc8040). +- [RFC8341(NACM)](https://tools.ietf.org/html/rfc8341). +- [XML 1.0](https://www.w3.org/TR/2008/REC-xml-20081126) +- [Namespaces in XML 1.0](https://www.w3.org/TR/2009/REC-xml-names-20091208) +- [XPATH 1.0](https://www.w3.org/TR/xpath-10) +- [W3C XML XSD](http://www.w3.org/TR/2004/REC-xmlschema-2-20041028) + ## Reference Clixon uses [Doxygen](http://www.doxygen.nl/index.html) for reference documentation. diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index bd4b0482..756d2cce 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -498,16 +498,27 @@ api_data_post(clicon_handle h, goto ok; } } - else if (json_parse_str(data, &xdata) < 0){ - if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0) - goto done; - if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ - clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); - goto done; + else { + if ((ret = json_parse_str(data, yspec, &xdata, &xerr)) < 0){ + if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0) + goto done; + if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ + clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); + goto done; + } + if (api_return_err(h, r, xe, pretty, use_xml) < 0) + goto done; + goto ok; + } + if (ret == 0){ + if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ + clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); + goto done; + } + if (api_return_err(h, r, xe, pretty, use_xml) < 0) + goto done; + goto ok; } - if (api_return_err(h, r, xe, pretty, use_xml) < 0) - goto done; - goto ok; } /* 4.4.1: The message-body MUST contain exactly one instance of the * expected data resource. @@ -533,19 +544,6 @@ api_data_post(clicon_handle h, /* Replace xbot with x, ie bottom of api-path with data */ if (xml_addsub(xbot, x) < 0) goto done; - if (!parse_xml){ /* If JSON, translate namespace from module:name to xmlns=uri */ - if (json2xml_ns(yspec, x, &xerr) < 0) - goto done; - if (xerr){ - if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ - clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); - goto done; - } - if (api_return_err(h, r, xe, pretty, use_xml) < 0) - goto done; - goto ok; - } - } /* Create text buffer for transfer to backend */ if ((cbx = cbuf_new()) == NULL) goto done; @@ -805,7 +803,7 @@ api_data_put(clicon_handle h, } } else{ - if (json_parse_str(data, &xdata) < 0){ + if ((ret = json_parse_str(data, yspec, &xdata, &xerr)) < 0){ if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0) goto done; if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ @@ -816,6 +814,15 @@ api_data_put(clicon_handle h, goto done; goto ok; } + if (ret == 0){ + if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ + clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); + goto done; + } + if (api_return_err(h, r, xe, pretty, use_xml) < 0) + goto done; + goto ok; + } } /* The message-body MUST contain exactly one instance of the * expected data resource. @@ -832,20 +839,6 @@ api_data_put(clicon_handle h, goto ok; } x = xml_child_i(xdata,0); - if (!parse_xml){ /* If JSON, translate namespace from module:name to xmlns=uri */ - if (json2xml_ns(yspec, x, &xerr) < 0) - goto done; - if (xerr){ - if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ - clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); - goto done; - } - if (api_return_err(h, r, xe, pretty, use_xml) < 0) - goto done; - goto ok; - } - } - /* Add operation (create/replace) as attribute */ if ((xa = xml_new("operation", x, NULL)) == NULL) goto done; @@ -1329,6 +1322,7 @@ api_operations_post_input(clicon_handle h, cxobj *xinput; cxobj *x; cbuf *cbret = NULL; + int ret; clicon_debug(1, "%s %s", __FUNCTION__, data); if ((cbret = cbuf_new()) == NULL){ @@ -1350,7 +1344,7 @@ api_operations_post_input(clicon_handle h, } } else { /* JSON */ - if (json_parse_str(data, &xdata) < 0){ + if ((ret = json_parse_str(data, yspec, &xdata, &xerr)) < 0){ if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0) goto done; if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ @@ -1361,12 +1355,7 @@ api_operations_post_input(clicon_handle h, goto done; goto fail; } - /* Special case for JSON: It looks like: - * Need to translate to - */ - if (json2xml_ns(yspec, xdata, &xerr) < 0) - goto done; - if (xerr){ + if (ret == 0){ if ((xe = xpath_first(xerr, "rpc-error")) == NULL){ clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)"); goto done; diff --git a/lib/clixon/clixon_json.h b/lib/clixon/clixon_json.h index 70280284..d4b2b3d4 100644 --- a/lib/clixon/clixon_json.h +++ b/lib/clixon/clixon_json.h @@ -42,9 +42,9 @@ int xml2json_cbuf(cbuf *cb, cxobj *x, int pretty); int xml2json_cbuf_vec(cbuf *cb, cxobj **vec, size_t veclen, int pretty); int xml2json(FILE *f, cxobj *x, int pretty); +int json_print(FILE *f, cxobj *x); int xml2json_vec(FILE *f, cxobj **vec, size_t veclen, int pretty); -int json2xml_ns(yang_stmt *yspec, cxobj *x, cxobj **xerr); -int json_parse_str(char *str, cxobj **xt); -int json_parse_file(int fd, yang_stmt *yspec, cxobj **xt); +int json_parse_str(char *str, yang_stmt *yspec, cxobj **xt, cxobj **xret); +int json_parse_file(int fd, yang_stmt *yspec, cxobj **xt, cxobj **xret); #endif /* _CLIXON_JSON_H */ diff --git a/lib/clixon/clixon_yang.h b/lib/clixon/clixon_yang.h index c5347d89..821cf64d 100644 --- a/lib/clixon/clixon_yang.h +++ b/lib/clixon/clixon_yang.h @@ -186,6 +186,7 @@ int yang_print_cbuf(cbuf *cb, yang_stmt *yn, int marginal); int if_feature(yang_stmt *yspec, char *module, char *feature); int ys_populate(yang_stmt *ys, void *arg); yang_stmt *yang_parse_file(int fd, const char *name, yang_stmt *ysp); +yang_stmt *yang_parse_filename(const char *filename, yang_stmt *ysp); int yang_apply(yang_stmt *yn, enum rfc_6020 key, yang_applyfn_t fn, void *arg); int yang_datanode(yang_stmt *ys); diff --git a/lib/clixon/clixon_yang_type.h b/lib/clixon/clixon_yang_type.h index 8d2bab75..a0f8a08f 100644 --- a/lib/clixon/clixon_yang_type.h +++ b/lib/clixon/clixon_yang_type.h @@ -71,6 +71,7 @@ int yang_type_resolve(yang_stmt *yorig, yang_stmt *ys, yang_stmt **restype, int *options, cvec **cvv, cvec *patterns, cvec *regexps, uint8_t *fraction); +enum cv_type yang_type2cv(yang_stmt *ys); #endif /* _CLIXON_YANG_TYPE_H_ */ diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index 0b24ef23..60e762c8 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -316,6 +316,7 @@ xmldb_readfile(clicon_handle h, char *dbfile = NULL; int fd = -1; char *format; + int ret; if (xmldb_db2file(h, db, &dbfile) < 0) goto done; @@ -338,7 +339,7 @@ xmldb_readfile(clicon_handle h, goto done; } if (strcmp(format, "json")==0){ - if ((json_parse_file(fd, yspec, &x0)) < 0) + if ((ret = json_parse_file(fd, yspec, &x0, NULL)) < 0) /* XXX: ret == 0*/ goto done; } else if ((xml_parse_file(fd, "", yspec, &x0)) < 0) diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index ceaf290a..6049df0b 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -59,7 +59,11 @@ #include "clixon_hash.h" #include "clixon_handle.h" #include "clixon_yang.h" +#include "clixon_yang_type.h" +#include "clixon_options.h" #include "clixon_xml.h" +#include "clixon_xml_sort.h" +#include "clixon_xml_map.h" #include "clixon_netconf_lib.h" #include "clixon_json.h" #include "clixon_json_parse.h" @@ -265,6 +269,44 @@ json_str_escape_cdata(cbuf *cb, return retval; } +/*! If set, quoute the json value with double quotes + * @þaram[in] xb XML body object + @ @retval 0 Value should not be quouted, XML value is int, boolean,.. + @ @retval 1 Value should be quouted, XML value is string,.. + */ +static int +jsonvaluestr(cxobj *xb) +{ + int retval = 1; + cxobj *xp; + yang_stmt *yp; + enum rfc_6020 keyword; + + if ((xp = xml_parent(xb)) == NULL || + (yp = xml_spec(xp)) == NULL) + goto done; /* unknown */ + keyword = yang_keyword_get(yp); + if ((keyword == Y_LEAF || keyword == Y_LEAF_LIST)) + switch (yang_type2cv(yp)){ + case CGV_INT8: + case CGV_INT16: + case CGV_INT32: + case CGV_INT64: + case CGV_UINT8: + case CGV_UINT16: + case CGV_UINT32: + case CGV_UINT64: + case CGV_DEC64: + case CGV_BOOL: + retval = 0; + break; + default: + break; + } + done: + return retval; +} + /*! Do the actual work of translating XML to JSON * @param[out] cb Cligen text buffer containing json on exit * @param[in] x XML tree structure containing XML to translate @@ -309,7 +351,7 @@ xml2json1_cbuf(cbuf *cb, int level, int pretty, int flat, - int bodystr) + char *modname0) { int retval = -1; int i; @@ -318,27 +360,16 @@ xml2json1_cbuf(cbuf *cb, enum array_element_type xc_arraytype; yang_stmt *ys; yang_stmt *ymod; /* yang module */ - yang_stmt *yspec = NULL; /* yang spec */ - int bodystr0=1; - char *prefix=NULL; /* prefix / local namespace name */ - char *namespace=NULL; /* namespace uri */ - char *modname=NULL; /* Module name */ int commas; + char *modname = NULL; - /* If x is labelled with a default namespace, it should be translated - * to a module name. - * Harder if x has a prefix, then that should also be translated to associated - * module name - */ - prefix = xml_prefix(x); - namespace = xml_find_type_value(x, prefix, "xmlns", CX_ATTR); - - if ((ys = xml_spec(x)) != NULL) /* yang spec associated with x */ - yspec = ys_spec(ys); - /* Find module name associated with namspace URI */ - if (namespace && yspec && - (ymod = yang_find_module_by_namespace(yspec, namespace)) != NULL){ + if ((ys = xml_spec(x)) != NULL){ + ymod = ys_real_module(ys); modname = yang_argument_get(ymod); + if (modname0 && strcmp(modname, modname0) == 0) + modname=NULL; + else + modname0 = modname; /* modname0 is ancestor ns passed to child */ } childt = child_type(x); if (pretty==2) @@ -347,21 +378,20 @@ xml2json1_cbuf(cbuf *cb, childtype2str(childt)); switch(arraytype){ case BODY_ARRAY:{ - if (bodystr){ - /* XXX String if right type */ - cprintf(cb, "\""); + if (jsonvaluestr(x)) { /* Only print quotation if string-type */ + cprintf(cb, "\""); if (json_str_escape_cdata(cb, xml_value(x)) < 0) goto done; cprintf(cb, "\""); } - else + else /* No quotation marks */ cprintf(cb, "%s", xml_value(x)); break; } case NO_ARRAY: if (!flat){ cprintf(cb, "%*s\"", pretty?(level*JSON_INDENT):0, ""); - if (modname) /* XXX should remove this? */ + if (modname) cprintf(cb, "%s:", modname); cprintf(cb, "%s\": ", xml_name(x)); } @@ -426,26 +456,6 @@ xml2json1_cbuf(cbuf *cb, * arraytype=* but child-type is BODY_CHILD * This is code for writing 42 as "a":42 and not "a":"42" */ - if (childt == BODY_CHILD && ys!=NULL && - (yang_keyword_get(ys) == Y_LEAF || yang_keyword_get(ys) == Y_LEAF_LIST)) - switch (cv_type_get(yang_cv_get(ys))){ - case CGV_INT8: - case CGV_INT16: - case CGV_INT32: - case CGV_INT64: - case CGV_UINT8: - case CGV_UINT16: - case CGV_UINT32: - case CGV_UINT64: - case CGV_DEC64: - case CGV_BOOL: - bodystr0 = 0; - break; - default: - bodystr0 = 1; - break; - } - commas = xml_child_nr_notype(x, CX_ATTR) - 1; for (i=0; i 0) { cprintf(cb, ",%s", pretty?"\n":""); @@ -527,6 +537,9 @@ xml2json1_cbuf(cbuf *cb, } /*! Translate an XML tree to JSON in a CLIgen buffer + * + * XML-style namespace notation in tree, but RFC7951 in output assume yang + * populated * * @param[in,out] cb Cligen buffer to write to * @param[in] x XML tree to translate from @@ -551,28 +564,18 @@ xml2json_cbuf(cbuf *cb, { int retval = 1; int level = 0; - char *prefix; - char *namespace; cprintf(cb, "%*s{%s", pretty?level*JSON_INDENT:0,"", pretty?"\n":""); - /* If x is labelled with a default namespace, it should be translated - * to a module name. - * Harder if x has a prefix, then that should also be translated to associated - * module name - */ - prefix = xml_prefix(x); - if (xml2ns(x, prefix, &namespace) < 0) - goto done; - /* Some complexities in grafting namespace in existing trees to new */ - if (xml_find_type_value(x, prefix, "xmlns", CX_ATTR) == NULL && namespace) - if (xmlns_set(x, prefix, namespace) < 0) - goto done; if (xml2json1_cbuf(cb, x, NO_ARRAY, - level+1, pretty,0,1) < 0) + level+1, + pretty, + 0, + NULL /* ancestor modname / namespace */ + ) < 0) goto done; cprintf(cb, "%s%*s}%s", pretty?"\n":"", @@ -605,24 +608,16 @@ xml2json_cbuf_vec(cbuf *cb, { int retval = -1; int level = 0; - int i; cxobj *xp = NULL; + int i; cxobj *xc; - char *prefix; - char *namespace; if ((xp = xml_new("xml2json", NULL, NULL)) == NULL) goto done; /* Some complexities in grafting namespace in existing trees to new */ for (i=0; i --> */ int -json2xml_ns(yang_stmt *yspec, - cxobj *x, - cxobj **xerr) +json_xmlns_translate(yang_stmt *yspec, + cxobj *x, + cxobj **xerr) { int retval = -1; yang_stmt *ymod; char *namespace0; char *namespace; - char *name = NULL; char *prefix = NULL; cxobj *xc; + int ret; - if (nodeid_split(xml_name(x), &prefix, &name) < 0) - goto done; + prefix = xml_prefix(x); /* prefix is here module name */ if (prefix != NULL){ if ((ymod = yang_find_module_by_name(yspec, prefix)) == NULL){ - if (netconf_unknown_namespace_xml(xerr, "application", + if (xerr && + netconf_unknown_namespace_xml(xerr, "application", prefix, "No yang module found corresponding to prefix") < 0) goto done; - goto ok; + goto fail; } namespace = yang_find_mynamespace(ymod); /* Get existing default namespace in tree */ if (xml2ns(x, NULL, &namespace0) < 0) goto done; /* Set xmlns="" default namespace attribute (if diff from default) */ - if (namespace0==NULL || strcmp(namespace0, namespace)) + if (namespace0==NULL || strcmp(namespace0, namespace)){ if (xmlns_set(x, NULL, namespace) < 0) goto done; - /* Remove prefix from name */ - if (xml_name_set(x, name) < 0) - goto done; + /* and remove prefix */ + xml_prefix_set(x, NULL); + } } xc = NULL; while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL){ - if (json2xml_ns(yspec, xc, xerr) < 0) + if ((ret = json_xmlns_translate(yspec, xc, xerr)) < 0) goto done; - if (*xerr != NULL) - break; + if (ret == 0) + goto fail; } - ok: - retval = 0; + retval = 1; done: - if (prefix) - free(prefix); - if (name) - free(name); return retval; + fail: + retval = 0; + goto done; } - + /*! Parse a string containing JSON and return an XML tree + * + * Parsing using yacc according to JSON syntax. Names with : + * are split and interpreted as in RFC7951 + * * @param[in] str Input string containing JSON + * @param[in] yspec If set, also do yang validation * @param[in] name Log string, typically filename * @param[out] xt XML top of tree typically w/o children on entry (but created) + * @param[out] xerr Reason for invalid returned as netconf err msg + * + * @see _xml_parse for XML variant + * @retval 1 OK and valid + * @retval 0 Invalid (only if yang spec) + * @retval -1 Error with clicon_err called + * @see http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * @see RFC 7951 */ static int json_parse(char *str, - const char *name, - cxobj *xt) + yang_stmt *yspec, + const char *name, + cxobj *xt, + cxobj **xerr) { int retval = -1; struct clicon_json_yacc_arg jy = {0,}; + int ret; - // clicon_debug(1, "%s", __FUNCTION__); + clicon_debug(1, "%s", __FUNCTION__); jy.jy_parse_string = str; jy.jy_name = name; jy.jy_linenum = 1; @@ -807,20 +834,35 @@ json_parse(char *str, clicon_err(OE_XML, 0, "JSON parser error with no error code (should not happen)"); goto done; } - retval = 0; + if (yspec){ + /* Names are split into name/prefix, but now add namespace info */ + if ((ret = json_xmlns_translate(yspec, xt, xerr)) < 0) + goto done; + if (ret == 0) + goto fail; + /* Populate, ie associate xml nodes with yang specs */ + if (xml_apply0(xt, CX_ELMNT, xml_spec_populate, yspec) < 0) + goto done; + if (xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0) + goto done; + } + retval = 1; done: - // clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); json_parse_exit(&jy); json_scan_exit(&jy); + clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); return retval; + fail: /* invalid */ + retval = 0; + goto done; } /*! Parse string containing JSON and return an XML tree * - * @param[in] str String containing JSON - * @param[out] xt On success a top of XML parse tree is created with name 'top' - * @retval 0 OK - * @retval -1 Error with clicon_err called. Includes parse errors + * @param[in] str String containing JSON + * @param[in] yspec Yang specification, or NULL + * @param[in,out] xt On success a top of XML parse tree is created with name 'top' + * @param[out] xerr Reason for invalid returned as netconf err msg * * @code * cxobj *cx = NULL; @@ -829,39 +871,64 @@ json_parse(char *str, * xml_free(cx); * @endcode * @note you need to free the xml parse tree after use, using xml_free() + * @see json_parse_file + * @retval 1 OK and valid + * @retval 0 Invalid (only if yang spec) w xerr set + * @retval -1 Error with clicon_err called + * @see json_parse_file with a file descriptor (and more description) */ int -json_parse_str(char *str, - cxobj **xt) +json_parse_str(char *str, + yang_stmt *yspec, + cxobj **xt, + cxobj **xerr) { + clicon_debug(1, "%s", __FUNCTION__); if (*xt == NULL) if ((*xt = xml_new("top", NULL, NULL)) == NULL) return -1; - return json_parse(str, "", *xt); + return json_parse(str, yspec, "", *xt, xerr); } /*! Read a JSON definition from file and parse it into a parse-tree. * - * @param[in] fd A file descriptor containing the JSON file (as ASCII characters) - * @param[in] yspec Yang specification, or NULL XXX Not yet used - * @param[in,out] xt Pointer to (XML) parse tree. If empty, create. - * @retval 0 OK - * @retval -1 Error with clicon_err called + * File will be parsed as follows: + * (1) parsed according to JSON; # Only this check if yspec is NULL + * (2) sanity checked wrt yang + * (3) namespaces check (using : notation + * (4) an xml parse tree will be returned + * Note, only (1) and (4) will be done if yspec is NULL. + * Part of (3) is to split json names if they contain colon, + * eg: name="a:b" -> prefix="a", name="b" + * But this is not done if yspec=NULL, and is not part of the JSON spec + * + * @param[in] fd File descriptor to the JSON file (ASCII string) + * @param[in] yspec Yang specification, or NULL + * @param[in,out] xt Pointer to (XML) parse tree. If empty, create. + * @param[out] xerr Reason for invalid returned as netconf err msg * * @code * cxobj *xt = NULL; - * if (json_parse_file(0, NULL, &xt) < 0) + * if (json_parse_file(0, yspec, &xt) < 0) * err; * xml_free(xt); * @endcode * @note you need to free the xml parse tree after use, using xml_free() * @note, If xt empty, a top-level symbol will be added so that will be: * @note May block on file I/O + * + * @retval 1 OK and valid + * @retval 0 Invalid (only if yang spec) w xerr set + * @retval -1 Error with clicon_err called + * + * @see json_parse_str + * @see RFC7951 */ -int +int json_parse_file(int fd, yang_stmt *yspec, - cxobj **xt) + cxobj **xt, + cxobj **xerr) { int retval = -1; int ret; @@ -889,8 +956,12 @@ json_parse_file(int fd, if (*xt == NULL) if ((*xt = xml_new(JSON_TOP_SYMBOL, NULL, NULL)) == NULL) goto done; - if (len && json_parse(ptr, "", *xt) < 0) - goto done; + if (len){ + if ((ret = json_parse(ptr, yspec, "", *xt, xerr)) < 0) + goto done; + if (ret == 0) + goto fail; + } break; } if (len>=jsonbuflen-1){ /* Space: one for the null character */ @@ -904,7 +975,7 @@ json_parse_file(int fd, ptr = jsonbuf; } } - retval = 0; + retval = 1; done: if (retval < 0 && *xt){ free(*xt); @@ -913,6 +984,9 @@ json_parse_file(int fd, if (jsonbuf) free(jsonbuf); return retval; + fail: + retval = 0; + goto done; } diff --git a/lib/src/clixon_json_parse.y b/lib/src/clixon_json_parse.y index 07da89a4..8bbc6396 100644 --- a/lib/src/clixon_json_parse.y +++ b/lib/src/clixon_json_parse.y @@ -33,6 +33,7 @@ * JSON Parser * From http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf + * And RFC7951 JSON Encoding of Data Modeled with YANG Structural tokens: [ left square bracket @@ -72,7 +73,6 @@ object. */ - %start json %union { @@ -125,6 +125,7 @@ object. #include "clixon_err.h" #include "clixon_log.h" #include "clixon_queue.h" +#include "clixon_string.h" #include "clixon_hash.h" #include "clixon_handle.h" #include "clixon_yang.h" @@ -157,26 +158,39 @@ json_parse_init(struct clicon_json_yacc_arg *jy) return 0; } - int json_parse_exit(struct clicon_json_yacc_arg *jy) { return 0; } - + +/*! Create xml object from json object name (eg "string") + * Split name into prefix:name (extended JSON RFC7951) + */ static int json_current_new(struct clicon_json_yacc_arg *jy, char *name) { - int retval = -1; - cxobj *xn; + int retval = -1; + cxobj *x; + char *prefix = NULL; + char *id = NULL; clicon_debug(2, "%s", __FUNCTION__); - if ((xn = xml_new(name, jy->jy_current, NULL)) == NULL) + /* Find colon separator and if found split into prefix:name */ + if (nodeid_split(name, &prefix, &id) < 0) + goto done; + if ((x = xml_new(id, jy->jy_current, NULL)) == NULL) goto done; - jy->jy_current = xn; + if (prefix && xml_prefix_set(x, prefix) < 0) + goto done; + jy->jy_current = x; retval = 0; done: + if (prefix) + free(prefix); + if (id) + free(id); return retval; } diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index f7663ba2..1783dd05 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -238,7 +238,8 @@ xml_prefix_set(cxobj *xn, * @retval 0 OK * @retval -1 Error * @see xmlns_check XXX can these be merged? - * @note, this function uses a cache. Any case where cache should be cleared? + * @see xml2ns_set cache is set + * @note, this function uses a cache. */ int xml2ns(cxobj *x, @@ -286,12 +287,12 @@ xml2ns(cxobj *x, * @param[out] namespace URI namespace (or NULL). Will be copied * @retval 0 OK * @retval -1 Error - * @see xml2ns + * @see xml2ns */ int xmlns_set(cxobj *x, char *prefix, - char *namespace) + char *ns) { int retval = -1; cxobj *xa; @@ -307,8 +308,15 @@ xmlns_set(cxobj *x, goto done; xml_type_set(xa, CX_ATTR); } - if (xml_value_set(xa, namespace) < 0) + if (xml_value_set(xa, ns) < 0) goto done; + /* (re)set namespace cache (as used in xml2ns) */ + if (x->x_ns_cache) + free(x->x_ns_cache); + if ((x->x_ns_cache = strdup(ns)) == NULL){ + clicon_err(OE_XML, errno, "strdup"); + goto done; + } retval = 0; done: return retval; @@ -868,6 +876,9 @@ xml_addsub(cxobj *xp, { cxobj *oldp; int i; + char *pns = NULL; /* parent namespace */ + char *cns = NULL; /* child namespace */ + cxobj *xa; if ((oldp = xml_parent(xc)) != NULL){ /* Find child order i in old parent*/ @@ -884,6 +895,18 @@ xml_addsub(cxobj *xp, return -1; /* Set new parent in child */ xml_parent_set(xc, xp); + /* Ensure default namespace is not duplicated + * here only remove duplicate default namespace, there may be more */ + /* 1. Get parent default namespace */ + xml2ns(xp, NULL, &pns); + /* 2. Get child default namespace */ + if (pns && + (xa = xml_find_type(xc, NULL, "xmlns", CX_ATTR)) != NULL && + (cns = xml_value(xa)) != NULL){ + /* 3. check if same, if so remove child's */ + if (strcmp(pns, cns) == 0) + xml_purge(xa); + } } return 0; } @@ -1704,8 +1727,10 @@ _xml_parse(const char *str, goto done; /* Sort the complete tree after parsing */ if (yspec){ + /* Populate, ie associate xml nodes with yang specs */ if (xml_apply0(xt, CX_ELMNT, xml_spec_populate, yspec) < 0) goto done; + /* Sort according to yang */ if (xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0) goto done; } diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 6cddca59..475aa312 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -3022,70 +3022,5 @@ done: } -/* - * Turn this on for uni-test programs - * Usage: clixon_string join - * Example compile: - gcc -g -o clixon_xml_map -I. -I../clixon ./clixon_xml_map.c -lclixon -lcligen - * Example run: -/interfaces/interface=%s/name --> interfaces/interface/name -/interfaces/interface=%s/ipv4/address=%s e --> /interfaces/interface=e/ipv4/address -/interfaces/interface=%s,%s/ipv4/address=%s e f --> /interfaces/interface=e,f/ipv4/address -/interfaces/interface=%s/ipv4/address=%s,%s e f --> /interfaces/interface=e/ipv4/address=f - -/interfaces/interface=%s/ipv4/address=%s/prefix-length eth 1.2.3.4 --> -/interfaces/interface=eth/ipv4/address=1.2.3.4/prefix-length - -*/ -#if 0 /* Test program */ - -static int -usage(char *argv0) -{ - fprintf(stderr, "usage:%s , ,...\n", argv0); - exit(0); -} - -int -main(int argc, char **argv) -{ - int nvec; - char **vec; - char *str0; - char *str1; - int i; - char *api_path_fmt; - cg_var *cv; - cvec *cvv; - char *api_path=NULL; - - clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR); - if (argc < 2){ - usage(argv[0]); - return 0; - } - api_path_fmt = argv[1]; - if ((cvv = cvec_new(0)) == NULL){ - perror("cvec_new"); - return -1; - } - cv = cv_new(CGV_STRING); - cv_string_set(cv, "CLI base command"); - cvec_append_var(cvv, cv); - for (i=2; i Unprefixed name * @param[in] ya XML parser yacc handler struct - * @param[in] prefix Prefix, namespace, or NULL * @param[in] localpart Name * @note the call to xml_child_spec() may not have xmlns attribute read yet XXX */ diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index cd47c102..3c16852c 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -2292,7 +2292,7 @@ yang_parse_find_match(clicon_handle h, * (cloned from cligen) * @param[in] h CLICON handle * @param[in] filename Name of file - * @param[in] ysp Yang specification. Should ave been created by caller using yspec_new + * @param[in] ysp Yang specification. Should have been created by caller using yspec_new * @retval ymod Top-level yang (sub)module * @retval NULL Error encountered @@ -2305,9 +2305,9 @@ yang_parse_find_match(clicon_handle h, * yang_parse_str # Set up yacc parser and call it given a string * clixon_yang_parseparse # Actual yang parsing using yacc */ -static yang_stmt * -yang_parse_filename(const char *filename, - yang_stmt *ysp) +yang_stmt * +yang_parse_filename(const char *filename, + yang_stmt *ysp) { yang_stmt *ymod = NULL; int fd = -1; diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index b1f86634..cc1de405 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -48,6 +48,7 @@ * | \ / yang_type_cache_regex_set * ys_populate_leaf, +--> compile_pattern2regexp (compile regexps) * xml_cv_cache (NULL) +--> cv_validate1 --> cv_validate_pattern (exec regexps) + * yang_type2cv (simplified) * * NOTE * 1) ys_cv_validate/ys_cv_validate_union_one and @@ -1387,3 +1388,24 @@ yang_type_get(yang_stmt *ys, return retval; } +/*! Utility function to translate a leaf/leaf-list to its base CV-type only + * @see yang_type_get Full leaf/list type api + */ +enum cv_type +yang_type2cv(yang_stmt *ys) +{ + yang_stmt *yrestype; /* resolved type */ + char *restype; /* resolved type */ + char *type; /* original type */ + enum cv_type cvtype = CGV_ERR; + + /* Find type specification */ + if (yang_type_get(ys, &type, &yrestype, NULL, NULL, NULL, NULL, NULL) + < 0) + goto done; + restype = yrestype?yrestype->ys_argument:NULL; + if (clicon_type2cv(type, restype, ys, &cvtype) < 0) /* This handles non-resolved also */ + goto done; + done: + return cvtype; +} diff --git a/test/test_json.sh b/test/test_json.sh index 7511c002..999814b1 100755 --- a/test/test_json.sh +++ b/test/test_json.sh @@ -19,4 +19,40 @@ expecteofx "$clixon_util_json" 0 '{"a":[0,1,2,3]}' "0123 new "json parse list json" # should be {"a":[0,1,2,3]} expecteofx "$clixon_util_json -j" 0 '{"a":[0,1,2,3]}' '{"a": "0"}{"a": "1"}{"a": "2"}{"a": "3"}' +fyang=$dir/json.yang +fjson=$dir/json.json +cat < $fyang +module json{ + prefix ex; + namespace "urn:example:clixon"; + leaf a{ + type int32; + } + container c{ + leaf a{ + type int32; + } + leaf s{ + type string; + } + } +} +EOF + +JSON='{"json:a": -23}' + +new "json leaf back to json" +expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON" + +JSON='{"json:c": {"a": 937}}' +new "json parse container back to json" +expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON" + +# This is wrong +if false; then +JSON='{"json:c": {"s": " x & x < y ]]>"}}' +new "json parse cdata xml" +expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON" +fi + rm -rf $dir diff --git a/util/clixon_util_json.c b/util/clixon_util_json.c index ac60ae20..2a3a1c58 100644 --- a/util/clixon_util_json.c +++ b/util/clixon_util_json.c @@ -70,12 +70,13 @@ static int usage(char *argv0) { - fprintf(stderr, "usage:%s [options]\n" + fprintf(stderr, "usage:%s [options] JSON as input on stdin\n" "where options are\n" "\t-h \t\tHelp\n" "\t-D \tDebug\n" "\t-j \t\tOutput as JSON\n" - "\t-l \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n", + "\t-l \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n" + "\t-y \tyang filename to parse (must be stand-alone)\n" , argv0); exit(0); } @@ -84,17 +85,21 @@ int main(int argc, char **argv) { - int retval = -1; - cxobj *xt = NULL; - cxobj *xc; - cbuf *cb = cbuf_new(); - int c; - int logdst = CLICON_LOG_STDERR; - int json = 0; + int retval = -1; + cxobj *xt = NULL; + cxobj *xc; + cbuf *cb = cbuf_new(); + int c; + int logdst = CLICON_LOG_STDERR; + int json = 0; + char *yang_filename = NULL; + yang_stmt *yspec = NULL; + cxobj *xerr = NULL; /* malloced must be freed */ + int ret; optind = 1; opterr = 0; - while ((c = getopt(argc, argv, "hD:jl:")) != -1) + while ((c = getopt(argc, argv, "hD:jl:y:")) != -1) switch (c) { case 'h': usage(argv[0]); @@ -110,13 +115,28 @@ main(int argc, if ((logdst = clicon_log_opt(optarg[0])) < 0) usage(argv[0]); break; + case 'y': + yang_filename = optarg; + break; default: usage(argv[0]); break; } clicon_log_init(__FILE__, debug?LOG_DEBUG:LOG_INFO, logdst); - if (json_parse_file(0, NULL, &xt) < 0) + if (yang_filename){ + if ((yspec = yspec_new()) == NULL) + goto done; + if (yang_parse_filename(yang_filename, yspec) == NULL){ + fprintf(stderr, "yang parse error %s\n", clicon_err_reason); + return -1; + } + } + if ((ret = json_parse_file(0, yspec, &xt, &xerr)) < 0) goto done; + if (ret == 0){ + xml_print(stderr, xerr); + goto done; + } xc = NULL; while ((xc = xml_child_each(xt, xc, -1)) != NULL) if (json) @@ -127,6 +147,8 @@ main(int argc, fflush(stdout); retval = 0; done: + if (yspec) + yspec_free(yspec); if (xt) xml_free(xt); if (cb)