diff --git a/CHANGELOG.md b/CHANGELOG.md index e074d1ca..19b2a8c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ [search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml) ### API changes on existing features (you may need to change your code) +* Session-id CLI functionality delayed: "lazy evaluation" + * C-api: Changed `clicon_session_id_get(clicon_handle h, uint32_t *id)` + * From a cli perspective this is a revert to 4.1 behaviour, where the cli does not immediately exit on start if the backend is not running, but with the new session-id function * On failed validation of leafrefs, error message changed from: `No such leaf` to `No leaf matching path `. * CLI Error message (clicon_rpc_generate_error()) changed when backend returns netconf error to be more descriptive: * Original: `Config error: Validate failed. Edit and try again or discard changes: Invalid argument` @@ -21,8 +24,6 @@ ### Minor changes -* Session-id CLI functionality delayed: "lazy evaluation" - * From a cli perspective this is a revert to 4.1 behaviour, where the cli does not immediately exit on start if the backend is not running, but with the new session-id function * Obsoleted and removed XMLDB format "tree". This function did not work. Only xml and json allowed. * Test framework * Added `-- -S ` command-line to main example to be able to return any state to main example. @@ -33,19 +34,16 @@ ### Corrected Bugs +* Fixed: Pretty-printed XML using prefixes not parsed correctly. + * eg ` ` could lead to errors, wheras (` `) works fine. * XML namespace merge bug fixed. Example: two xmlns attributes could both survive a merge whereas one should replace the other. - -## 4.3.1 (2 February 2020) - -Patch release based on testing by Dave Cornejo, Netgate - -### Corrected Bugs * Compile option `VALIDATE_STATE_XML` introduced in `include/custom.h` to control whether code for state data validation is compiled or not. * Fixed: Validation of user state data led to wrong validation, if state relied on config data, eg leafref/must/when etc. * Fixed: No revision in yang module led to errors in validation of state data * Fixed: Leafref validation did not cover case of when the "path" statement is declared within a typedef, only if it was declared in the data part directly under leaf. * Fixed: Yang `must` xpath statements containing prefixes stopped working due to namespace context updates + ## 4.3.0 (1 January 2020) There were several issues with multiple namespaces with augmented yangs in 4.2 that have been fixed in 4.3. Some other highlights include: several issues with XPaths including "canonical namespace context" support, a reorganization of the YANG files shipped with the release, and a wildchar in the CLICON_MODE variable. diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 79411453..12bf047a 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -1499,8 +1499,9 @@ from_client_msg(clicon_handle h, ce->ce_id = id; /* Populate incoming XML tree with yang - * should really have been dealt with by decode above - * maybe not necessary since it should be */ - if (xml_spec_populate_rpc(h, x, yspec) < 0) + * but it still is needed - test_cli debug test fails + */ + if (xml_spec_populate_rpc_input(h, x, yspec) < 0) goto done; if ((ret = xml_yang_validate_rpc(h, x, &xret)) < 0) goto done; diff --git a/apps/netconf/netconf_main.c b/apps/netconf/netconf_main.c index 5efd6c77..a540628e 100644 --- a/apps/netconf/netconf_main.c +++ b/apps/netconf/netconf_main.c @@ -149,7 +149,7 @@ netconf_input_packet(clicon_handle h, free(str0); if ((xrpc=xpath_first(xreq, NULL, "//rpc")) != NULL){ isrpc++; - if (xml_spec_populate_rpc(h, xrpc, yspec) < 0) + if (xml_spec_populate_rpc_input(h, xrpc, yspec) < 0) goto done; if ((ret = xml_yang_validate_rpc(h, xrpc, &xret)) < 0) goto done; diff --git a/apps/restconf/restconf_methods_post.c b/apps/restconf/restconf_methods_post.c index 2592ebd6..e863d88f 100644 --- a/apps/restconf/restconf_methods_post.c +++ b/apps/restconf/restconf_methods_post.c @@ -861,7 +861,7 @@ api_operations_post(clicon_handle h, } #endif /* 6. Validate incoming RPC and fill in defaults */ - if (xml_spec_populate_rpc(h, xtop, yspec) < 0) /* */ + if (xml_spec_populate_rpc_input(h, xtop, yspec) < 0) /* */ goto done; if ((ret = xml_yang_validate_rpc(h, xtop, &xret)) < 0) goto done; diff --git a/include/clixon_custom.h b/include/clixon_custom.h index f816dd63..a10fe613 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -65,16 +65,18 @@ */ #define XPATH_LIST_OPTIMIZE -/*! Add search indexes, so that binary search can be made for non-key list indexes +/*! Add explicit search indexes, so that binary search can be made for non-key list indexes * This also applies if there are multiple keys and you want to search on only the second for * example. */ -#undef XML_EXTRA_INDEX +#undef XML_EXPLICIT_INDEX /*! Validate user state callback content - * Use may register state callbacks using ca_statedata callback - * When this option is set, the XML returned from the callback is validated after merging with the running - * db. If it fails, an internal error is returned to the originating user. - * If the option is not set, the XML returned by the user is not validated. + * User register state callbacks using the ca_statedata callback + * When this option is set, the XML returned from the callback is validated after merging with the + * running db. If it fails, an internal error is returned to the originating user. + * If the option is not set, the XML returned by the user is not validated. This could be useful if + * there is some error in the validation, as a safety catch. + * (Eventually this option will be permanently enabled) */ #define VALIDATE_STATE_XML diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 4fa1d273..a68aa733 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -35,6 +35,8 @@ * XML support functions. * @see https://www.w3.org/TR/2008/REC-xml-20081126 * https://www.w3.org/TR/2009/REC-xml-names-20091208/ + * Canonical XML version (just for info) + * https://www.w3.org/TR/xml-c14n */ #ifndef _CLIXON_XML_H #define _CLIXON_XML_H @@ -158,9 +160,9 @@ cxobj *xml_wrap(cxobj *xc, char *tag); int xml_purge(cxobj *xc); int xml_child_rm(cxobj *xp, int i); int xml_rm(cxobj *xc); +int xml_rm_children(cxobj *x, enum cxobj_type type); int xml_rootchild(cxobj *xp, int i, cxobj **xcp); int xml_rootchild_node(cxobj *xp, cxobj *xc); - int xml_enumerate_children(cxobj *xp); int xml_enumerate_reset(cxobj *xp); int xml_enumerate_get(cxobj *x); diff --git a/lib/clixon/clixon_xml_map.h b/lib/clixon/clixon_xml_map.h index 9a5107a2..61e2c9c7 100644 --- a/lib/clixon/clixon_xml_map.h +++ b/lib/clixon/clixon_xml_map.h @@ -59,10 +59,10 @@ int xml_tree_prune_flagged(cxobj *xt, int flag, int test); int xml_default(cxobj *x, void *arg); int xml_sanity(cxobj *x, void *arg); int xml_non_config_data(cxobj *xt, void *arg); -int xml_spec_populate_rpc(clicon_handle h, cxobj *x, yang_stmt *yspec); +int xml_spec_populate_rpc_input(clicon_handle h, cxobj *x, yang_stmt *yspec); int xml_spec_populate(cxobj *x, void *arg); int xml2xpath(cxobj *x, char **xpath); -int check_namespaces(cxobj *x0, cxobj *x1, cxobj *x1p); +int assign_namespaces(cxobj *x0, cxobj *x1, cxobj *x1p); int xml_merge(cxobj *x0, cxobj *x1, yang_stmt *yspec, char **reason); int yang_enum_int_value(cxobj *node, int32_t *val); diff --git a/lib/clixon/clixon_yang.h b/lib/clixon/clixon_yang.h index 07dfe548..cf31bb33 100644 --- a/lib/clixon/clixon_yang.h +++ b/lib/clixon/clixon_yang.h @@ -51,7 +51,7 @@ * Yang flags used in */ #define YANG_FLAG_MARK 0x01 /* (Dynamic) marker for dynamic algorithms, eg expand */ -#ifdef XML_EXTRA_INDEX +#ifdef XML_EXPLICIT_INDEX #define YANG_FLAG_INDEX 0x02 /* This yang node under list is (extra) index. --> you can access * list elements using this index with binary search */ #endif diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index b3866703..4dc2be88 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -79,7 +79,7 @@ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_err.c clixon_event.c \ clixon_sha1.c clixon_datastore.c clixon_datastore_write.c clixon_datastore_read.c \ clixon_netconf_lib.c clixon_stream.c clixon_nacm.c -YACCOBJS := lex.clixon_xml_parse.o clixon_xml_parse.tab.o \ +YACCOBJS = lex.clixon_xml_parse.o clixon_xml_parse.tab.o \ lex.clixon_yang_parse.o clixon_yang_parse.tab.o \ lex.clixon_json_parse.o clixon_json_parse.tab.o \ lex.clixon_xpath_parse.o clixon_xpath_parse.tab.o \ diff --git a/lib/src/clixon_data.c b/lib/src/clixon_data.c index 2436f0b0..6ddc3d5e 100644 --- a/lib/src/clixon_data.c +++ b/lib/src/clixon_data.c @@ -615,4 +615,3 @@ clicon_session_id_set(clicon_handle h, clicon_hash_add(cdat, "session-id", &id, sizeof(uint32_t)); return 0; } - diff --git a/lib/src/clixon_datastore_write.c b/lib/src/clixon_datastore_write.c index 577fee3f..dc34a5e8 100644 --- a/lib/src/clixon_datastore_write.c +++ b/lib/src/clixon_datastore_write.c @@ -348,7 +348,7 @@ text_modify(clicon_handle h, * Check if namespace exists in x0 parent * if not add new binding and replace in x0. */ - if (check_namespaces(x1, x0, x0p) < 0) + if (assign_namespaces(x1, x0, x0p) < 0) goto done; changed++; if (op==OP_NONE) @@ -514,7 +514,7 @@ text_modify(clicon_handle h, * Check if namespace exists in x0 parent * if not add new binding and replace in x0. */ - if (check_namespaces(x1, x0, x0p) < 0) + if (assign_namespaces(x1, x0, x0p) < 0) goto done; if (op==OP_NONE) xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */ diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index df7d4926..6440b49e 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -1156,9 +1156,12 @@ json_parse(char *str, goto done; if (ret == 0) goto fail; - /* Populate, ie associate xml nodes with yang specs */ - if (xml_apply0(xt, CX_ELMNT, xml_spec_populate, yspec) < 0) + /* Populate, ie associate xml nodes with yang specs. + * XXX But this wrong if xt is not on top-level, can give false positives. + */ + if (xml_apply0(xt, CX_ELMNT, xml_spec_populate, yspec) < 0){ goto done; + } if (xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0) goto done; /* Now find leafs with identityrefs (+transitive) and translate diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index 44a3ed61..9357169b 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -330,8 +330,10 @@ clicon_option_add(clicon_handle h, if (strcmp(name, "CLICON_FEATURE")==0 || strcmp(name, "CLICON_YANG_DIR")==0){ - if ((x = clicon_conf_xml(h)) == NULL) + if ((x = clicon_conf_xml(h)) == NULL){ + clicon_err(OE_UNIX, ENOENT, "option %s not found (clicon_conf_xml_set has not been called?)", name); goto done; + } if (xml_parse_va(&x, NULL, "<%s>%s", name, value, name) < 0) goto done; diff --git a/lib/src/clixon_path.c b/lib/src/clixon_path.c index e2ec092b..2e2e765b 100644 --- a/lib/src/clixon_path.c +++ b/lib/src/clixon_path.c @@ -1394,7 +1394,7 @@ instance_id_resolve(clixon_path *cplist, goto done; } -/*! Search XML tree using Clixon path struct +/*! Search XML tree using (internal) Clixon path struct * * @param[in] xt Top xml-tree where to search * @param[in] yt Yang statement of top symbol (can be yang-spec if top-level) @@ -1612,7 +1612,7 @@ clixon_xml_find_instance_id(cxobj *xt, goto done; if (debug) clixon_path_print(stderr, cplist); - /* Resolve module:name to yang-stmt, fail if not successful */ + /* Resolve module:name to pointer to yang-stmt, fail if not successful */ if ((ret = instance_id_resolve(cplist, yt)) < 0) goto done; if (ret == 0) diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index 3541d0a1..00e75a79 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -400,7 +400,6 @@ clixon_plugin_exit(clicon_handle h) return 0; } - /*! Run the restconf user-defined credentials callback if present * Find first authentication callback and call that, then return. * The callback is to set the authenticated user diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index 14b97273..83449332 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -35,6 +35,8 @@ * XML support functions. * @see https://www.w3.org/TR/2008/REC-xml-20081126 * https://www.w3.org/TR/2009/REC-xml-names-20091208 + * Canonical XML version (just for info) + * https://www.w3.org/TR/xml-c14n */ #ifdef HAVE_CONFIG_H @@ -1276,6 +1278,35 @@ xml_rm(cxobj *xc) return retval; } +/*! Remove all children of specific type + * @param[in] x XML node + * @param[in] type Remove all children of xn of this type + * @retval 0 OK + * @retval -1 Error + */ +int +xml_rm_children(cxobj *x, + enum cxobj_type type) +{ + int retval = -1; + cxobj *xc; + int i; + + for (i=0; i " + * since it is valid leaf(-list) content. + * But if it is not a container, list or something, else, the content should be stripped. + * But we cannot do this because of false positives (above) + */ + if (xml_spec(x) == NULL && /* Not assigned yet, ensure to do just once,... */ + yang_keyword_get(y) != Y_LEAF && + yang_keyword_get(y) != Y_LEAF_LIST){ + if (xml_rm_children(x, CX_BODY) < 0) + goto done; + } +#endif + xml_spec_set(x, y); + } } + else{ #ifdef DEBUG - else clicon_debug(1, "%s y:NULL", __FUNCTION__); #endif + } retval = 0; done: return retval; } - /*! Given an XML node, build an xpath to root, internal function * @retval 0 OK * @retval -1 Error. eg XML malformed @@ -1159,19 +1181,19 @@ xmlns_assign(cxobj *x) goto done; */ int -check_namespaces(cxobj *x0, /* source */ - cxobj *x1, /* target */ - cxobj *x1p) +assign_namespaces(cxobj *x0, /* source */ + cxobj *x1, /* target */ + cxobj *x1p) { - int retval = -1; - char *namespace = NULL; - char *prefix0 = NULL;; - char *prefix1 = NULL; - char *prefixb = NULL; /* identityref body prefix */ - cvec *nsc0 = NULL; - cvec *nsc = NULL; - int isroot; - char *pexist = NULL; + int retval = -1; + char *namespace = NULL; + char *prefix0 = NULL;; + char *prefix1 = NULL; + char *prefixb = NULL; /* identityref body prefix */ + cvec *nsc0 = NULL; + cvec *nsc = NULL; + int isroot; + char *pexist = NULL; yang_stmt *y; /* XXX: need to identify root better than hiereustics and strcmp,... */ @@ -1313,7 +1335,7 @@ xml_merge1(cxobj *x0, /* the target */ if (xml_value_set(x0b, x1bstr) < 0) goto done; } - if (check_namespaces(x1, x0, x0p) < 0) + if (assign_namespaces(x1, x0, x0p) < 0) goto done; } /* if LEAF|LEAF_LIST */ else { /* eg Y_CONTAINER, Y_LIST */ @@ -1321,7 +1343,7 @@ xml_merge1(cxobj *x0, /* the target */ if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL) goto done; } - if (check_namespaces(x1, x0, x0p) < 0) + if (assign_namespaces(x1, x0, x0p) < 0) goto done; /* Loop through children of the modification tree */ x1c = NULL; diff --git a/lib/src/clixon_xml_parse.h b/lib/src/clixon_xml_parse.h index 6be1607f..693edb83 100644 --- a/lib/src/clixon_xml_parse.h +++ b/lib/src/clixon_xml_parse.h @@ -50,7 +50,6 @@ struct xml_parse_yacc_arg{ cxobj *ya_xelement; /* xml active element */ cxobj *ya_xparent; /* xml parent element*/ - int ya_skipspace; /* If set, remove all non-terminal bodies (strip pretty-print) */ yang_stmt *ya_yspec; /* If set, top-level yang-spec */ int ya_lex_state; /* lex return state */ }; diff --git a/lib/src/clixon_xml_parse.l b/lib/src/clixon_xml_parse.l index 6f6631e7..87c04dba 100644 --- a/lib/src/clixon_xml_parse.l +++ b/lib/src/clixon_xml_parse.l @@ -34,8 +34,8 @@ * XML parser * @see https://www.w3.org/TR/2008/REC-xml-20081126 * https://www.w3.org/TR/2009/REC-xml-names-20091208 - * - + * Canonical XML version (just for info) + * https://www.w3.org/TR/xml-c14n */ %{ diff --git a/lib/src/clixon_xml_parse.y b/lib/src/clixon_xml_parse.y index 3a1e9419..562d3779 100644 --- a/lib/src/clixon_xml_parse.y +++ b/lib/src/clixon_xml_parse.y @@ -34,6 +34,8 @@ * XML parser * @see https://www.w3.org/TR/2008/REC-xml-20081126 * https://www.w3.org/TR/2009/REC-xml-names-20091208 + * Canonical XML version (just for info) + * https://www.w3.org/TR/xml-c14n */ %union { char *string; @@ -72,6 +74,7 @@ #include "clixon_log.h" #include "clixon_queue.h" #include "clixon_hash.h" +#include "clixon_string.h" #include "clixon_handle.h" #include "clixon_yang.h" #include "clixon_xml.h" @@ -127,14 +130,12 @@ xml_parse_whitespace(struct xml_parse_yacc_arg *ya, ya->ya_xelement = NULL; /* init */ /* If there is an element already, only add one whitespace child - * otherwise, keep all whitespace. + * otherwise, keep all whitespace. See code in xml_parse_bslash */ -#if 1 for (i=0; i */ +/*! A content terminated by ... or ... is ready + * + * Any whitespace between the subelements to a non-leaf is + * insignificant, i.e., an implementation MAY insert whitespace + * characters between subelements and are therefore stripped, but see comment in code below. + * @param[in] ya XML parser yacc handler struct + * @param[in] prefix + * @param[in] name + */ static int -xml_parse_bslash1(struct xml_parse_yacc_arg *ya, - char *name) +xml_parse_bslash(struct xml_parse_yacc_arg *ya, + char *prefix, + char *name) { int retval = -1; cxobj *x = ya->ya_xelement; cxobj *xc; + char *prefix0; + char *name0; - if (strcmp(xml_name(x), name)){ - clicon_err(OE_XML, XMLPARSE_ERRNO, "XML parse sanity check failed: %s vs %s", - xml_name(x), name); - goto done; - } - if (xml_prefix(x)!=NULL){ - clicon_err(OE_XML, XMLPARSE_ERRNO, "XML parse sanity check failed: %s:%s vs %s", - xml_prefix(x), xml_name(x), name); + /* These are existing tags */ + prefix0 = xml_prefix(x); + name0 = xml_name(x); + /* Check name or prerix unequal from begin-tag */ + if (clicon_strcmp(name0, name) || + clicon_strcmp(prefix0, prefix)){ + clicon_err(OE_XML, XMLPARSE_ERRNO, "Sanity check failed: %s%s%s vs %s%s%s", + prefix0?prefix0:"", prefix0?":":"", name0, + prefix?prefix:"", prefix?":":"", name); goto done; } /* Strip pretty-print. Ad-hoc algorithm * It ok with x:[body], but not with x:[ex,body] * It is also ok with x:[attr,body] - * So the rule is: if there is at least on element, then remove all bodies? + * So the rule is: if there is at least on element, then remove all bodies. + * See also code in xml_parse_whitespace + * But there is more: when YANG is assigned, if not leaf/leaf-lists, then all contents should + * be stripped, see xml_spec_populate() */ - if (ya->ya_skipspace){ - xc = NULL; - while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) - break; - if (xc != NULL){ /* at least one element */ - int i; - for (i=0; i */ -static int -xml_parse_bslash2(struct xml_parse_yacc_arg *ya, - char *namespace, - char *name) -{ - int retval = -1; - cxobj *x = ya->ya_xelement; - cxobj *xc; - - if (strcmp(xml_name(x), name)){ - clicon_err(OE_XML, XMLPARSE_ERRNO, "Sanity check failed: %s:%s vs %s:%s", - xml_prefix(x), - xml_name(x), - namespace, - name); - goto done; - } - if (xml_prefix(x)==NULL || - strcmp(xml_prefix(x), namespace)){ - clicon_err(OE_XML, XMLPARSE_ERRNO, "Sanity check failed: %s:%s vs %s:%s", - xml_prefix(x), - xml_name(x), - namespace, - name); - goto done; - } - /* Strip pretty-print. Ad-hoc algorithm - * It ok with x:[body], but not with x:[ex,body] - * It is also ok with x:[attr,body] - * So the rule is: if there is at least on element, then remove all bodies? - */ - if (ya->ya_skipspace){ - xc = NULL; - while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) - break; - if (xc != NULL){ /* at least one element */ - xc = NULL; - while ((xc = xml_child_each(x, xc, CX_BODY)) != NULL) { - xml_value_set(xc, ""); /* XXX remove */ - } - } - } - retval = 0; - done: - free(name); - free(namespace); - return retval; -} - /*! Parse XML attribute * Special cases: * - DefaultAttName: xmlns @@ -450,10 +405,10 @@ element1 : ESLASH {_YA->ya_xelement = NULL; endtag : BSLASH NAME '>' { clicon_debug(2, "endtag -> < "); - if (xml_parse_bslash1(_YA, $2) < 0) YYABORT; } + if (xml_parse_bslash(_YA, NULL, $2) < 0) YYABORT; } | BSLASH NAME ':' NAME '>' - { if (xml_parse_bslash2(_YA, $2, $4) < 0) YYABORT; + { if (xml_parse_bslash(_YA, $2, $4) < 0) YYABORT; clicon_debug(2, "endtag -> < "); } ; diff --git a/lib/src/clixon_xml_sort.c b/lib/src/clixon_xml_sort.c index 2230ddcb..21cbeccf 100644 --- a/lib/src/clixon_xml_sort.c +++ b/lib/src/clixon_xml_sort.c @@ -579,7 +579,7 @@ xml_find_keys1(cxobj *xp, * @see xml_find_index for a generic search function */ static int -xml_find_keys(cxobj *xp, +xml_find_keys(cxobj *xp, cxobj *x1, yang_stmt *yc, int skip1, @@ -1160,6 +1160,36 @@ xml_find_index_yang(cxobj *xp, goto done; } +#ifdef XML_EXPLICIT_INDEX +/*! API for search in XML child list with yang available + * @retval 1 OK + * @retval 0 Revert, try again with no-yang search (or explicit index) + * @retval -1 Error + * XXX: can merge with xml_find_index_yang in some way, similar code + */ +static int +xml_find_index_explicit(cxobj *xp, + yang_stmt *yc, + cvec *cvk, + cxobj ***xvec, + size_t *xlen) +{ + int retval = -1; + cg_var *cvi; + + if (yang_keyword_get(yc) == Y_LIST && cvec_len(cvk) == 1){ + cvi = cvec_i(cvk, 0); + goto revert; + } + retval = 1; /* OK */ + done: + return retval; + revert: /* means try name-only*/ + retval = 0; + goto done; +} +#endif /* XML_EXPLICIT_INDEX */ + /*! API for search in XML child list using indexes and binary search if applicable * * Generic search function as alternative to xml_find() and others that finds YANG associated @@ -1237,9 +1267,18 @@ clixon_xml_find_index(cxobj *xp, if (yc){ if ((ret = xml_find_index_yang(xp, yc, cvk, xvec, xlen)) < 0) goto done; - if (ret == 0) - if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) + if (ret == 0){ /* This means yang method did not work for some reason + * such as not being list key indexes in cvk, for example + */ +#ifdef XML_EXPLICIT_INDEX + /* Check if (exactly one) explicit indexes in cvk */ + if ((ret = xml_find_index_explicit(xp, yc, cvk, xvec, xlen)) < 0) goto done; + if (ret == 0) +#endif + if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) + goto done; + } } else if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 70d2b06c..d0f545df 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -84,6 +84,10 @@ #include "clixon_yang_type.h" #include "clixon_yang_internal.h" /* internal included by this file only, not API*/ +#ifdef XML_EXPLICIT_INDEX +static int yang_search_index_extension(clicon_handle h, yang_stmt *yext, yang_stmt *ys); +#endif + /* * Local variables */ @@ -1824,9 +1828,17 @@ ys_populate_unknown(clicon_handle h, goto done; } } +#ifdef XML_EXPLICIT_INDEX + /* Add explicit index extension */ + if ((retval = yang_search_index_extension(h, yext, ys)) < 0) { + clicon_debug(1, "plugin_extension() failed"); + return -1; + } +#endif /* Make extension callbacks that may alter yang structure */ if (clixon_plugin_extension(h, yext, ys) < 0) goto done; + retval = 0; done: if (prefix) @@ -2634,14 +2646,61 @@ yang_type_cache_free(yang_type_cache *ycache) return 0; } -#ifdef XML_EXTRA_INDEX -/*! Mark element as index in list +#ifdef XML_EXPLICIT_INDEX +/*! Mark element as search_index in list + * @retval 0 OK + * @retval -1 Error */ int -yang_list_index_add(yang_stmt *yi) +yang_list_index_add(yang_stmt *ys) { - assert(yang_parent_get(yi) && yang_keyword_get(yang_parent_get(yi)) == Y_LIST); - yi->ys_flags |= YANG_FLAG_INDEX; - return 0; + int retval = -1; + yang_stmt *yp; + + if ((yp = yang_parent_get(ys)) == NULL || + yang_keyword_get(yp) != Y_LIST){ + clicon_log(LOG_WARNING, "search_index should in a list"); + goto ok; + } + yang_flag_set(ys, YANG_FLAG_INDEX); + ok: + retval = 0; + // done: + return retval; } -#endif /* XML_EXTRA_INDEX */ + +/*! Callback for yang clixon search_index extension + * + * @param[in] h Clixon handle + * @param[in] yext Yang node of extension + * @param[in] ys Yang node of (unknown) statement belonging to extension + * @retval 0 OK (warnings may appear) + * @retval -1 Error + */ +int +yang_search_index_extension(clicon_handle h, + yang_stmt *yext, + yang_stmt *ys) +{ + int retval = -1; + char *extname; + char *modname; + yang_stmt *ymod; + yang_stmt *yp; + + ymod = ys_module(yext); + modname = yang_argument_get(ymod); + extname = yang_argument_get(yext); + if (strcmp(modname, "clixon-config") != 0 || strcmp(extname, "search_index") != 0) + goto ok; + clicon_debug(1, "%s Enabled extension:%s:%s", __FUNCTION__, modname, extname); + yp = yang_parent_get(ys); + if (yang_list_index_add(yp) < 0) + goto done; + ok: + retval = 0; + done: + return retval; +} + +#endif /* XML_EXPLICIT_INDEX */ diff --git a/test/lib.sh b/test/lib.sh index 4beaac79..7f808fca 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -70,7 +70,7 @@ testname= # for starting a backend : ${BETIMEOUT:=10} -# If set, enable debugging (of backend) +# If set, enable debugging (of backend and restconf daemons) : ${DBG:=0} # Where to log restconf. Some systems may not have syslog, diff --git a/test/test_api_path.sh b/test/test_api_path.sh index 9a59402b..1f9c4b9b 100755 --- a/test/test_api_path.sh +++ b/test/test_api_path.sh @@ -15,8 +15,11 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi # Number of list/leaf-list entries : ${nr:=100} +# Number of tests to generate XML for +max=7 + # XML file (alt provide it in stdin after xpath) -for (( i=1; i<7; i++ )); do +for (( i=1; i<$max; i++ )); do eval xml$i=$dir/xml$i.xml done ydir=$dir/yang diff --git a/test/test_cli.sh b/test/test_cli.sh index 23a438a4..27bea4b8 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -113,9 +113,11 @@ expectfn "$clixon_cli -1 -f $cfg -l o load /tmp/foo" 0 "^$" new "cli check load" expectfn "$clixon_cli -1 -f $cfg -l o show conf cli" 0 "interfaces interface eth/0/0 ipv4 enabled true" -new "cli debug" +new "cli debug set" expectfn "$clixon_cli -1 -f $cfg -l o debug level 1" 0 "^$" + # How to test this? +new "cli debug reset" expectfn "$clixon_cli -1 -f $cfg -l o debug level 0" 0 "^$" new "cli rpc" diff --git a/test/test_instance_id.sh b/test/test_instance_id.sh index 10158ca2..10972939 100755 --- a/test/test_instance_id.sh +++ b/test/test_instance_id.sh @@ -9,8 +9,11 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi # Number of list/leaf-list entries : ${nr:=100} +# Number of tests to generate XML for +max=9 + # XML file (alt provide it in stdin after xpath) -for (( i=1; i<9; i++ )); do +for (( i=1; i<$max; i++ )); do eval xml$i=$dir/xml$i.xml done ydir=$dir/yang @@ -19,7 +22,7 @@ if [ ! -d $ydir ]; then mkdir $ydir fi -# XPATH binary search in ordered-by system lists +# Instance-id PATH binary search in ordered-by system lists cat < $ydir/moda.yang module moda{ namespace "urn:example:a"; diff --git a/test/test_leafref_state.sh b/test/test_leafref_state.sh index 5f6acb8a..c9f1dd1c 100755 --- a/test/test_leafref_state.sh +++ b/test/test_leafref_state.sh @@ -63,6 +63,7 @@ module leafref{ } EOF +# This is state data writte to file that backend reads from (on request) cat < $fstate x @@ -83,7 +84,7 @@ if [ $BE -ne 0 ]; then wait_backend fi -# Test top-level, default prefix, wring leafref prefix and typedef path +# Test top-level, default prefix, wrong leafref prefix and typedef path XML=$(cat < x @@ -127,6 +128,24 @@ expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' +# Negative tests, +# Double xmlns attribute +cat < $fstate + + x + +EOF + +new "Merge same tree - check double xmlns attribute" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +# Back to original +cat < $fstate + + x + +EOF + # delete x, add y XML=$(cat < diff --git a/test/test_netconf_whitespace.sh b/test/test_netconf_whitespace.sh new file mode 100755 index 00000000..362b2d5b --- /dev/null +++ b/test/test_netconf_whitespace.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Test netconf / xml whitespaces +# Insignicant and significant +# See https://www.w3.org/TR/xml-c14n +# where "clean", "dirty" and "mixed" whitespaces are defined +# +# A B +# +# A +# +# B +# A B +# C +# + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +cfg=$dir/conf_yang.xml +fyang=$dir/whitespace.yang + +cat < $cfg + + $cfg + ietf-netconf:startup + /usr/local/share/clixon + $fyang + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/lib/$APPNAME/backend + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + +EOF + +cat < $fyang +module whitespace{ + yang-version 1.1; + namespace "urn:example:whitespace"; + prefix fi; + container x{ + presence true; + list y { + key a; + leaf a{ + type string; + } + container b{ + presence true; + } + } + } +} +EOF + + +new "test params: -f $cfg" + +echo ' + foo\n ]]>]]>$start' > $dir/startup_db + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg -s startup + if [ $? -ne 0 ]; then + err + fi + new "start backend -s startup -f $cfg" + start_backend -s startup -f $cfg +fi + +new "waiting" +wait_backend + +new "get startup" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^foo]]>]]>$' + +new "remove" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^]]>]]>$' + +new "commit" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "add insignificant 'dirty' whitespaces" +expecteof "$clixon_netconf -qf $cfg" 0 ' \t \ foo\n ]]>]]>' '^]]>]]>$' + +new "get stripped whitespace" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^foo]]>]]>$' + +new "validate" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "add insignificant 'dirty' whitespaces with prefixes" +expecteof "$clixon_netconf -qf $cfg" 0 ' \t \ foo\n ]]>]]>' '^]]>]]>$' + +new "get stripped whitespace" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^foo]]>]]>$' + +new "validate" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "add insignificant 'clean' whitespaces" +expecteof "$clixon_netconf -qf $cfg" 0 ' \n \t ]]>]]>' '^]]>]]>$' + +new "add more insignificant 'clean' whitespaces" +expecteof "$clixon_netconf -qf $cfg" 0 'foo ]]>]]>' '^]]>]]>$' + +new "get stripped whitespace" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^foo]]>]]>' + +new "validate" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "add significant whitespaces" +expecteof "$clixon_netconf -qf $cfg" 0 ' foo + bar ]]>]]>' '^]]>]]>$' + +new "get significant whitespace" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^ foo + bar ]]>]]>' + +new "validate" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=$(pgrep -u root -f clixon_backend) +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +rm -rf $dir diff --git a/test/test_xml.sh b/test/test_xml.sh index f800a4ed..0bb9a04f 100755 --- a/test/test_xml.sh +++ b/test/test_xml.sh @@ -163,7 +163,7 @@ XML=$(cat < EOF ) -expecteof "$clixon_util_xml -o" 0 "$XML" "$XML" +expecteof "$clixon_util_xml -o" 0 "$XML" '^Frobnosticationhere.$' new "Second example 6.1 from https://www.w3.org/TR/2009/REC-xml-names-20091208" XML=$(cat < EOF ) -expecteof "$clixon_util_xml -o" 0 "$XML" "$XML" - +expecteof "$clixon_util_xml -o" 0 "$XML" '^Cheaper by the Dozen1568491379$' + rm -rf $dir # unset conditional parameters diff --git a/util/clixon_util_path.c b/util/clixon_util_path.c index ae79483a..b1028f6f 100644 --- a/util/clixon_util_path.c +++ b/util/clixon_util_path.c @@ -102,11 +102,20 @@ main(int argc, cbuf *cb = NULL; int api_path_p = 0; /* api-path or instance-id */ clicon_handle h; - struct stat st; + struct stat st; + cxobj *xcfg = NULL; + /* In the startup, logs to stderr & debug flag set later */ clicon_log_init("api-path", LOG_DEBUG, CLICON_LOG_STDERR); + /* Initialize clixon handle */ if ((h = clicon_handle_init()) == NULL) goto done; + /* Initialize config tree (needed for -Y below) */ + if ((xcfg = xml_new("clixon-config", NULL, NULL)) == NULL) + goto done; + if (clicon_conf_xml_set(h, xcfg) < 0) + goto done; + optind = 1; opterr = 0; while ((c = getopt(argc, argv, UTIL_PATH_OPTS)) != -1) @@ -121,7 +130,7 @@ main(int argc, case 'f': /* XML file */ filename = optarg; if ((fd = open(filename, O_RDONLY)) < 0){ - clicon_err(OE_UNIX, errno, "open(%s)", argv[1]); + clicon_err(OE_UNIX, errno, "open(%s)", optarg); goto done; } break; diff --git a/util/clixon_util_xml.c b/util/clixon_util_xml.c index 2693c8c8..025a5ede 100644 --- a/util/clixon_util_xml.c +++ b/util/clixon_util_xml.c @@ -113,8 +113,14 @@ main(int argc, /* In the startup, logs to stderr & debug flag set later */ clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR); + /* Initialize clixon handle */ if ((h = clicon_handle_init()) == NULL) goto done; + if ((xcfg = xml_new("clixon-config", NULL, NULL)) == NULL) + goto done; + if (clicon_conf_xml_set(h, xcfg) < 0) + goto done; + xcfg = xml_new("clixon-config", NULL, NULL); clicon_conf_xml_set(h, xcfg); diff --git a/util/clixon_util_xpath.c b/util/clixon_util_xpath.c index daa6246e..85e71b90 100644 --- a/util/clixon_util_xpath.c +++ b/util/clixon_util_xpath.c @@ -136,12 +136,19 @@ main(int argc, struct stat st; cvec *nsc = NULL; int canonical = 0; + cxobj *xcfg = NULL; + /* In the startup, logs to stderr & debug flag set later */ clicon_log_init("xpath", LOG_DEBUG, CLICON_LOG_STDERR); - + /* Initialize clixon handle */ if ((h = clicon_handle_init()) == NULL) goto done; - + /* Initialize config tree (needed for -Y below) */ + if ((xcfg = xml_new("clixon-config", NULL, NULL)) == NULL) + goto done; + if (clicon_conf_xml_set(h, xcfg) < 0) + goto done; + optind = 1; opterr = 0; while ((c = getopt(argc, argv, XPATH_OPTS)) != -1)