From 451adfaf1f426807fad432625201a0d8768070ef Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Fri, 28 Feb 2020 12:16:16 +0100 Subject: [PATCH] * Explicit search indexes * Added object-based `clixon_xvec` as a new programming construct for contiguous XML object vectors. --- CHANGELOG.md | 13 +- include/clixon_custom.h | 10 +- lib/clixon/clixon.h.in | 1 + lib/clixon/clixon_xml.h | 12 +- lib/clixon/clixon_xml_sort.h | 6 +- lib/clixon/clixon_xml_vec.h | 60 +++++ lib/src/Makefile.in | 2 +- lib/src/clixon_instance_id_parse.y | 2 +- lib/src/clixon_path.c | 5 +- lib/src/clixon_xml.c | 300 +++++++++++++++++++++-- lib/src/clixon_xml_map.c | 28 ++- lib/src/clixon_xml_sort.c | 379 +++++++++++++++++++++-------- lib/src/clixon_xml_vec.c | 315 ++++++++++++++++++++++++ test/test_openconfig.sh | 4 - test/test_search_index.sh | 48 ++-- util/clixon_util_path.c | 32 ++- 16 files changed, 1052 insertions(+), 165 deletions(-) create mode 100644 lib/clixon/clixon_xml_vec.h create mode 100644 lib/src/clixon_xml_vec.c diff --git a/CHANGELOG.md b/CHANGELOG.md index bb548d31..f98f1db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ Expected: February 2020 * Called on startup after initial XML parsing, but before module-specific upgrades * Enabled by definign the `.ca_datastore_upgrade` * [General-purpose upgrade documentation](https://clixon-docs.readthedocs.io/en/latest/backend.html#general-purpose) -* New and updated search functions using xpath, api-path and instance-id +* New and updated search functions using xpath, api-path and instance-id, and explicit indexes * New search functions using api-path and instance_id: * C search functions: `clixon_find_instance_id()` and `clixon_find_api_path()` * Binary search optimization in lists for indexed leafs in all three formats. @@ -37,7 +37,6 @@ Expected: February 2020 * You can also register explicit indexes for making binary search (not only list keys) * For more info, see docs at [paths](https://clixon-docs.readthedocs.io/en/latest/paths.html) and [search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml) - * Experimental: explicit search index, ie index any list variable, not just keys ### API changes on existing features (you may need to change your code) * New clixon-config@2020-02-22.yang revision @@ -59,11 +58,15 @@ Expected: February 2020 * CLI Error message (clicon_rpc_generate_error()) changed when backend returns netconf error to be more descriptive: * Original: `Config error: Validate failed. Edit and try again or discard changes: Invalid argument` * New (example): `Netconf error: application operation-failed Identityref validation failed, undefined not derived from acl-base . Validate failed. Edit and try again or discard changes" - +* Obsoleted and removed XMLDB format "tree". This function did not work. Only xml and json allowed. + ### Minor changes -* C-API: Added instrumentation: `xml_stats` and `xml_stats_global`. -* Obsoleted and removed XMLDB format "tree". This function did not work. Only xml and json allowed. +* C-API: + * Added instrumentation: `xml_stats` and `xml_stats_global`. + * Added object-based `clixon_xvec` as a new programming construct for contiguous XML object vectors. + * See files: `clixon_xml_vec.[ch]` + * Plan is to replace `cxobj **` with `clixon_xvec *` going forward. * Test framework * Added `-- -S ` command-line to main example to be able to return any state to main example. * Added `test/cicd` test scripts for running on a set of other hosts diff --git a/include/clixon_custom.h b/include/clixon_custom.h index 0e6ac62c..409ca911 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -32,7 +32,11 @@ ***** END LICENSE BLOCK ***** Custom file as boilerplate appended by clixon_config.h - Note that clixon_config.h is only included by clixon system files, not automatically by examples or apps + These are compile-time options. RUntime options are in clixon-config.yang. + In general they are kludges and "should be removed" when cod eis improved + and not proper system config options. + Note that clixon_config.h is only included by clixon system files, not automatically by examples + or apps */ #ifndef HAVE_STRNDUP @@ -72,7 +76,7 @@ * This also applies if there are multiple keys and you want to search on only the second for * example. */ -#undef XML_EXPLICIT_INDEX +#define XML_EXPLICIT_INDEX /*! Validate user state callback content * Users may register state callbacks using ca_statedata callback @@ -87,7 +91,7 @@ /*! Treat specially in a xmldb datastore. * config is treated as a "neutral" tag that does not have a yang spec. - * In particulat when binding xml to yang, if is encountered as top-of-tree, do not + * In particular when binding xml to yang, if is encountered as top-of-tree, do not * try to bind a yang-spec to this symbol. */ #define XMLDB_CONFIG_HACK diff --git a/lib/clixon/clixon.h.in b/lib/clixon/clixon.h.in index cb81a4e5..6e6513d0 100644 --- a/lib/clixon/clixon.h.in +++ b/lib/clixon/clixon.h.in @@ -95,6 +95,7 @@ #include #include #include +#include /* * Global variables generated by Makefile diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 6f531d2c..933777b2 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -33,7 +33,7 @@ ***** END LICENSE BLOCK ***** - * XML support functions. + * Clixon XML object (cxobj) support functions. * @see https://www.w3.org/TR/2008/REC-xml-20081126 * https://www.w3.org/TR/2009/REC-xml-names-20091208/ * Canonical XML version (just for info) @@ -105,6 +105,8 @@ typedef struct xml cxobj; /* struct defined in clicon_xml.c */ */ typedef int (xml_applyfn_t)(cxobj *x, void *arg); +typedef struct clixon_xml_vec clixon_xvec; /* struct defined in clicon_xvec.c */ + /* * xml_flag() flags: */ @@ -227,6 +229,14 @@ int xml_attr_insert2val(char *instr, enum insert_type *ins); int clicon_log_xml(int level, cxobj *x, char *format, ...) __attribute__ ((format (printf, 3, 4))); #else int clicon_log_xml(int level, cxobj *x, char *format, ...); +#endif +#ifdef XML_EXPLICIT_INDEX +int xml_search_index_p(cxobj *x); + +int xml_search_vector_get(cxobj *x, char *name, clixon_xvec **xvec); +int xml_search_child_insert(cxobj *xp, cxobj *x); +int xml_search_child_rm(cxobj *xp, cxobj *x); + #endif #endif /* _CLIXON_XML_H */ diff --git a/lib/clixon/clixon_xml_sort.h b/lib/clixon/clixon_xml_sort.h index 69aa67d8..0b75da5b 100644 --- a/lib/clixon/clixon_xml_sort.h +++ b/lib/clixon/clixon_xml_sort.h @@ -40,10 +40,14 @@ /* * Prototypes */ -int xml_cmp(cxobj *x1, cxobj *x2, int same, int skip1); +int xml_cmp(cxobj *x1, cxobj *x2, int same, int skip1, char *explicit); int xml_sort(cxobj *x0, void *arg); int xml_insert(cxobj *xp, cxobj *xc, enum insert_type ins, char *key_val, cvec *nsckey); int xml_sort_verify(cxobj *x, void *arg); +#ifdef XML_EXPLICIT_INDEX +int xml_search_indexvar_binary_pos(cxobj *xp, char *indexvar, clixon_xvec *xvec, + int low, int upper, int max, int *eq); +#endif int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp); int clixon_xml_find_index(cxobj *xp, yang_stmt *yp, char *namespace, char *name, cvec *cvk, cxobj ***xvec, size_t *xlen); diff --git a/lib/clixon/clixon_xml_vec.h b/lib/clixon/clixon_xml_vec.h new file mode 100644 index 00000000..79e3a268 --- /dev/null +++ b/lib/clixon/clixon_xml_vec.h @@ -0,0 +1,60 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2019 Olof Hagsand + Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * Clixon XML object vectors + */ +#ifndef _CLIXON_XML_VEC_H +#define _CLIXON_XML_VEC_H + +/* + * Types + */ +typedef struct clixon_xml_vec clixon_xvec; /* struct defined in clicon_xvec.c */ + +/* + * Prototypes + */ +clixon_xvec *clixon_xvec_new(void); +clixon_xvec *clixon_xvec_dup(clixon_xvec *xv0); +int clixon_xvec_free(clixon_xvec *xv); +int clixon_xvec_len(clixon_xvec *xv); +cxobj *clixon_xvec_i(clixon_xvec *xv, int i); +int clixon_xvec_append(clixon_xvec *xv, cxobj *x); +int clixon_xvec_prepend(clixon_xvec *xv, cxobj *x); +int clixon_xvec_insert_pos(clixon_xvec *xv, cxobj *x, int i); +int clixon_xvec_rm_pos(clixon_xvec *xv, int i); +int clixon_xvec_print(FILE *f, clixon_xvec *xv); + +#endif /* _CLIXON_XML_VEC_H */ diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index 4dc2be88..183451c9 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -69,7 +69,7 @@ INCLUDES = -I. @INCLUDES@ -I$(top_srcdir)/lib/clixon -I$(top_srcdir)/include -I$ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_err.c clixon_event.c \ clixon_string.c clixon_regex.c clixon_handle.c clixon_file.c \ - clixon_xml.c clixon_xml_sort.c clixon_xml_map.c clixon_json.c \ + clixon_xml.c clixon_xml_sort.c clixon_xml_map.c clixon_xml_vec.c clixon_json.c \ clixon_yang.c clixon_yang_type.c clixon_yang_module.c clixon_yang_parse_lib.c \ clixon_yang_cardinality.c clixon_xml_changelog.c clixon_xml_nsctx.c \ clixon_path.c clixon_validate.c \ diff --git a/lib/src/clixon_instance_id_parse.y b/lib/src/clixon_instance_id_parse.y index 53f9be31..aac7fbaa 100644 --- a/lib/src/clixon_instance_id_parse.y +++ b/lib/src/clixon_instance_id_parse.y @@ -357,7 +357,7 @@ key_pred : LSQBR key_pred_expr RSQBR { $$ = $2; clicon_debug(2,"key_pred = [ key_pred_expr ]"); } ; -key_pred_expr : node_id_k EQUAL qstring { $$ = keyval_set($1, $3); free($1); +key_pred_expr : node_id_k EQUAL qstring { $$ = keyval_set($1, $3); free($1); free($3); clicon_debug(2,"key_pred_expr = node_id_k = qstring"); } ; diff --git a/lib/src/clixon_path.c b/lib/src/clixon_path.c index 2033999c..f4e29f95 100644 --- a/lib/src/clixon_path.c +++ b/lib/src/clixon_path.c @@ -1441,8 +1441,9 @@ clixon_path_search(cxobj *xt, if (clixon_xml_find_pos(xp, yc, cv_uint32_get(cv), &xvecc, &xlenc) < 0) goto done; } - else if (clixon_xml_find_index(xp, yang_parent_get(yc), modns, yang_argument_get(yc), - cp->cp_cvk, &xvecc, &xlenc) < 0) + else if (clixon_xml_find_index(xp, yang_parent_get(yc), + modns, yang_argument_get(yc), + cp->cp_cvk, &xvecc, &xlenc) < 0) goto done; } if (xvecp) diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index 34345885..a32cb619 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -33,7 +33,7 @@ ***** END LICENSE BLOCK ***** - * XML support functions. + * Clixon XML object (cxobj) support functions. * @see https://www.w3.org/TR/2008/REC-xml-20081126 * https://www.w3.org/TR/2009/REC-xml-names-20091208 * Canonical XML version (just for info) @@ -69,6 +69,7 @@ #include "clixon_options.h" /* xml_spec_populate */ #include "clixon_yang_module.h" #include "clixon_xml_map.h" /* xml_spec_populate */ +#include "clixon_xml_vec.h" #include "clixon_xml_sort.h" #include "clixon_xml_parse.h" #include "clixon_xml_nsctx.h" @@ -91,6 +92,35 @@ * Types */ +#ifdef XML_EXPLICIT_INDEX +static int xml_search_index_free(cxobj *x); + +/* A search index pair consisting of a name of an (index) variable and a vector of xml children + * the variable should be a potential child of the XML node + * The vector should have the same elements as the regular XML childvec, but in different order + * + * +-----+-----+-----+ + * search index vector i: | b | c | a | + * +-----+-----+-----+ + * + * index: "i" + * +-----+-----+-----+ + * x_childvec: | a | b | c | + * +-----+-----+-----+ + * | | | + * v v v + * +---+ +---+ +---+ + * value of "i" | 5 | | 0 | | 2 | + * +---+ +---+ +---+ + + */ +struct search_index{ + qelem_t si_q; /* Queue header */ + char *si_name; /* Name of index variable (must be (potential) child of xml node at hand */ + clixon_xvec *si_xvec; /* Sorted vector of xml object pointers (should be of YANG type LIST) */ +}; +#endif + /*! xml tree node, with name, type, parent, children, etc * Note that this is a private type not visible from externally, use * access functions. @@ -128,20 +158,23 @@ struct xml{ char *x_name; /* name of node */ char *x_prefix; /* namespace localname N, called prefix */ struct xml *x_up; /* parent node in hierarchy if any */ - struct xml **x_childvec; /* vector of children nodes */ + struct xml **x_childvec; /* vector of children nodes (XXX: use clixon_vec ) */ int x_childvec_len;/* Number of children */ int x_childvec_max;/* Length of allocated vector */ enum cxobj_type x_type; /* type of node: element, attribute, body */ - cbuf *x_value_cb; /* attribute and body nodes have values */ + cbuf *x_value_cb; /* attribute and body nodes have values (XXX: this consumes + memory) cv? */ int x_flags; /* Flags according to XML_FLAG_* */ yang_stmt *x_spec; /* Pointer to specification, eg yang, by reference, dont free */ - cg_var *x_cv; /* Cached value as cligen variable - (eg xml_cmp) */ + cg_var *x_cv; /* Cached value as cligen variable (eg xml_cmp) */ cvec *x_ns_cache; /* Cached vector of namespaces */ int _x_vector_i; /* internal use: xml_child_each */ int _x_i; /* internal use for sorting: see xml_enumerate and xml_cmp */ +#ifdef XML_EXPLICIT_INDEX + struct search_index *x_search_index; /* explicit search index vectors */ +#endif }; /* @@ -961,12 +994,13 @@ xml_child_append(cxobj *x, cxobj *xc) { x->x_childvec_len++; - if (x->x_childvec_len > x->x_childvec_max) + if (x->x_childvec_len > x->x_childvec_max){ x->x_childvec_max = x->x_childvec_max?2*x->x_childvec_max:XML_CHILDVEC_MAX_DEFAULT; - x->x_childvec = realloc(x->x_childvec, x->x_childvec_max*sizeof(cxobj*)); - if (x->x_childvec == NULL){ - clicon_err(OE_XML, errno, "realloc"); - return -1; + x->x_childvec = realloc(x->x_childvec, x->x_childvec_max*sizeof(cxobj*)); + if (x->x_childvec == NULL){ + clicon_err(OE_XML, errno, "realloc"); + return -1; + } } x->x_childvec[x->x_childvec_len-1] = xc; return 0; @@ -985,12 +1019,13 @@ xml_child_insert_pos(cxobj *xp, size_t size; xp->x_childvec_len++; - if (xp->x_childvec_len > xp->x_childvec_max) + if (xp->x_childvec_len > xp->x_childvec_max){ xp->x_childvec_max = xp->x_childvec_max?2*xp->x_childvec_max:XML_CHILDVEC_MAX_DEFAULT; - xp->x_childvec = realloc(xp->x_childvec, xp->x_childvec_max*sizeof(cxobj*)); - if (xp->x_childvec == NULL){ - clicon_err(OE_XML, errno, "realloc"); - return -1; + xp->x_childvec = realloc(xp->x_childvec, xp->x_childvec_max*sizeof(cxobj*)); + if (xp->x_childvec == NULL){ + clicon_err(OE_XML, errno, "realloc"); + return -1; + } } size = (xml_child_nr(xp) - i - 1)*sizeof(cxobj *); memmove(&xp->x_childvec[i+1], &xp->x_childvec[i], size); @@ -1197,6 +1232,10 @@ xml_addsub(cxobj *xp, } /* clear namespace context cache of child */ nscache_clear(xc); +#ifdef XML_EXPLICIT_INDEX + if (xml_search_index_p(xc)) + xml_search_child_insert(xp, xc); +#endif } retval = 0; done: @@ -1309,6 +1348,10 @@ xml_child_rm(cxobj *xp, clicon_err(OE_XML, 0, "Child not found"); goto done; } +#ifdef XML_EXPLICIT_INDEX + if (xml_search_index_p(xc)) + xml_search_child_rm(xp, xc); +#endif xp->x_childvec[i] = NULL; xml_parent_set(xc, NULL); xp->x_childvec_len--; @@ -1729,6 +1772,9 @@ xml_free(cxobj *x) cv_free(x->x_cv); if (x->x_ns_cache) xml_nsctx_free(x->x_ns_cache); +#ifdef XML_EXPLICIT_INDEX + xml_search_index_free(x); +#endif free(x); _stats_nr--; return 0; @@ -2490,6 +2536,7 @@ xml_dup(cxobj *x0) return x1; } +#if 1 /* XXX At some point migrate this code to the clixon_xml_vec.[ch] API */ /*! Copy XML vector from vec0 to vec1 * @param[in] vec0 Source XML tree vector * @param[in] len0 Length of source XML tree vector @@ -2584,6 +2631,7 @@ cxvec_prepend(cxobj *x, done: return retval; } +#endif /*! Apply a function call recursively on all xml node children recursively * Recursively traverse all xml nodes in a parse-tree and apply fn(arg) for @@ -2978,3 +3026,225 @@ clicon_log_xml(int level, free(msg); return retval; } + +#ifdef XML_EXPLICIT_INDEX +/* + * + */ + +/*! Is this XML object a search index, ie it is registered as a yang clixon cc:search_index + * Is this xml node a search index and does it have a parent that is a list and a grandparent + * where a search-vector can be placed + * @param[in] x XML object + * @retval 1 Yes + * @retval 0 No + */ +int +xml_search_index_p(cxobj *x) +{ + yang_stmt *y; + cxobj *xp; + + /* The index variable has a yang spec */ + if ((y = xml_spec(x)) == NULL) + return 0; + /* The index variable is a registered search index */ + if (yang_flag_get(y, YANG_FLAG_INDEX) == 0) + return 0; + /* The index variable has a parent which has a LIST yang spec */ + if ((xp = xml_parent(x)) == NULL) + return 0; + if ((y = xml_spec(xp)) == NULL) + return 0; + if (yang_keyword_get(y) != Y_LIST) + return 0; + /* The index variable has a grand-parent */ + if (xml_parent(xp) == NULL) + return 0; + return 1; +} + + +/*! Free all search vector pairs of this XML node + * @param[in] x XML object + * @retval 0 OK + * @retval -1 Error + */ +static int +xml_search_index_free(cxobj *x) +{ + struct search_index *si; + + while ((si = x->x_search_index) != NULL) { + DELQ(si, x->x_search_index, struct search_index *); + if (si->si_name) + free(si->si_name); + if (si->si_xvec) + clixon_xvec_free(si->si_xvec); + free(si); + } + return 0; +} + +/*! Add single search vector pair to this XML node + * @param[in] x XML object + * @param[in] name Name of index variable + * @retval 0 OK + * @retval -1 Error + */ +static struct search_index * +xml_search_index_add(cxobj *x, + char *name) +{ + struct search_index *si = NULL; + + if ((si = malloc(sizeof(struct search_index))) == NULL){ + clicon_err(OE_XML, errno, "malloc"); + goto done; + } + memset(si, 0, sizeof(struct search_index)); + if ((si->si_name = strdup(name)) == NULL){ + clicon_err(OE_XML, errno, "strdup"); + free(si); + si = NULL; + goto done; + } + if ((si->si_xvec = clixon_xvec_new()) == NULL){ + free(si->si_name); + free(si); + si = NULL; + goto done; + } + ADDQ(si, x->x_search_index); + done: + return si; +} + +/*! Add single search vector pair to this XML node + * @param[in] x XML object + * @param[in] name Name of index variable + * @retval 0 OK + * @retval -1 Error + */ +static struct search_index * +xml_search_index_get(cxobj *x, + char *name) +{ + struct search_index *si = NULL; + + if ((si = x->x_search_index) != NULL) { + do { + if (strcmp(si->si_name, name) == 0){ + goto done; + break; + } + si = NEXTQ(struct search_index *, si); + } while (si && si != x->x_search_index); + } + done: + return si; +} + +/*--------------------------------------------------*/ + +/*! Get sorted index vector for list for variable "name" + * @param[in] xp XML parent object + * @param[in] name Name of index variable + * @param[out] xvec XML object search vector + * @retval 0 OK + */ +int +xml_search_vector_get(cxobj *xp, + char *name, + clixon_xvec **xvec) +{ + struct search_index *si; + + *xvec = NULL; + if ((si = xp->x_search_index) != NULL) { + do { + if (strcmp(si->si_name, name) == 0){ + *xvec = si->si_xvec; + break; + } + si = NEXTQ(struct search_index *, si); + } while (si && si != xp->x_search_index); + } + return 0; +} + +/*! Insert a new cxobj into search index vector for list for variable "name" + * @param[in] xp XML parent object (the list element) + * @param[in] xi XML index object (that should be added) + */ +int +xml_search_child_insert(cxobj *xp, + cxobj *xi) +{ + int retval = -1; + char *indexvar; + struct search_index *si; + cxobj *xpp; + int i; + int len; + + indexvar = xml_name(xi); + if ((xpp = xml_parent(xp)) == NULL) + goto ok; + /* Find base vector in grandparent */ + if ((si = xml_search_index_get(xpp, indexvar)) == NULL){ + /* If not found add base vector in grand-parent */ + if ((si = xml_search_index_add(xpp, indexvar)) == NULL) + goto done; + } + /* Find element position using binary search and then remove */ + len = clixon_xvec_len(si->si_xvec); + if ((i = xml_search_indexvar_binary_pos(xp, indexvar, si->si_xvec, 0, len, len, NULL)) < 0) + goto done; + assert(clixon_xvec_i(si->si_xvec, i) != xp); + if (clixon_xvec_insert_pos(si->si_xvec, xp, i) < 0) + goto done; + ok: + retval = 0; + done: + return retval; +} + +/*! Remove a single cxobj from search vector + * @param[in] xp XML parent object (the list element) + * @param[in] xi XML index object (that should be added) + */ +int +xml_search_child_rm(cxobj *xp, + cxobj *xi) +{ + int retval = -1; + cxobj *xpp; + char *indexvar; + int i; + int len; + struct search_index *si; + int eq = 0; + + indexvar = xml_name(xi); + if ((xpp = xml_parent(xp)) == NULL) + goto ok; + /* Find base vector in grandparent */ + if ((si = xml_search_index_get(xpp, indexvar)) == NULL) + goto ok; + + /* Find element using binary search and then remove */ + len = clixon_xvec_len(si->si_xvec); + if ((i = xml_search_indexvar_binary_pos(xp, indexvar, si->si_xvec, 0, len, len, &eq)) < 0) + goto done; + // if (clixon_xvec_i(si->si_xvec, i) == xp) + if (eq) + if (clixon_xvec_rm_pos(si->si_xvec, i) < 0) + goto done; + ok: + retval = 0; + done: + return retval; +} + +#endif /* XML_EXPLICIT_INDEX */ diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index c4311d11..5eb383a4 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -479,7 +479,7 @@ xml_diff1(yang_stmt *ys, continue; } /* Both x0c and x1c exists, check if they are equal. */ - eq = xml_cmp(x0c, x1c, 0, 0); + eq = xml_cmp(x0c, x1c, 0, 0, NULL); if (eq < 0){ if (cxvec_append(x0c, x0vec, x0veclen) < 0) goto done; @@ -1214,6 +1214,10 @@ populate_self_parent(cxobj *xt, goto fail; } xml_spec_set(xt, y); +#ifdef XML_EXPLICIT_INDEX + if (xml_search_index_p(xt)) + xml_search_child_insert(xp, xt); +#endif retval = 2; done: return retval; @@ -1292,12 +1296,12 @@ populate_self_top(cxobj *xt, goto done; } -/*! Find yang spec association of XML node +/*! Find yang spec association of tree of XML nodes * - * This may be unnecessary if yspec is set on manual creation. Also note that for incoming or outgoing RPC - * need specialized function. - * XXX: maybe it can be built into the same function, but you dont know whether it is input or output rpc, so - * it seems like you need specialized functions. + * Populate xt:s children as top-level symbols + * This may be unnecessary if yspec is set on manual creation. Also note that for incoming or + * outgoing RPC need specialized function. maybe it can be built into the same function, but + * you dont know whether it is input or output rpc. * @param[in] xt XML tree node * @param[in] yspec Yang spec * @param[out] xerr Reason for failure, or NULL @@ -1333,6 +1337,10 @@ xml_spec_populate(cxobj *xt, return retval; } +/*! Find yang spec association of tree of XML nodes + * + * Populate xt:s children outgoing from that xt is populated + */ int xml_spec_populate_parent(cxobj *xt, cxobj **xerr) @@ -1354,6 +1362,10 @@ xml_spec_populate_parent(cxobj *xt, return retval; } +/*! Find yang spec association of tree of XML nodes + * + * Populate xt as top-level node + */ int xml_spec_populate0(cxobj *xt, yang_stmt *yspec, @@ -1380,6 +1392,10 @@ xml_spec_populate0(cxobj *xt, return retval; } +/*! Find yang spec association of tree of XML nodes + * + * Populate xt as if xt:s parent is populated + */ int xml_spec_populate0_parent(cxobj *xt, cxobj **xerr) diff --git a/lib/src/clixon_xml_sort.c b/lib/src/clixon_xml_sort.c index 1a991c56..080a3637 100644 --- a/lib/src/clixon_xml_sort.c +++ b/lib/src/clixon_xml_sort.c @@ -68,6 +68,7 @@ #include "clixon_xml_map.h" #include "clixon_yang_type.h" #include "clixon_yang_module.h" +#include "clixon_xml_vec.h" #include "clixon_xml_sort.h" /*! Get xml body value as cligen variable @@ -142,6 +143,7 @@ xml_cv_cache(cxobj *x, * @param[in] same If set, x1 and x2 are member of same parent & enumeration * is used (see explanation below) * @param[in] skip1 Key matching skipped for keys not in x1 (see explanation) + * @param[in] explicit For list nodes, use explicit index variables, not keys * @retval 0 If equal * @retval <0 If x1 is less than x2 * @retval >0 If x1 is greater than x2 @@ -175,10 +177,11 @@ xml_cv_cache(cxobj *x, * but is not equal to 71bar */ int -xml_cmp(cxobj *x1, - cxobj *x2, - int same, - int skip1) +xml_cmp(cxobj *x1, + cxobj *x2, + int same, + int skip1, + char *indexvar) { yang_stmt *y1; yang_stmt *y2; @@ -274,8 +277,41 @@ xml_cmp(cxobj *x1, equal = 1; } break; - case Y_LIST: /* Match with key values - * Use Y_LIST cache (see struct yang_stmt) */ + case Y_LIST: /* Match with key values */ + if (indexvar != NULL){ +#ifdef XML_EXPLICIT_INDEX + x1b = xml_find(x1, indexvar); + x2b = xml_find(x2, indexvar); + if (x1b == NULL && x2b == NULL) + ; + else if (x1b == NULL) + equal = -1; + else if (x2b == NULL) + equal = 1; + else{ + b1 = xml_body(x1b); + b2 = xml_body(x2b); + if (b1 == NULL && b2 == NULL) + ; + else if (b1 == NULL) + equal = -1; + else if (b2 == NULL) + equal = 1; + else{ + if (xml_cv_cache(x1b, &cv1) < 0) /* error case */ + goto done; + if (xml_cv_cache(x2b, &cv2) < 0) /* error case */ + goto done; + assert(cv1 && cv2); + equal = cv_cmp(cv1, cv2); + } + } + if (equal) + break; +#endif /* XML_EXPLICIT_INDEX */ + } + else { + /* Use Y_LIST cache (see struct yang_stmt) */ cvk = yang_cvec_get(y1); /* Use Y_LIST cache, see ys_populate_list() */ cvi = NULL; while ((cvi = cvec_each(cvk, cvi)) != NULL) { @@ -312,6 +348,7 @@ xml_cmp(cxobj *x1, if (equal) break; } /* while cvi */ + } break; default: /* This is a very special case such as for two choices - which is not validation correct, but we @@ -332,7 +369,7 @@ static int xml_cmp_qsort(const void* arg1, const void* arg2) { - return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1, 0); + return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1, 0, NULL); } /*! Sort children of an XML node @@ -389,7 +426,7 @@ xml_find_keys_notsorted(cxobj *xp, yc = xml_spec(xc); if (yangi != yang_order(yc)) /* wrong yang */ break; - if (xml_cmp(xc, x1, 0, skip1) == 0){ + if (xml_cmp(xc, x1, 0, skip1, NULL) == 0){ if (cxvec_append(xc, xvec, xlen) < 0) goto done; goto ok; /* found */ @@ -400,7 +437,7 @@ xml_find_keys_notsorted(cxobj *xp, yc = xml_spec(xc); if (yangi != yang_order(yc)) /* wrong yang */ break; - if (xml_cmp(xc, x1, 0, skip1) == 0){ + if (xml_cmp(xc, x1, 0, skip1, NULL) == 0){ if (cxvec_append(xc, xvec, xlen) < 0) goto done; goto ok; /* found */ @@ -413,23 +450,25 @@ xml_find_keys_notsorted(cxobj *xp, } /*! Find more equal objects in a vector up and down in the array of the present - * @param[in] xp Parent XML node (go through its childre) - * @param[in] x1 XML node to match - * @param[in] yangi Yang order number (according to spec) - * @param[in] mid Where to start from (may be in middle of interval) - * @param[out] xvec Vector of matching XML return objects (can be empty) - * @param[out] xlen Length of xvec - * @retval 0 OK, see xvec (may be empty) - * @retval -1 Error + * @param[in] childvec Vector of children of parent + * @param[in] childlen Length of child vector + * @param[in] x1 XML node to match + * @param[in] yangi Yang order number (according to spec) + * @param[in] mid Where to start from (may be in middle of interval) + * @param[out] xvec Vector of matching XML return objects (can be empty) + * @param[out] xlen Length of xvec + * @retval 0 OK, see xvec (may be empty) + * @retval -1 Error */ static int -more_equals(cxobj *xp, - cxobj *x1, - int yangi, - int mid, - int skip1, - cxobj ***xvec, - size_t *xlen) +search_multi_equals(cxobj **childvec, + size_t childlen, + cxobj *x1, + int yangi, + int mid, + int skip1, + cxobj ***xvec, + size_t *xlen) { int retval = -1; int i; @@ -437,21 +476,21 @@ more_equals(cxobj *xp, yang_stmt *yc; for (i=mid-1; i>=0; i--){ /* First decrement */ - xc = xml_child_i(xp, i); + xc = childvec[i]; yc = xml_spec(xc); if (yangi != yang_order(yc)) /* wrong yang */ break; - if (xml_cmp(x1, xc, 0, skip1) != 0) + if (xml_cmp(x1, xc, 0, skip1, NULL) != 0) break; if (cxvec_prepend(xc, xvec, xlen) < 0) goto done; } - for (i=mid+1; i=0; i--){ /* First decrement */ + xc = clixon_xvec_i(childvec, i); + yc = xml_spec(xc); + if (yangi != yang_order(yc)) /* wrong yang */ + break; + if (xml_cmp(x1, xc, 0, skip1, NULL) != 0) + break; + if (cxvec_prepend(xc, xvec, xlen) < 0) + goto done; + } + for (i=mid+1; iupper %d %d", low, upper); + goto done; + } + if (low == upper){ + retval = low; + goto done; + } + mid = (low + upper) / 2; + if (mid >= max){ /* beyond range */ + clicon_err(OE_XML, 0, "Beyond range %d %d %d", low, mid, upper); + goto done; + } + xc = clixon_xvec_i(ivec, mid); + /* >0 means search upper interval, <0 lower interval, = 0 is equal */ + cmp = xml_cmp(x1, xc, 0, 0, indexvar); + if (low +1 == upper){ /* termination criterium */ + } + if (cmp == 0){ + retval = mid; /* equal */ + if (eq) + *eq = 1; + goto done; + } + else { + if (low +1 == upper){ /* termination criterium */ + if (eq) /* No exact match */ + *eq = 0; + if (cmp < 0) /* return either 0 if cmp<0 or +1 if cmp>0 */ + retval = mid; + else + retval = mid+1; + goto done; + } + if (cmp < 0) + return xml_search_indexvar_binary_pos(x1, indexvar, ivec, low, mid, max, eq); + else + return xml_search_indexvar_binary_pos(x1, indexvar, ivec, mid+1, upper, max, eq); + } + done: + return retval; +} +#endif /* XML_EXPLICIT_INDEX */ + /*! Find XML child under xp matching x1 using binary search * @param[in] xp Parent xml node. * @param[in] x1 Find this object among xp:s children @@ -470,21 +621,23 @@ more_equals(cxobj *xp, * @param[in] low Lower bound of childvec search interval * @param[in] upper Lower bound of childvec search interval * @param[in] skip1 Key matching skipped for keys not in x1 + * @param[in] indexvar Override list key value search with explicit search index of x1 * @param[out] xvec Vector of matching XML return objects (can be empty) * @param[out] xlen Length of xvec * @retval 0 OK, see xvec (may be empty) * @retval -1 Error */ static int -xml_find_keys1(cxobj *xp, - cxobj *x1, - int sorted, - int yangi, - int low, - int upper, - int skip1, - cxobj ***xvec, - size_t *xlen) +xml_search_binary(cxobj *xp, + cxobj *x1, + int sorted, + int yangi, + int low, + int upper, + int skip1, + char *indexvar, + cxobj ***xvec, + size_t *xlen) { int retval = -1; int mid; @@ -503,7 +656,42 @@ xml_find_keys1(cxobj *xp, cmp = yangi-yang_order(y); /* Here is right yang order == same yang? */ if (cmp == 0){ - cmp = xml_cmp(x1, xc, 0, skip1); +#ifdef XML_EXPLICIT_INDEX + if (indexvar != NULL){ + clixon_xvec *ivec = NULL; + int ilen; + int pos; + int eq = 0; + /* Check if (exactly one) explicit indexes in cvk */ + if (xml_search_vector_get(xp, indexvar, &ivec) < 0) + goto done; + if (ivec){ + ilen = clixon_xvec_len(ivec); +#if 1 /* XXX This needs some heavy testing */ + if ((pos = xml_search_indexvar_binary_pos(x1, indexvar, + ivec, 0, + ilen, ilen, &eq)) < 0) + goto done; + if (eq){ /* Found */ + if (cxvec_append(clixon_xvec_i(ivec,pos), xvec, xlen) < 0) + goto done; + /* there may be more? */ + if (search_multi_equals_xvec(ivec, x1, yangi, pos, + 0, xvec, xlen) < 0) + goto done; + } +#else + if (xml_search_indexvar_binary(x1, indexvar, + ivec, 0, + ilen, ilen, + xvec, xlen) < 0) + goto done; +#endif + } + goto ok; + } +#endif + cmp = xml_cmp(x1, xc, 0, skip1, NULL); if (cmp && !sorted){ /* Ordered by user (if not equal) */ retval = xml_find_keys_notsorted(xp, x1, yangi, mid, skip1, xvec, xlen); goto done; @@ -513,20 +701,22 @@ xml_find_keys1(cxobj *xp, if (cxvec_append(xc, xvec, xlen) < 0) goto done; /* there may be more? */ - if (more_equals(xp, x1, yangi, mid, skip1, xvec, xlen) < 0) + if (search_multi_equals(xml_childvec_get(xp), xml_child_nr(xp), + x1, yangi, mid, + skip1, xvec, xlen) < 0) goto done; } else if (cmp < 0) - xml_find_keys1(xp, x1, sorted, yangi, low, mid-1, skip1, xvec, xlen); + xml_search_binary(xp, x1, sorted, yangi, low, mid-1, skip1, indexvar, xvec, xlen); else - xml_find_keys1(xp, x1, sorted, yangi, mid+1, upper, skip1, xvec, xlen); + xml_search_binary(xp, x1, sorted, yangi, mid+1, upper, skip1, indexvar, xvec, xlen); ok: retval = 0; done: return retval; } -/*! Find XML child under xp matching x1 using binary search for list/leaf-list keys +/*! Search XML child under xp matching x1 using yang-based binary search for list/leaf-list keys * * Match is tried xp with x1 with either name only (container/leaf) or using keys (list/leaf-lists) * Any non-key leafs or other structure of x1 is not matched. @@ -535,6 +725,7 @@ xml_find_keys1(cxobj *xp, * @param[in] x1 Find this object among xp:s children * @param[in] yc Yang spec of x1 * @param[in] skip1 Key matching skipped for keys not in x1 + * @param[in] indexvar Override list key value search with explicit search index var of x1 * @param[out] xvec Vector of matching XML return objects (can be empty) * @param[out] xlen Length of xvec * @retval 0 OK, see xvec (may be empty) @@ -542,10 +733,11 @@ xml_find_keys1(cxobj *xp, * @see xml_find_index for a generic search function */ static int -xml_find_keys(cxobj *xp, +xml_search_yang(cxobj *xp, cxobj *x1, yang_stmt *yc, int skip1, + char *indexvar, cxobj ***xvec, size_t *xlen) { @@ -566,7 +758,7 @@ xml_find_keys(cxobj *xp, else if (yang_keyword_get(yc) == Y_LIST || yang_keyword_get(yc) == Y_LEAF_LIST) sorted = (yang_find(yc, Y_ORDERED_BY, "user") == NULL); yangi = yang_order(yc); - return xml_find_keys1(xp, x1, sorted, yangi, low, upper, skip1, xvec, xlen); + return xml_search_binary(xp, x1, sorted, yangi, low, upper, skip1, indexvar, xvec, xlen); } /*! Insert xn in xp:s sorted child list (special case of ordered-by user) @@ -724,7 +916,7 @@ xml_insert2(cxobj *xp, goto done; } else /* Ordered by system */ - cmp = xml_cmp(xn, xc, 0, 0); + cmp = xml_cmp(xn, xc, 0, 0, NULL); } else{ /* Not equal yang - compute diff */ cmp = yni - yang_order(yc); @@ -841,7 +1033,7 @@ xml_sort_verify(cxobj *x0, xml_enumerate_children(x0); while ((x = xml_child_each(x0, x, -1)) != NULL) { if (xprev != NULL){ /* Check xprev <= x */ - if (xml_cmp(xprev, x, 1, 0) > 0) + if (xml_cmp(xprev, x, 1, 0, NULL) > 0) goto done; } xprev = x; @@ -914,7 +1106,7 @@ match_base_child(cxobj *x0, break; } /* Get match. */ - if (xml_find_keys(x0, x1c, yc, 0, &xvec, &xlen) < 0) + if (xml_search_yang(x0, x1c, yc, 0, 0, &xvec, &xlen) < 0) goto done; if (xlen) *x0cp = xvec[0]; @@ -1028,7 +1220,20 @@ xml_find_noyang_name(cxobj *xp, return retval; } -/*! API for search in XML child list with yang available +/*! Try to find an XML child from parent with yang available using list keys and leaf-lists + * + * Must be populated with Yang specs, parent must be list or leaf-list, and (for list) search + * index must be keys in the order they are declared. + * First identify that this search qualifies for yang-based list/leaf-list optimized search, + * - if no, revert (return 0) so that the overlying algorithm can try next or fallback to + * linear seacrh + * - if yes, then construct a dummy search object and find it in the list of xp:s children + * using binary search + * @param[in] xp Parent xml node. + * @param[in] yc Yang spec of list child (preferred) See rule (2) above + * @param[in] cvk List of keys and values as CLIgen vector on the form k1=foo, k2=bar + * @param[out] xvec Array of found nodes + * @param[out] xlen Len of xvec * @retval 1 OK * @retval 0 Revert, try again with no-yang search * @retval -1 Error @@ -1051,6 +1256,8 @@ xml_find_index_yang(cxobj *xp, cg_var *ycv = NULL; int i; char *name; + int revert = 0; + char *indexvar = NULL; name = yang_argument_get(yc); if ((cb = cbuf_new()) == NULL){ @@ -1069,15 +1276,17 @@ xml_find_index_yang(cxobj *xp, goto done; } /* Parameter in cvk is not key or not in right key order, then we cannot call - * xml_find_keys and we need to revert to noyang + * xml_find_keys and we need to revert to noynag */ if ((ycv = cvec_i(ycvk, i)) == NULL || strcmp(kname, cv_string_get(ycv))){ - goto revert; + revert++; break; } cprintf(cb, "<%s>%s", kname, cv_string_get(cvi), kname); i++; } + if (revert) + break; cprintf(cb, "", name); break; case Y_LEAF_LIST: @@ -1092,7 +1301,24 @@ xml_find_index_yang(cxobj *xp, cprintf(cb, "<%s/>", name); break; } - clicon_debug(1, "%s XML:%s", __FUNCTION__, cbuf_get(cb)); +#ifdef XML_EXPLICIT_INDEX + if (revert){ + char *iname; + yang_stmt *yi; + if (cvec_len(cvk) != 1 || + (cvi = cvec_i(cvk, 0)) == NULL || + (iname = cv_name_get(cvi)) == NULL || + (yi = yang_find_datanode(yc, iname)) == NULL || + yang_flag_get(yi, YANG_FLAG_INDEX) == 0) + goto revert; + cbuf_reset(cb); + cprintf(cb, "<%s><%s>%s", name, iname, cv_string_get(cvi), iname, name); + indexvar = iname; + } +#else + if (revert) + goto revert; +#endif if (xml_parse_string(cbuf_get(cb), yc, &xc) < 0) goto done; if (xml_rootchild(xc, 0, &xc) < 0) @@ -1109,7 +1335,7 @@ xml_find_index_yang(cxobj *xp, if (xml_spec_set(xk, yk) < 0) goto done; } - if (xml_find_keys(xp, xc, yc, 1, xvec, xlen) < 0) + if (xml_search_yang(xp, xc, yc, 1, indexvar, xvec, xlen) < 0) goto done; retval = 1; /* OK */ done: @@ -1118,41 +1344,11 @@ xml_find_index_yang(cxobj *xp, if (xc) xml_free(xc); return retval; - revert: /* means try name-only*/ + revert: /* means give up yang/key search, try next (eg explicit/noyang) */ retval = 0; goto done; } -#ifdef XML_EXPLICIT_INDEX -/*! API for search in XML child list with yang available - * @retval 1 OK - * @retval 0 Revert, try again with no-yang search (or explicit index) - * @retval -1 Error - * XXX: can merge with xml_find_index_yang in some way, similar code - */ -static int -xml_find_index_explicit(cxobj *xp, - yang_stmt *yc, - cvec *cvk, - cxobj ***xvec, - size_t *xlen) -{ - int retval = -1; - cg_var *cvi; - - if (yang_keyword_get(yc) == Y_LIST && cvec_len(cvk) == 1){ - cvi = cvec_i(cvk, 0); - goto revert; - } - retval = 1; /* OK */ - done: - return retval; - revert: /* means try name-only*/ - retval = 0; - goto done; -} -#endif /* XML_EXPLICIT_INDEX */ - /*! API for search in XML child list using indexes and binary search if applicable * * Generic search function as alternative to xml_find() and others that finds YANG associated @@ -1180,7 +1376,7 @@ xml_find_index_explicit(cxobj *xp, * @param[in] xp Parent xml node. * @param[in] yc Yang spec of list child (preferred) See rule (2) above * @param[in] yp Yang spec of parent node or yang-spec/yang (Alternative if yc not given). - * @param[in] ns Namespace (needed only if name is derived from top-symbol) + * @param[in] namespace Namespace (needed only if name is derived from top-symbol) * @param[in] name Name of child (not required if yc given) * @param[in] cvk List of keys and values as CLIgen vector on the form k1=foo, k2=bar * @param[out] xvec Array of found nodes @@ -1192,15 +1388,13 @@ xml_find_index_explicit(cxobj *xp, * size_t xlen = 0; * cvec *cvk = NULL; vector of index keys * ... Populate cvk with key/values eg a:5 b:6 - * if (clixon_xml_find_index(xp, yp, "a", ns, cvk, &xvec, &xlen) < 0) + * if (clixon_xml_find_index(xp, yp, NULL, "a", ns, cvk, &xvec, &xlen) < 0) * err; * @endcode * Discussion: * (1) Rule 2 on how to get the child name election seems unecessary complex. First, it would be * nice to just state name and parent 2c. But xp as top-objects may sometimes be dummies, for * parsing, copy-buffers, or simply for XML nodes that do not have YANG. - * (2) Short form could be: "first" only returning first object to get rid of vectors. - * (3) Another short form could be: keyname,keyval instead of cvk for single keys. */ int clixon_xml_find_index(cxobj *xp, @@ -1233,14 +1427,8 @@ clixon_xml_find_index(cxobj *xp, if (ret == 0){ /* This means yang method did not work for some reason * such as not being list key indexes in cvk, for example */ -#ifdef XML_EXPLICIT_INDEX - /* Check if (exactly one) explicit indexes in cvk */ - if ((ret = xml_find_index_explicit(xp, yc, cvk, xvec, xlen)) < 0) + if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) goto done; - if (ret == 0) -#endif - if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) - goto done; } } else @@ -1294,4 +1482,3 @@ clixon_xml_find_pos(cxobj *xp, done: return retval; } - diff --git a/lib/src/clixon_xml_vec.c b/lib/src/clixon_xml_vec.c new file mode 100644 index 00000000..d87e4afb --- /dev/null +++ b/lib/src/clixon_xml_vec.c @@ -0,0 +1,315 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2019 Olof Hagsand + Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * Clixon XML object vectors + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* cligen */ +#include + +/* clixon */ +#include "clixon_err.h" +#include "clixon_string.h" +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_log.h" +#include "clixon_yang.h" +#include "clixon_xml.h" +#include "clixon_xml_vec.h" + +//typedef struct clixon_xml_vec clixon_xvec; /* struct defined in clicon_xml_vec.c */ + +/* How many XML children to start with if any (and then add exponentialy) */ +#define XVEC_MAX_DEFAULT 4 /* start value */ +#define XVEC_MAX_THRESHOLD 1024 /* exponential growth to here, then linear */ + +/*! Clixon xml vector concrete implementaion of the abstract clixon_xvec type + * Contiguous vector (not linked list) so that binary search can be done by direct index access + */ +struct clixon_xml_vec { + cxobj **xv_vec; /* Sorted vector of xml object pointers */ + int xv_len; /* Length of vector */ + int xv_max; /* Vector allocation */ +}; + +/*! Increment cxobj vector in an XML object vector + * + * Exponential growth to a threshold, then linear + * @param[in] xv XML tree vector + * @retval 0 OK + * @retval -1 Error + */ +static int +clixon_xvec_inc(clixon_xvec *xv) +{ + int retval = -1; + + xv->xv_len++; + if (xv->xv_len > xv->xv_max){ + if (xv->xv_max == 0) + xv->xv_max = XVEC_MAX_DEFAULT; + if (xv->xv_max < XVEC_MAX_THRESHOLD) + xv->xv_max *= 2; + else + xv->xv_max += XVEC_MAX_THRESHOLD; + if ((xv->xv_vec = realloc(xv->xv_vec, sizeof(cxobj *) * xv->xv_max)) == NULL){ + clicon_err(OE_XML, errno, "realloc"); + goto done; + } + } + retval = 0; + done: + return retval; +} + +/*! Create new XML object vector + * + * Exponential growth to a threshold, then linear + * @param[in] xv XML tree vector + * @retval 0 OK + * @retval -1 Error + */ +clixon_xvec * +clixon_xvec_new(void) +{ + clixon_xvec *xv = NULL; + + if ((xv = malloc(sizeof(*xv))) == NULL){ + clicon_err(OE_UNIX, errno, "malloc"); + goto done; + } + memset(xv, 0, sizeof(*xv)); + xv->xv_len = 0; + xv->xv_max = 0; + + done: + return xv; +} + +/*! Create and copy XML vector + * + * @param[in] xv0 XML tree vector + * @retval xv1 Duplicated XML vector + * @retval NULL Error + */ +clixon_xvec * +clixon_xvec_dup(clixon_xvec *xv0) +{ + clixon_xvec *xv1 = NULL; /* retval */ + + if ((xv1 = clixon_xvec_new()) == NULL) + goto done; + *xv1 = *xv0; + xv1->xv_vec = NULL; + if (xv1->xv_max && + (xv1->xv_vec = calloc(xv1->xv_max, sizeof(cxobj*))) == NULL){ + clicon_err(OE_UNIX, errno, "calloc"); + free(xv1); + xv1 = NULL; + goto done; + } + memcpy(xv1->xv_vec, xv0->xv_vec, xv0->xv_len*sizeof(cxobj*)); + done: + return xv1; +} + +/*! Free XML object list + */ +int +clixon_xvec_free(clixon_xvec *xv) +{ + if (xv->xv_vec) + free(xv->xv_vec); + if (xv) + free(xv); + return 0; +} + +/*! Return length of XML object list + * @param[in] xv XML tree vector + * @retval len Length of XML object vector + */ +int +clixon_xvec_len(clixon_xvec *xv) +{ + return xv->xv_len; +} + +/*! Return i:th XML object in XML object vector + * @param[in] xv XML tree vector + * @retval x OK + * @retval NULL Not found + */ +cxobj* +clixon_xvec_i(clixon_xvec *xv, + int i) +{ + if (i < xv->xv_len) + return xv->xv_vec[i]; + else + return NULL; +} + +/*! Append a new xml tree to an existing xml vector last in the list + * + * @param[in] xv XML tree vector + * @param[in] x XML tree (append this to vector) + * @retval 0 OK + * @retval -1 Error + * @code + * if (clixon_xvec_append(xv, x) < 0) + * err; + * @endcode + * @see clixon_cxvec_prepend + */ +int +clixon_xvec_append(clixon_xvec *xv, + cxobj *x) + +{ + int retval = -1; + + if (clixon_xvec_inc(xv) < 0) + goto done; + xv->xv_vec[xv->xv_len] = x; + retval = 0; + done: + return retval; +} + +/*! Prepend a new xml tree to an existing xml vector first in the list + * + * @param[in] xv XML tree vector + * @param[in] x XML tree (append this to vector) + * @retval 0 OK + * @retval -1 Error + * @code + * if (clixon_xvec_prepend(xv, x) < 0) + * err; + * @endcode + * @see clixon_cxvec_append + */ +int +clixon_xvec_prepend(clixon_xvec *xv, + cxobj *x) +{ + int retval = -1; + + if (clixon_xvec_inc(xv) < 0) + goto done; + memmove(&xv->xv_vec[1], &xv->xv_vec[0], sizeof(cxobj *) * (xv->xv_len-1)); + xv->xv_vec[0] = x; + retval = 0; + done: + return retval; +} + +/*! Insert XML node x at position i in XML object vector + * + * @param[in] xv XML tree vector + * @param[in] x XML tree (append this to vector) + * @param[in] i Position + * @retval 0 OK + * @retval -1 Error + */ +int +clixon_xvec_insert_pos(clixon_xvec *xv, + cxobj *x, + int i) +{ + int retval = -1; + size_t size; + + if (clixon_xvec_inc(xv) < 0) + goto done; + size = (xv->xv_len - i -1)*sizeof(cxobj *); + memmove(&xv->xv_vec[i+1], &xv->xv_vec[i], size); + xv->xv_vec[i] = x; + retval = 0; + done: + return retval; +} + +/*! Remove XML node x from position i in XML object vector + * + * @param[in] xv XML tree vector + * @param[in] i Position + * @retval 0 OK + * @retval -1 Error + */ +int +clixon_xvec_rm_pos(clixon_xvec *xv, + int i) +{ + size_t size; + + size = (xv->xv_len - i + 1)*sizeof(cxobj *); + memmove(&xv->xv_vec[i], &xv->xv_vec[i+1], size); + xv->xv_len--; + return 0; +} + +/*! Print an XML object vector to an output stream and encode chars "<>&" + * + * @param[in] f UNIX output stream + * @param[in] xv XML tree vector + * @retval 0 OK + */ +int +clixon_xvec_print(FILE *f, + clixon_xvec *xv) +{ + int i; + + for (i=0; ixv_len; i++) + clicon_xml2file(f, xv->xv_vec[i], 0, 1); + return 0; +} + diff --git a/test/test_openconfig.sh b/test/test_openconfig.sh index f91dcd12..2ef90f55 100755 --- a/test/test_openconfig.sh +++ b/test/test_openconfig.sh @@ -1,10 +1,6 @@ #!/usr/bin/env bash # Parse yang openconfig yangs from https://github.com/openconfig/public # Notes: -# - openconfig test suites are patched to counter Clixon issues as follows: -# - release/models/mpls/openconfig-mpls-te.yang -# issue: https://github.com/clicon/clixon/issues/6 -# - Env-var MODELS should be 1 # - Env-var OPENCONFIG should point to checkout place. (define it in site.sh for example) # Magic line must be first in script (see README.md) diff --git a/test/test_search_index.sh b/test/test_search_index.sh index abf545aa..6f6d1e6e 100755 --- a/test/test_search_index.sh +++ b/test/test_search_index.sh @@ -11,12 +11,10 @@ # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi -if false; then # NOTYET - : ${clixon_util_path:=clixon_util_path -D $DBG -Y /usr/local/share/clixon} # Number of list/leaf-list entries -: ${nr:=10} +: ${nr:=10000} # Number of tests to generate XML for +1 max=2 @@ -50,38 +48,52 @@ module moda{ type string; } leaf i{ - description "extra index"; - type string; + description "explicit index variable"; + type int32; cc:search_index; } + leaf j{ + description "non-index variable"; + type int32; + } } } } EOF -# key random -rnd=$(( ( RANDOM % $nr ) )) -# Let key index rndi be reverse of rnd -rndi=$(( $nr - $rnd - 1 )) - # Single string key # Assign index i in reverse order new "generate list with $nr single string key to $xml1" echo -n '' > $xml1 for (( i=0; i<$nr; i++ )); do - let ii=$nr-$i-1 - echo -n "a$ifoo$ii$ii" >> $xml1 + let ii=$nr-$i-1 + echo -n "a$ifoo$i$ii$ii" >> $xml1 done echo -n '' >> $xml1 -# How should I know it is an optimized search? -new "instance-id single string key i=i$rndi" -echo "$clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:i=\"i$rndi\"]" -expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:i=\"i$rndi\"])" 0 "^0: a$rndfoo$rndi$rndi$" +# First check correctness +for (( ii=0; ii<10; ii++ )); do + # key random + rnd=$(( ( RANDOM % $nr ) )) + # Let key index rndi be reverse of rnd + rndi=$(( $nr - $rnd - 1 )) + new "instance-id single string key i=$rndi (rnd:$rnd)" + expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:i=\"$rndi\"])" 0 "^0: a$rndfoo$rnd$rndi$rndi$" +done -#rm -rf $dir +# Then measure time for index and non-index, assume correct +# For small nr, the tiome to parse is so much larger than searching (and also parsing involves +# searching) which makes it hard to make a test comparing accessing the index variable "i" and the +# non-index variable "j". +new "index search latency i=$rndi" +{ time -p $clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:i=\"$rndi\"] -n 10 > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +new "non-index search latency j=$rndi" +{ time -p $clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:j=\"$rndi\"] > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +rm -rf $dir unset nr unset clixon_util_path # for other script reusing it -fi + diff --git a/util/clixon_util_path.c b/util/clixon_util_path.c index dbba53ff..7e9a3740 100644 --- a/util/clixon_util_path.c +++ b/util/clixon_util_path.c @@ -58,7 +58,7 @@ #include "clixon/clixon.h" /* Command line options to be passed to getopt(3) */ -#define UTIL_PATH_OPTS "hD:f:ap:y:Y:" +#define UTIL_PATH_OPTS "hD:f:ap:y:Y:n:" static int usage(char *argv0) @@ -72,6 +72,7 @@ usage(char *argv0) "\t-p \tPATH string\n" "\t-y \tYang filename or dir (load all files)\n" "\t-Y \tYang dirs (can be several)\n" + "\t-n \tRepeat the call n times(for profiling)\n" "and the following extra rules:\n" "\tif -f is not given, XML input is expected on stdin\n" "\tif -p is not given, is expected as the first line on stdin\n" @@ -90,7 +91,7 @@ main(int argc, int i; cxobj *x = NULL; cxobj **xvec = NULL; - size_t xlen; + size_t xlen = 0; int c; int len; char *buf = NULL; @@ -106,6 +107,7 @@ main(int argc, struct stat st; cxobj *xcfg = NULL; cxobj *xerr = NULL; /* malloced must be freed */ + int nr = 1; /* In the startup, logs to stderr & debug flag set later */ clicon_log_init("api-path", LOG_DEBUG, CLICON_LOG_STDERR); @@ -149,6 +151,9 @@ main(int argc, if (clicon_option_add(h, "CLICON_YANG_DIR", optarg) < 0) goto done; break; + case 'n': + nr = atoi(optarg); + break; default: usage(argv[0]); break; @@ -238,17 +243,20 @@ main(int argc, if (xml_apply0(x, -1, xml_sort_verify, h) < 0) clicon_log(LOG_NOTICE, "%s: sort verify failed", __FUNCTION__); } - if (api_path_p){ - if ((ret = clixon_xml_find_api_path(x, yspec, &xvec, &xlen, "%s", path)) < 0) + /* Repeat for profiling (default is nr = 1) */ + for (i=0; i