From 3748eefb8eeb31513576a9b740695401f114d786 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Tue, 11 Feb 2020 20:17:15 +0100 Subject: [PATCH 1/4] datastore repair test branch --- apps/backend/backend_commit.c | 16 ++--- example/main/example_backend.c | 95 +++++++++++++++++++++++++- lib/clixon/clixon_plugin.h | 17 ++++- lib/clixon/clixon_xml_map.h | 1 + lib/src/clixon_plugin.c | 27 ++++++++ lib/src/clixon_xml_map.c | 2 +- test/test_datastore_repair.sh | 118 +++++++++++++++++++++++++++++++++ 7 files changed, 265 insertions(+), 11 deletions(-) create mode 100755 test/test_datastore_repair.sh diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 890be1d5..f42a08c5 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -197,14 +197,14 @@ startup_common(clicon_handle h, xt = NULL; goto ok; } - if (msd){ - /* Here xt is old syntax */ - if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) - goto done; - if (ret == 0) - goto fail; - /* Here xt is new syntax */ - } + if (clixon_plugin_xmldb_repair(h, db, xt) < 0) + goto done; + /* Here xt is old syntax */ + if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) + goto done; + if (ret == 0) + goto fail; + if ((yspec = clicon_dbspec_yang(h)) == NULL){ clicon_err(OE_YANG, 0, "Yang spec not set"); goto done; diff --git a/example/main/example_backend.c b/example/main/example_backend.c index b8db088d..440d2d92 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -173,6 +173,98 @@ main_abort(clicon_handle h, return 0; } +struct map_str2str{ + char *ms_path; + char *ms_ns; +}; +static const struct map_str2str path_namespace_map[] = { + {"/a:x/a:y/a:z/a:w", "urn:example:b"}, + {"/a:x/a:y/a:z", "urn:example:b"}, + {NULL, NULL} +}; + +static int +main_repair_one(cxobj *xt, + char *mypath, + char *mynamespace, + cvec *nsc) +{ + int retval = -1; + cxobj **xvec = NULL; + size_t xlen; + char *myprefix = NULL; + int i; + cxobj *x; + char *namespace; + char *pexist = NULL; /* Existing prefix */ + + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) + goto done; + if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){ + clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map", + mynamespace); + goto done; + } + for (i=0; ims_path; ms++) + if (main_repair_one(xt, ms->ms_path, ms->ms_ns, nsc) < 0) + goto done; + ok: + retval = 0; + done: + if (nsc) + cvec_free(nsc); + return retval; +} + /*! Routing example notification timer handler. Here is where the periodic action is */ static int @@ -684,7 +776,8 @@ static clixon_plugin_api api = { .ca_trans_commit=main_commit, /* trans commit */ .ca_trans_revert=main_revert, /* trans revert */ .ca_trans_end=main_end, /* trans end */ - .ca_trans_abort=main_abort /* trans abort */ + .ca_trans_abort=main_abort, /* trans abort */ + .ca_xmldb_repair=main_repair /* xmldb repair. */ }; /*! Backend plugin initialization diff --git a/lib/clixon/clixon_plugin.h b/lib/clixon/clixon_plugin.h index 2b8445c1..092f8d8b 100644 --- a/lib/clixon/clixon_plugin.h +++ b/lib/clixon/clixon_plugin.h @@ -159,6 +159,16 @@ typedef int (trans_cb_t)(clicon_handle h, transaction_data td); */ typedef char *(cli_prompthook_t)(clicon_handle, char *mode); +/*! Datastore repair callback + * Gets called on startup after initial XML parsing, but before upgrading and before validation + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running" + * @param[in] xt XML tree. Repair this "in place" + * @retval -1 Error + * @retval 0 OK + */ +typedef int (xmldb_repair_t)(clicon_handle h, char *db, cxobj *xt); + /*! Startup status for use in startup-callback * Note that for STARTUP_ERR and _INVALID, running runs in failsafe mode * and startup contains the erroneous or invalid database. @@ -206,8 +216,9 @@ struct clixon_plugin_api{ trans_cb_t *cb_trans_revert; /* Transaction revert */ trans_cb_t *cb_trans_end; /* Transaction completed */ trans_cb_t *cb_trans_abort; /* Transaction aborted */ - } cau_backend; + xmldb_repair_t *cb_xmldb_repair; /* Repair datastore on load */ + } cau_backend; } u; }; /* Access fields */ @@ -224,6 +235,8 @@ struct clixon_plugin_api{ #define ca_trans_revert u.cau_backend.cb_trans_revert #define ca_trans_end u.cau_backend.cb_trans_end #define ca_trans_abort u.cau_backend.cb_trans_abort +#define ca_xmldb_repair u.cau_backend.cb_xmldb_repair + /* * Macros @@ -271,6 +284,8 @@ int clixon_plugin_auth(clicon_handle h, void *arg); int clixon_plugin_extension(clicon_handle h, yang_stmt *yext, yang_stmt *ys); +int clixon_plugin_xmldb_repair(clicon_handle h, char *db, cxobj *xt); + /* rpc callback API */ int rpc_callback_register(clicon_handle h, clicon_rpc_cb cb, void *arg, char *namespace, char *name); int rpc_callback_delete_all(clicon_handle h); diff --git a/lib/clixon/clixon_xml_map.h b/lib/clixon/clixon_xml_map.h index 9a5107a2..00731904 100644 --- a/lib/clixon/clixon_xml_map.h +++ b/lib/clixon/clixon_xml_map.h @@ -56,6 +56,7 @@ int xml_diff(yang_stmt *yspec, cxobj *x0, cxobj *x1, cxobj ***changed_x0, cxobj ***changed_x1, size_t *changedlen); int xml_tree_prune_flagged_sub(cxobj *xt, int flag, int test, int *upmark); int xml_tree_prune_flagged(cxobj *xt, int flag, int test); +int add_namespace(cxobj *x1, cxobj *x1p, char *prefix1, char *namespace); int xml_default(cxobj *x, void *arg); int xml_sanity(cxobj *x, void *arg); int xml_non_config_data(cxobj *xt, void *arg); diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index 8bfdefd8..d28230f7 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -467,6 +467,33 @@ clixon_plugin_extension(clicon_handle h, } return retval; } +/*! Call plugin module repair in all plugins + * + * Repair datastore on load before or as an alternative to the upgrading mechanism + * @param[in] h Clicon handle + * Call plugin start functions (if defined) + * @note Start functions used to have argc/argv. Use clicon_argv_get() instead + */ +int +clixon_plugin_xmldb_repair(clicon_handle h, + char *db, + cxobj *xt) +{ + clixon_plugin *cp; + int i; + xmldb_repair_t *repairfn; + + for (i = 0; i < _clixon_nplugins; i++) { + cp = &_clixon_plugins[i]; + if ((repairfn = cp->cp_api.ca_xmldb_repair) == NULL) + continue; + if (repairfn(h, db, xt) < 0) { + clicon_debug(1, "%s() failed", __FUNCTION__); + return -1; + } + } + return 0; +} /*-------------------------------------------------------------------- * RPC callbacks for both client/frontend and backend plugins. diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 2d822638..52565c3c 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -708,7 +708,7 @@ xml_tree_prune_flagged(cxobj *xt, /*! Add prefix:namespace pair to xml node, set cache, prefix, etc */ -static int +int add_namespace(cxobj *x1, /* target */ cxobj *x1p, char *prefix1, diff --git a/test/test_datastore_repair.sh b/test/test_datastore_repair.sh new file mode 100755 index 00000000..800fb8a7 --- /dev/null +++ b/test/test_datastore_repair.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Keep a vector of {xpath, namespace} pairs, match xpath in parsed XML and check if symbols belong to namespace +# If not, set that namespace. +# This is a primitive upgrade mechanism if th emore general upgrade mechanism can not be used + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +cfg=$dir/conf_yang.xml +fyangA=$dir/A.yang +fyangB=$dir/B.yang + +# Create configuration +cat < $cfg + + $cfg + ietf-netconf:startup + /usr/local/share/clixon + $dir + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/lib/example/backend + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + true + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + +EOF + +# Yang module A (base) +cat < $fyangA +module A{ + prefix a; + revision 2020-02-11; + namespace "urn:example:a"; + container x { + container y { + } + } +} +EOF + +# Yang module B (augments A) +cat < $fyangB +module B{ + prefix b; + revision 2020-02-11; + namespace "urn:example:b"; + import A { + prefix "a"; + } + augment "/a:x/ngrt:y" { + container z { + leaf w { + type string; + } + } + } +} +EOF + +# permission kludges +sudo touch $dir/startup_db +sudo chmod 666 $dir/startup_db +# Create startup db of "old" db with incorrect augment namespace tagging +cat < $dir/startup_db + + + + + foo + + + + +EOF + +# This is how it should look after repair, using prefixes +AFTER=$(cat <foo +EOF +) + +# Or using default: +# foo + +new "test params: -f $cfg" +# Bring your own backend +if [ $BE -ne 0 ]; then + # kill old backend (if any) + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s startup -f $cfg" + start_backend -s startup -f $cfg + + new "waiting" + wait_backend +fi + +new "netconf get config" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$AFTER]]>]]>$" + +new "Kill backend" +# Check if premature kill +pid=$(pgrep -u root -f clixon_backend) +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +#rm -rf $dir From b6812793f9ea9df87087bf72ca51c3b00117081f Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 12 Feb 2020 16:40:52 +0100 Subject: [PATCH 2/4] General-purpose upgrade --- apps/backend/backend_commit.c | 6 +- example/main/example_backend.c | 202 +++++++++++++++++--------------- lib/clixon/clixon_plugin.h | 20 ++-- lib/clixon/clixon_string.h | 10 +- lib/clixon/clixon_xml_map.h | 2 +- lib/clixon/clixon_yang_module.h | 6 +- lib/src/clixon_data.c | 1 + lib/src/clixon_datastore.c | 3 +- lib/src/clixon_datastore_read.c | 10 +- lib/src/clixon_plugin.c | 24 ++-- lib/src/clixon_proto_client.c | 1 + lib/src/clixon_xml_map.c | 46 +++++++- lib/src/clixon_xpath.c | 4 +- lib/src/clixon_xpath_eval.c | 13 +- lib/src/clixon_yang.c | 1 + lib/src/clixon_yang_module.c | 4 +- lib/src/clixon_yang_type.c | 1 + test/test_datastore_repair.sh | 97 ++++++++------- 18 files changed, 286 insertions(+), 165 deletions(-) diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index f42a08c5..1f2ea859 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -197,9 +197,11 @@ startup_common(clicon_handle h, xt = NULL; goto ok; } - if (clixon_plugin_xmldb_repair(h, db, xt) < 0) - goto done; /* Here xt is old syntax */ + /* General purpose datastore upgrade */ + if (clixon_plugin_datastore_upgrade(h, db, xt, msd) < 0) + goto done; + /* Module-specific upgrade callbacks */ if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) goto done; if (ret == 0) diff --git a/example/main/example_backend.c b/example/main/example_backend.c index 440d2d92..b2a83b94 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -173,98 +173,6 @@ main_abort(clicon_handle h, return 0; } -struct map_str2str{ - char *ms_path; - char *ms_ns; -}; -static const struct map_str2str path_namespace_map[] = { - {"/a:x/a:y/a:z/a:w", "urn:example:b"}, - {"/a:x/a:y/a:z", "urn:example:b"}, - {NULL, NULL} -}; - -static int -main_repair_one(cxobj *xt, - char *mypath, - char *mynamespace, - cvec *nsc) -{ - int retval = -1; - cxobj **xvec = NULL; - size_t xlen; - char *myprefix = NULL; - int i; - cxobj *x; - char *namespace; - char *pexist = NULL; /* Existing prefix */ - - if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) - goto done; - if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){ - clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map", - mynamespace); - goto done; - } - for (i=0; ims_path; ms++) - if (main_repair_one(xt, ms->ms_path, ms->ms_ns, nsc) < 0) - goto done; - ok: - retval = 0; - done: - if (nsc) - cvec_free(nsc); - return retval; -} - /*! Routing example notification timer handler. Here is where the periodic action is */ static int @@ -502,6 +410,112 @@ example_extension(clicon_handle h, return retval; } +/* Here follows code for general-purpose datastore upgrade + * Nodes affected are identified by paths. + * In this example nodes' namespaces are changed, or they are removed altogether + */ +/* Rename the namespaces of these paths. + * That is, paths (on the left) should get namespaces (to the right) + */ +static const map_str2str namespace_map[] = { + {"/a:x/a:y/a:z/descendant-or-self::node()", "urn:example:b"}, + {NULL, NULL} +}; + +/* Remove these paths */ +static const char *remove_map[] = { + "/a:remove_me", + NULL +}; + +/*! General-purpose datastore upgrade callback called once on startup + * + * Gets called on startup after initial XML parsing, but before module-specific upgrades + * and before validation. + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running", "startup" or "tmp" + * @param[in] xt XML tree. Upgrade this "in place" + * @param[in] msd Info on datastore module-state, if any + * @retval -1 Error + * @retval 0 OK + */ +int +example_upgrade(clicon_handle h, + char *db, + cxobj *xt, + modstate_diff_t *msd) +{ + int retval = -1; + cvec *nsc = NULL; /* Canonical namespace */ + yang_stmt *yspec; yang_stmt *yspec; + const struct map_str2str *ms; /* map iterator */ + cxobj **xvec = NULL; /* vector of result nodes */ + size_t xlen; + int i; + const char **pp; + + if (strcmp(db, "startup") != 0) /* skip other than startup datastore */ + goto ok; + if (msd->md_status) /* skip if there is proper module-state in datastore */ + goto ok; + yspec = clicon_dbspec_yang(h); /* Get all yangs */ + /* Get canonical namespaces for using "normalized" prefixes */ + if (xml_nsctx_yangspec(yspec, &nsc) < 0) + goto done; + /* 1. Remove paths */ + for (pp = remove_map; *pp; ++pp){ + /* Find all nodes matching n */ + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, *pp) < 0) + goto done; + /* Remove them */ + /* Loop through all nodes matching mypath and change theoir namespace */ + for (i=0; ims_s0; ms++){ + char *mypath; + char *mynamespace; + char *myprefix = NULL; + + mypath = ms->ms_s0; + mynamespace = ms->ms_s1; + if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){ + clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map", + mynamespace); + goto done; + } + /* Find all nodes matching mypath */ + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) + goto done; + /* Loop through all nodes matching mypath and change theoir namespace */ + for (i=0; imd_status = 1; /* Create diff trees */ if (xml_parse_string("", yspec, &msd->md_del) < 0) goto done; @@ -258,6 +259,13 @@ text_read_modstate(clicon_handle h, /* 3) For each module state m in the file */ while ((xm = xml_child_each(xmodst, xm, CX_ELMNT)) != NULL) { + if (strcmp(xml_name(xm), "module-set-id") == 0){ + if (xml_body(xm) && (msd->md_set_id = strdup(xml_body(xm))) == NULL){ + clicon_err(OE_UNIX, errno, "strdup"); + goto done; + } + } + continue; /* ignore other tags, such as module-set-id */ if (strcmp(xml_name(xm), "module")) continue; /* ignore other tags, such as module-set-id */ if ((name = xml_find_body(xm, "name")) == NULL) diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index d28230f7..f997702e 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -59,6 +59,7 @@ #include "clixon_handle.h" #include "clixon_yang.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" /* List of plugins XXX @@ -467,27 +468,34 @@ clixon_plugin_extension(clicon_handle h, } return retval; } -/*! Call plugin module repair in all plugins +/*! Call plugingeneral-purpose datastore upgrade in all plugins * - * Repair datastore on load before or as an alternative to the upgrading mechanism + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running", "startup" or "tmp" + * @param[in] xt XML tree. Upgrade this "in place" + * @param[in] msd Module-state diff, info on datastore module-state + * @retval -1 Error + * @retval 0 OK + * Upgrade datastore on load before or as an alternative to module-specific upgrading mechanism * @param[in] h Clicon handle * Call plugin start functions (if defined) * @note Start functions used to have argc/argv. Use clicon_argv_get() instead */ int -clixon_plugin_xmldb_repair(clicon_handle h, - char *db, - cxobj *xt) +clixon_plugin_datastore_upgrade(clicon_handle h, + char *db, + cxobj *xt, + modstate_diff_t *msd) { clixon_plugin *cp; int i; - xmldb_repair_t *repairfn; + datastore_upgrade_t *repairfn; for (i = 0; i < _clixon_nplugins; i++) { cp = &_clixon_plugins[i]; - if ((repairfn = cp->cp_api.ca_xmldb_repair) == NULL) + if ((repairfn = cp->cp_api.ca_datastore_upgrade) == NULL) continue; - if (repairfn(h, db, xt) < 0) { + if (repairfn(h, db, xt, msd) < 0) { clicon_debug(1, "%s() failed", __FUNCTION__); return -1; } diff --git a/lib/src/clixon_proto_client.c b/lib/src/clixon_proto_client.c index bdd87f0f..552e5048 100644 --- a/lib/src/clixon_proto_client.c +++ b/lib/src/clixon_proto_client.c @@ -64,6 +64,7 @@ #include "clixon_xml.h" #include "clixon_options.h" #include "clixon_data.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_string.h" #include "clixon_xpath_ctx.h" diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 52565c3c..6cb697b2 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -64,6 +64,7 @@ #include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_options.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_xml_nsctx.h" #include "clixon_xpath_ctx.h" @@ -708,7 +709,7 @@ xml_tree_prune_flagged(cxobj *xt, /*! Add prefix:namespace pair to xml node, set cache, prefix, etc */ -int +static int add_namespace(cxobj *x1, /* target */ cxobj *x1p, char *prefix1, @@ -746,6 +747,49 @@ add_namespace(cxobj *x1, /* target */ return retval; } +/*! Change namespace of XML node + * + * @param[in] x XML node + * @param[in] namespace Change to this namespace (if it does not already belong to it) + * @param[in] prefix If change, use this namespace + * @param 0 OK + * @param -1 Error + */ +int +xml_namespace_change(cxobj *x, + char *namespace, + char *prefix) +{ + int retval = -1; + cxobj *xp; + char *ns0 = NULL; /* existing namespace */ + char *prefix0 = NULL; /* existing prefix */ + + ns0 = NULL; + if (xml2ns(x, xml_prefix(x), &ns0) < 0) + goto done; + if (strcmp(ns0, namespace) == 0) + goto ok; /* Already has right namespace */ + /* Is namespace already declared? */ + if (xml2prefix(x, namespace, &prefix0) == 1){ + /* Yes it is declared and the prefix is pexists */ + if (xml_prefix_set(x, prefix0) < 0) + goto done; + } + else{ /* Namespace does not exist, add it */ + if ((xp = xml_parent(x)) == NULL){ + clicon_err(OE_XML, ENOENT, "XML node must have parent"); + goto done; + } + if (add_namespace(x, xp, prefix0, namespace) < 0) + goto done; + } + ok: + retval = 0; + done: + return retval; +} + /*! Add default values (if not set) * @param[in] xt XML tree with some node marked * @param[in] arg Ignored diff --git a/lib/src/clixon_xpath.c b/lib/src/clixon_xpath.c index 9f658c2f..385608bf 100644 --- a/lib/src/clixon_xpath.c +++ b/lib/src/clixon_xpath.c @@ -122,11 +122,11 @@ static const map_str2int xpath_tree_map[] = { static const map_str2int axis_type_map[] = { {"NaN", A_NAN}, {"ancestor", A_ANCESTOR}, - {"ancestor-or-selgf", A_ANCESTOR_OR_SELF}, + {"ancestor-or-self", A_ANCESTOR_OR_SELF}, {"attribute", A_ATTRIBUTE}, {"child", A_CHILD}, {"descendant", A_DESCENDANT}, - {"descendeant-or-self", A_DESCENDANT_OR_SELF}, + {"descendant-or-self", A_DESCENDANT_OR_SELF}, {"following", A_FOLLOWING}, {"following-sibling", A_FOLLOWING_SIBLING}, {"namespace", A_NAMESPACE}, diff --git a/lib/src/clixon_xpath_eval.c b/lib/src/clixon_xpath_eval.c index dd3c0965..f712fcad 100644 --- a/lib/src/clixon_xpath_eval.c +++ b/lib/src/clixon_xpath_eval.c @@ -362,8 +362,19 @@ xp_eval_step(xp_ctx *xc0, } ctx_nodeset_replace(xc, vec, veclen); break; - case A_DESCENDANT: case A_DESCENDANT_OR_SELF: + for (i=0; ixc_size; i++){ + xv = xc->xc_nodeset[i]; + if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0) + goto done; + } + for (i=0; ixc_nodeset, &xc->xc_size) < 0) + goto done; + } + break; + case A_DESCENDANT: for (i=0; ixc_size; i++){ xv = xc->xc_nodeset[i]; if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0) diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 09e6a822..9807c62a 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -87,6 +87,7 @@ #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_data.h" #include "clixon_options.h" diff --git a/lib/src/clixon_yang_module.c b/lib/src/clixon_yang_module.c index 4210fa6a..2888fbd3 100644 --- a/lib/src/clixon_yang_module.c +++ b/lib/src/clixon_yang_module.c @@ -74,10 +74,10 @@ #include "clixon_xpath.h" #include "clixon_options.h" #include "clixon_data.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_netconf_lib.h" #include "clixon_xml_map.h" -#include "clixon_yang_module.h" #include "clixon_yang_internal.h" /* internal */ modstate_diff_t * @@ -97,6 +97,8 @@ modstate_diff_free(modstate_diff_t *md) { if (md == NULL) return 0; + if (md->md_set_id) + free(md->md_set_id); if (md->md_del) xml_free(md->md_del); if (md->md_mod) diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index 0cd50346..fb59b3e4 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -94,6 +94,7 @@ #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_options.h" #include "clixon_yang.h" diff --git a/test/test_datastore_repair.sh b/test/test_datastore_repair.sh index 800fb8a7..2a98d1e6 100755 --- a/test/test_datastore_repair.sh +++ b/test/test_datastore_repair.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash -# Keep a vector of {xpath, namespace} pairs, match xpath in parsed XML and check if symbols belong to namespace -# If not, set that namespace. -# This is a primitive upgrade mechanism if th emore general upgrade mechanism can not be used +# Test of the general-purpose (raw) upgrade mechanism. +# Input is a startup db without mod-state info. +# It has wrong namespace bindings and needs to remove some nodes +# Output is a valid config woith correct namespaces and removed nods +# The code for this is in the main example backend plugin. # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi @@ -40,6 +42,12 @@ module A{ container y { } } + list remove_me { + key k; + leaf k { + type string; + } + } } EOF @@ -65,7 +73,48 @@ EOF # permission kludges sudo touch $dir/startup_db sudo chmod 666 $dir/startup_db + +# This is how it should look after repair, using prefixes +AFTER=$(cat <foo +EOF +) + +testrun(){ + new "test params: -f $cfg" + # Bring your own backend + if [ $BE -ne 0 ]; then + # kill old backend (if any) + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s startup -f $cfg" + start_backend -s startup -f $cfg + + new "waiting" + wait_backend + fi + + new "netconf get config" + expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$AFTER]]>]]>$" + + if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + fi + +} # end testrun + # Create startup db of "old" db with incorrect augment namespace tagging +# without modstate cat < $dir/startup_db @@ -75,44 +124,12 @@ cat < $dir/startup_db + This node is obsolete + this too EOF -# This is how it should look after repair, using prefixes -AFTER=$(cat <foo -EOF -) +new "general-purpose upgrade without modstate" +testrun -# Or using default: -# foo - -new "test params: -f $cfg" -# Bring your own backend -if [ $BE -ne 0 ]; then - # kill old backend (if any) - new "kill old backend" - sudo clixon_backend -zf $cfg - if [ $? -ne 0 ]; then - err - fi - new "start backend -s startup -f $cfg" - start_backend -s startup -f $cfg - - new "waiting" - wait_backend -fi - -new "netconf get config" -expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$AFTER]]>]]>$" - -new "Kill backend" -# Check if premature kill -pid=$(pgrep -u root -f clixon_backend) -if [ -z "$pid" ]; then - err "backend already dead" -fi -# kill backend -stop_backend -f $cfg - -#rm -rf $dir +rm -rf $dir From cdd22bc33dd4483b8a4647768299fccdc39bbb0f Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 12 Feb 2020 17:01:25 +0100 Subject: [PATCH 3/4] example typo --- example/main/example_backend.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/example/main/example_backend.c b/example/main/example_backend.c index b2a83b94..27035ce9 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -419,12 +419,14 @@ example_extension(clicon_handle h, */ static const map_str2str namespace_map[] = { {"/a:x/a:y/a:z/descendant-or-self::node()", "urn:example:b"}, + /* add more paths to be renamed here */ {NULL, NULL} }; /* Remove these paths */ static const char *remove_map[] = { "/a:remove_me", + /* add more paths to be deleted here */ NULL }; @@ -447,7 +449,7 @@ example_upgrade(clicon_handle h, { int retval = -1; cvec *nsc = NULL; /* Canonical namespace */ - yang_stmt *yspec; yang_stmt *yspec; + yang_stmt *yspec; const struct map_str2str *ms; /* map iterator */ cxobj **xvec = NULL; /* vector of result nodes */ size_t xlen; From fa257ebb88cba656b3091f1a6b3cdf6c8cc7ddab Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 12 Feb 2020 17:37:23 +0100 Subject: [PATCH 4/4] datastore upgrade bugs --- example/main/example_backend.c | 25 +++++++++++++++++-------- lib/src/clixon_datastore_read.c | 1 - lib/src/clixon_xpath_eval.c | 4 ++++ test/test_datastore_repair.sh | 6 +++--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/example/main/example_backend.c b/example/main/example_backend.c index 27035ce9..a12679dc 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -71,11 +71,15 @@ static int _reset = 0; */ static int _state = 0; -/*! Variable to control upgrade callbacks. +/*! Variable to control module-specific upgrade callbacks. * If set, call test-case for upgrading ietf-interfaces, otherwise call * auto-upgrade */ -static int _upgrade = 0; +static int _module_upgrade = 0; + +/*! Variable to control general-purpose upgrade callbacks. + */ +static int _general_upgrade = 0; /*! Variable to control transaction logging (for debug) * If set, call syslog for every transaction callback @@ -456,9 +460,11 @@ example_upgrade(clicon_handle h, int i; const char **pp; + if (_general_upgrade == 0) + goto ok; if (strcmp(db, "startup") != 0) /* skip other than startup datastore */ goto ok; - if (msd->md_status) /* skip if there is proper module-state in datastore */ + if (msd && msd->md_status) /* skip if there is proper module-state in datastore */ goto ok; yspec = clicon_dbspec_yang(h); /* Get all yangs */ /* Get canonical namespaces for using "normalized" prefixes */ @@ -793,7 +799,7 @@ static clixon_plugin_api api = { .ca_trans_revert=main_revert, /* trans revert */ .ca_trans_end=main_end, /* trans end */ .ca_trans_abort=main_abort, /* trans abort */ - .ca_datastore_upgrade=example_upgrade /* gneral-purpose upgrade. */ + .ca_datastore_upgrade=example_upgrade /* general-purpose upgrade. */ }; /*! Backend plugin initialization @@ -818,7 +824,7 @@ clixon_plugin_init(clicon_handle h) goto done; opterr = 0; optind = 1; - while ((c = getopt(argc, argv, "rsut:")) != -1) + while ((c = getopt(argc, argv, "rsuUt:")) != -1) switch (c) { case 'r': _reset = 1; @@ -826,8 +832,11 @@ clixon_plugin_init(clicon_handle h) case 's': _state = 1; break; - case 'u': - _upgrade = 1; + case 'u': /* module-specific upgrade */ + _module_upgrade = 1; + break; + case 'U': /* general-purpose upgrade */ + _general_upgrade = 1; break; case 't': /* transaction log */ _transaction_log = 1; @@ -885,7 +894,7 @@ clixon_plugin_init(clicon_handle h) /* Upgrade callback: if you start the backend with -- -u you will get the * test interface example. Otherwise the auto-upgrade feature is enabled. */ - if (_upgrade){ + if (_module_upgrade == 1){ if (upgrade_callback_register(h, upgrade_2016, "urn:example:interfaces", 20140508, 20160101, NULL) < 0) goto done; if (upgrade_callback_register(h, upgrade_2018, "urn:example:interfaces", 20160101, 20180220, NULL) < 0) diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index ab0d16e1..2bcc10dd 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -265,7 +265,6 @@ text_read_modstate(clicon_handle h, goto done; } } - continue; /* ignore other tags, such as module-set-id */ if (strcmp(xml_name(xm), "module")) continue; /* ignore other tags, such as module-set-id */ if ((name = xml_find_body(xm, "name")) == NULL) diff --git a/lib/src/clixon_xpath_eval.c b/lib/src/clixon_xpath_eval.c index f712fcad..1c3a7b77 100644 --- a/lib/src/clixon_xpath_eval.c +++ b/lib/src/clixon_xpath_eval.c @@ -373,6 +373,10 @@ xp_eval_step(xp_ctx *xc0, if (cxvec_append(x, &xc->xc_nodeset, &xc->xc_size) < 0) goto done; } + if (vec){ + free(vec); + vec = NULL; + } break; case A_DESCENDANT: for (i=0; ixc_size; i++){ diff --git a/test/test_datastore_repair.sh b/test/test_datastore_repair.sh index 2a98d1e6..640e9b48 100755 --- a/test/test_datastore_repair.sh +++ b/test/test_datastore_repair.sh @@ -81,7 +81,7 @@ EOF ) testrun(){ - new "test params: -f $cfg" + new "test params: -f $cfg -- -U" # Bring your own backend if [ $BE -ne 0 ]; then # kill old backend (if any) @@ -90,8 +90,8 @@ testrun(){ if [ $? -ne 0 ]; then err fi - new "start backend -s startup -f $cfg" - start_backend -s startup -f $cfg + new "start backend -s startup -f $cfg -- -U" + start_backend -s startup -f $cfg -- -U new "waiting" wait_backend