diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index f42a08c5..1f2ea859 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -197,9 +197,11 @@ startup_common(clicon_handle h, xt = NULL; goto ok; } - if (clixon_plugin_xmldb_repair(h, db, xt) < 0) - goto done; /* Here xt is old syntax */ + /* General purpose datastore upgrade */ + if (clixon_plugin_datastore_upgrade(h, db, xt, msd) < 0) + goto done; + /* Module-specific upgrade callbacks */ if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) goto done; if (ret == 0) diff --git a/example/main/example_backend.c b/example/main/example_backend.c index 440d2d92..b2a83b94 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -173,98 +173,6 @@ main_abort(clicon_handle h, return 0; } -struct map_str2str{ - char *ms_path; - char *ms_ns; -}; -static const struct map_str2str path_namespace_map[] = { - {"/a:x/a:y/a:z/a:w", "urn:example:b"}, - {"/a:x/a:y/a:z", "urn:example:b"}, - {NULL, NULL} -}; - -static int -main_repair_one(cxobj *xt, - char *mypath, - char *mynamespace, - cvec *nsc) -{ - int retval = -1; - cxobj **xvec = NULL; - size_t xlen; - char *myprefix = NULL; - int i; - cxobj *x; - char *namespace; - char *pexist = NULL; /* Existing prefix */ - - if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) - goto done; - if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){ - clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map", - mynamespace); - goto done; - } - for (i=0; ims_path; ms++) - if (main_repair_one(xt, ms->ms_path, ms->ms_ns, nsc) < 0) - goto done; - ok: - retval = 0; - done: - if (nsc) - cvec_free(nsc); - return retval; -} - /*! Routing example notification timer handler. Here is where the periodic action is */ static int @@ -502,6 +410,112 @@ example_extension(clicon_handle h, return retval; } +/* Here follows code for general-purpose datastore upgrade + * Nodes affected are identified by paths. + * In this example nodes' namespaces are changed, or they are removed altogether + */ +/* Rename the namespaces of these paths. + * That is, paths (on the left) should get namespaces (to the right) + */ +static const map_str2str namespace_map[] = { + {"/a:x/a:y/a:z/descendant-or-self::node()", "urn:example:b"}, + {NULL, NULL} +}; + +/* Remove these paths */ +static const char *remove_map[] = { + "/a:remove_me", + NULL +}; + +/*! General-purpose datastore upgrade callback called once on startup + * + * Gets called on startup after initial XML parsing, but before module-specific upgrades + * and before validation. + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running", "startup" or "tmp" + * @param[in] xt XML tree. Upgrade this "in place" + * @param[in] msd Info on datastore module-state, if any + * @retval -1 Error + * @retval 0 OK + */ +int +example_upgrade(clicon_handle h, + char *db, + cxobj *xt, + modstate_diff_t *msd) +{ + int retval = -1; + cvec *nsc = NULL; /* Canonical namespace */ + yang_stmt *yspec; yang_stmt *yspec; + const struct map_str2str *ms; /* map iterator */ + cxobj **xvec = NULL; /* vector of result nodes */ + size_t xlen; + int i; + const char **pp; + + if (strcmp(db, "startup") != 0) /* skip other than startup datastore */ + goto ok; + if (msd->md_status) /* skip if there is proper module-state in datastore */ + goto ok; + yspec = clicon_dbspec_yang(h); /* Get all yangs */ + /* Get canonical namespaces for using "normalized" prefixes */ + if (xml_nsctx_yangspec(yspec, &nsc) < 0) + goto done; + /* 1. Remove paths */ + for (pp = remove_map; *pp; ++pp){ + /* Find all nodes matching n */ + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, *pp) < 0) + goto done; + /* Remove them */ + /* Loop through all nodes matching mypath and change theoir namespace */ + for (i=0; ims_s0; ms++){ + char *mypath; + char *mynamespace; + char *myprefix = NULL; + + mypath = ms->ms_s0; + mynamespace = ms->ms_s1; + if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){ + clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map", + mynamespace); + goto done; + } + /* Find all nodes matching mypath */ + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) + goto done; + /* Loop through all nodes matching mypath and change theoir namespace */ + for (i=0; imd_status = 1; /* Create diff trees */ if (xml_parse_string("", yspec, &msd->md_del) < 0) goto done; @@ -258,6 +259,13 @@ text_read_modstate(clicon_handle h, /* 3) For each module state m in the file */ while ((xm = xml_child_each(xmodst, xm, CX_ELMNT)) != NULL) { + if (strcmp(xml_name(xm), "module-set-id") == 0){ + if (xml_body(xm) && (msd->md_set_id = strdup(xml_body(xm))) == NULL){ + clicon_err(OE_UNIX, errno, "strdup"); + goto done; + } + } + continue; /* ignore other tags, such as module-set-id */ if (strcmp(xml_name(xm), "module")) continue; /* ignore other tags, such as module-set-id */ if ((name = xml_find_body(xm, "name")) == NULL) diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index d28230f7..f997702e 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -59,6 +59,7 @@ #include "clixon_handle.h" #include "clixon_yang.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" /* List of plugins XXX @@ -467,27 +468,34 @@ clixon_plugin_extension(clicon_handle h, } return retval; } -/*! Call plugin module repair in all plugins +/*! Call plugingeneral-purpose datastore upgrade in all plugins * - * Repair datastore on load before or as an alternative to the upgrading mechanism + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running", "startup" or "tmp" + * @param[in] xt XML tree. Upgrade this "in place" + * @param[in] msd Module-state diff, info on datastore module-state + * @retval -1 Error + * @retval 0 OK + * Upgrade datastore on load before or as an alternative to module-specific upgrading mechanism * @param[in] h Clicon handle * Call plugin start functions (if defined) * @note Start functions used to have argc/argv. Use clicon_argv_get() instead */ int -clixon_plugin_xmldb_repair(clicon_handle h, - char *db, - cxobj *xt) +clixon_plugin_datastore_upgrade(clicon_handle h, + char *db, + cxobj *xt, + modstate_diff_t *msd) { clixon_plugin *cp; int i; - xmldb_repair_t *repairfn; + datastore_upgrade_t *repairfn; for (i = 0; i < _clixon_nplugins; i++) { cp = &_clixon_plugins[i]; - if ((repairfn = cp->cp_api.ca_xmldb_repair) == NULL) + if ((repairfn = cp->cp_api.ca_datastore_upgrade) == NULL) continue; - if (repairfn(h, db, xt) < 0) { + if (repairfn(h, db, xt, msd) < 0) { clicon_debug(1, "%s() failed", __FUNCTION__); return -1; } diff --git a/lib/src/clixon_proto_client.c b/lib/src/clixon_proto_client.c index bdd87f0f..552e5048 100644 --- a/lib/src/clixon_proto_client.c +++ b/lib/src/clixon_proto_client.c @@ -64,6 +64,7 @@ #include "clixon_xml.h" #include "clixon_options.h" #include "clixon_data.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_string.h" #include "clixon_xpath_ctx.h" diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 52565c3c..6cb697b2 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -64,6 +64,7 @@ #include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_options.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_xml_nsctx.h" #include "clixon_xpath_ctx.h" @@ -708,7 +709,7 @@ xml_tree_prune_flagged(cxobj *xt, /*! Add prefix:namespace pair to xml node, set cache, prefix, etc */ -int +static int add_namespace(cxobj *x1, /* target */ cxobj *x1p, char *prefix1, @@ -746,6 +747,49 @@ add_namespace(cxobj *x1, /* target */ return retval; } +/*! Change namespace of XML node + * + * @param[in] x XML node + * @param[in] namespace Change to this namespace (if it does not already belong to it) + * @param[in] prefix If change, use this namespace + * @param 0 OK + * @param -1 Error + */ +int +xml_namespace_change(cxobj *x, + char *namespace, + char *prefix) +{ + int retval = -1; + cxobj *xp; + char *ns0 = NULL; /* existing namespace */ + char *prefix0 = NULL; /* existing prefix */ + + ns0 = NULL; + if (xml2ns(x, xml_prefix(x), &ns0) < 0) + goto done; + if (strcmp(ns0, namespace) == 0) + goto ok; /* Already has right namespace */ + /* Is namespace already declared? */ + if (xml2prefix(x, namespace, &prefix0) == 1){ + /* Yes it is declared and the prefix is pexists */ + if (xml_prefix_set(x, prefix0) < 0) + goto done; + } + else{ /* Namespace does not exist, add it */ + if ((xp = xml_parent(x)) == NULL){ + clicon_err(OE_XML, ENOENT, "XML node must have parent"); + goto done; + } + if (add_namespace(x, xp, prefix0, namespace) < 0) + goto done; + } + ok: + retval = 0; + done: + return retval; +} + /*! Add default values (if not set) * @param[in] xt XML tree with some node marked * @param[in] arg Ignored diff --git a/lib/src/clixon_xpath.c b/lib/src/clixon_xpath.c index 9f658c2f..385608bf 100644 --- a/lib/src/clixon_xpath.c +++ b/lib/src/clixon_xpath.c @@ -122,11 +122,11 @@ static const map_str2int xpath_tree_map[] = { static const map_str2int axis_type_map[] = { {"NaN", A_NAN}, {"ancestor", A_ANCESTOR}, - {"ancestor-or-selgf", A_ANCESTOR_OR_SELF}, + {"ancestor-or-self", A_ANCESTOR_OR_SELF}, {"attribute", A_ATTRIBUTE}, {"child", A_CHILD}, {"descendant", A_DESCENDANT}, - {"descendeant-or-self", A_DESCENDANT_OR_SELF}, + {"descendant-or-self", A_DESCENDANT_OR_SELF}, {"following", A_FOLLOWING}, {"following-sibling", A_FOLLOWING_SIBLING}, {"namespace", A_NAMESPACE}, diff --git a/lib/src/clixon_xpath_eval.c b/lib/src/clixon_xpath_eval.c index dd3c0965..f712fcad 100644 --- a/lib/src/clixon_xpath_eval.c +++ b/lib/src/clixon_xpath_eval.c @@ -362,8 +362,19 @@ xp_eval_step(xp_ctx *xc0, } ctx_nodeset_replace(xc, vec, veclen); break; - case A_DESCENDANT: case A_DESCENDANT_OR_SELF: + for (i=0; ixc_size; i++){ + xv = xc->xc_nodeset[i]; + if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0) + goto done; + } + for (i=0; ixc_nodeset, &xc->xc_size) < 0) + goto done; + } + break; + case A_DESCENDANT: for (i=0; ixc_size; i++){ xv = xc->xc_nodeset[i]; if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0) diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 09e6a822..9807c62a 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -87,6 +87,7 @@ #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_data.h" #include "clixon_options.h" diff --git a/lib/src/clixon_yang_module.c b/lib/src/clixon_yang_module.c index 4210fa6a..2888fbd3 100644 --- a/lib/src/clixon_yang_module.c +++ b/lib/src/clixon_yang_module.c @@ -74,10 +74,10 @@ #include "clixon_xpath.h" #include "clixon_options.h" #include "clixon_data.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_netconf_lib.h" #include "clixon_xml_map.h" -#include "clixon_yang_module.h" #include "clixon_yang_internal.h" /* internal */ modstate_diff_t * @@ -97,6 +97,8 @@ modstate_diff_free(modstate_diff_t *md) { if (md == NULL) return 0; + if (md->md_set_id) + free(md->md_set_id); if (md->md_del) xml_free(md->md_del); if (md->md_mod) diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index 0cd50346..fb59b3e4 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -94,6 +94,7 @@ #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_options.h" #include "clixon_yang.h" diff --git a/test/test_datastore_repair.sh b/test/test_datastore_repair.sh index 800fb8a7..2a98d1e6 100755 --- a/test/test_datastore_repair.sh +++ b/test/test_datastore_repair.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash -# Keep a vector of {xpath, namespace} pairs, match xpath in parsed XML and check if symbols belong to namespace -# If not, set that namespace. -# This is a primitive upgrade mechanism if th emore general upgrade mechanism can not be used +# Test of the general-purpose (raw) upgrade mechanism. +# Input is a startup db without mod-state info. +# It has wrong namespace bindings and needs to remove some nodes +# Output is a valid config woith correct namespaces and removed nods +# The code for this is in the main example backend plugin. # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi @@ -40,6 +42,12 @@ module A{ container y { } } + list remove_me { + key k; + leaf k { + type string; + } + } } EOF @@ -65,7 +73,48 @@ EOF # permission kludges sudo touch $dir/startup_db sudo chmod 666 $dir/startup_db + +# This is how it should look after repair, using prefixes +AFTER=$(cat <foo +EOF +) + +testrun(){ + new "test params: -f $cfg" + # Bring your own backend + if [ $BE -ne 0 ]; then + # kill old backend (if any) + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s startup -f $cfg" + start_backend -s startup -f $cfg + + new "waiting" + wait_backend + fi + + new "netconf get config" + expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$AFTER]]>]]>$" + + if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + fi + +} # end testrun + # Create startup db of "old" db with incorrect augment namespace tagging +# without modstate cat < $dir/startup_db @@ -75,44 +124,12 @@ cat < $dir/startup_db + This node is obsolete + this too EOF -# This is how it should look after repair, using prefixes -AFTER=$(cat <foo -EOF -) +new "general-purpose upgrade without modstate" +testrun -# Or using default: -# foo - -new "test params: -f $cfg" -# Bring your own backend -if [ $BE -ne 0 ]; then - # kill old backend (if any) - new "kill old backend" - sudo clixon_backend -zf $cfg - if [ $? -ne 0 ]; then - err - fi - new "start backend -s startup -f $cfg" - start_backend -s startup -f $cfg - - new "waiting" - wait_backend -fi - -new "netconf get config" -expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$AFTER]]>]]>$" - -new "Kill backend" -# Check if premature kill -pid=$(pgrep -u root -f clixon_backend) -if [ -z "$pid" ]; then - err "backend already dead" -fi -# kill backend -stop_backend -f $cfg - -#rm -rf $dir +rm -rf $dir