diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 890be1d5..1f2ea859 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -197,14 +197,16 @@ startup_common(clicon_handle h, xt = NULL; goto ok; } - if (msd){ - /* Here xt is old syntax */ - if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) - goto done; - if (ret == 0) - goto fail; - /* Here xt is new syntax */ - } + /* Here xt is old syntax */ + /* General purpose datastore upgrade */ + if (clixon_plugin_datastore_upgrade(h, db, xt, msd) < 0) + goto done; + /* Module-specific upgrade callbacks */ + if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) + goto done; + if (ret == 0) + goto fail; + if ((yspec = clicon_dbspec_yang(h)) == NULL){ clicon_err(OE_YANG, 0, "Yang spec not set"); goto done; diff --git a/example/main/example_backend.c b/example/main/example_backend.c index b8db088d..a12679dc 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -71,11 +71,15 @@ static int _reset = 0; */ static int _state = 0; -/*! Variable to control upgrade callbacks. +/*! Variable to control module-specific upgrade callbacks. * If set, call test-case for upgrading ietf-interfaces, otherwise call * auto-upgrade */ -static int _upgrade = 0; +static int _module_upgrade = 0; + +/*! Variable to control general-purpose upgrade callbacks. + */ +static int _general_upgrade = 0; /*! Variable to control transaction logging (for debug) * If set, call syslog for every transaction callback @@ -410,6 +414,116 @@ example_extension(clicon_handle h, return retval; } +/* Here follows code for general-purpose datastore upgrade + * Nodes affected are identified by paths. + * In this example nodes' namespaces are changed, or they are removed altogether + */ +/* Rename the namespaces of these paths. + * That is, paths (on the left) should get namespaces (to the right) + */ +static const map_str2str namespace_map[] = { + {"/a:x/a:y/a:z/descendant-or-self::node()", "urn:example:b"}, + /* add more paths to be renamed here */ + {NULL, NULL} +}; + +/* Remove these paths */ +static const char *remove_map[] = { + "/a:remove_me", + /* add more paths to be deleted here */ + NULL +}; + +/*! General-purpose datastore upgrade callback called once on startup + * + * Gets called on startup after initial XML parsing, but before module-specific upgrades + * and before validation. + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running", "startup" or "tmp" + * @param[in] xt XML tree. Upgrade this "in place" + * @param[in] msd Info on datastore module-state, if any + * @retval -1 Error + * @retval 0 OK + */ +int +example_upgrade(clicon_handle h, + char *db, + cxobj *xt, + modstate_diff_t *msd) +{ + int retval = -1; + cvec *nsc = NULL; /* Canonical namespace */ + yang_stmt *yspec; + const struct map_str2str *ms; /* map iterator */ + cxobj **xvec = NULL; /* vector of result nodes */ + size_t xlen; + int i; + const char **pp; + + if (_general_upgrade == 0) + goto ok; + if (strcmp(db, "startup") != 0) /* skip other than startup datastore */ + goto ok; + if (msd && msd->md_status) /* skip if there is proper module-state in datastore */ + goto ok; + yspec = clicon_dbspec_yang(h); /* Get all yangs */ + /* Get canonical namespaces for using "normalized" prefixes */ + if (xml_nsctx_yangspec(yspec, &nsc) < 0) + goto done; + /* 1. Remove paths */ + for (pp = remove_map; *pp; ++pp){ + /* Find all nodes matching n */ + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, *pp) < 0) + goto done; + /* Remove them */ + /* Loop through all nodes matching mypath and change theoir namespace */ + for (i=0; ims_s0; ms++){ + char *mypath; + char *mynamespace; + char *myprefix = NULL; + + mypath = ms->ms_s0; + mynamespace = ms->ms_s1; + if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){ + clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map", + mynamespace); + goto done; + } + /* Find all nodes matching mypath */ + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) + goto done; + /* Loop through all nodes matching mypath and change theoir namespace */ + for (i=0; imd_status = 1; /* Create diff trees */ if (xml_parse_string("", yspec, &msd->md_del) < 0) goto done; @@ -258,6 +259,12 @@ text_read_modstate(clicon_handle h, /* 3) For each module state m in the file */ while ((xm = xml_child_each(xmodst, xm, CX_ELMNT)) != NULL) { + if (strcmp(xml_name(xm), "module-set-id") == 0){ + if (xml_body(xm) && (msd->md_set_id = strdup(xml_body(xm))) == NULL){ + clicon_err(OE_UNIX, errno, "strdup"); + goto done; + } + } if (strcmp(xml_name(xm), "module")) continue; /* ignore other tags, such as module-set-id */ if ((name = xml_find_body(xm, "name")) == NULL) diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index 8bfdefd8..f997702e 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -59,6 +59,7 @@ #include "clixon_handle.h" #include "clixon_yang.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" /* List of plugins XXX @@ -467,6 +468,40 @@ clixon_plugin_extension(clicon_handle h, } return retval; } +/*! Call plugingeneral-purpose datastore upgrade in all plugins + * + * @param[in] h Clicon handle + * @param[in] db Name of datastore, eg "running", "startup" or "tmp" + * @param[in] xt XML tree. Upgrade this "in place" + * @param[in] msd Module-state diff, info on datastore module-state + * @retval -1 Error + * @retval 0 OK + * Upgrade datastore on load before or as an alternative to module-specific upgrading mechanism + * @param[in] h Clicon handle + * Call plugin start functions (if defined) + * @note Start functions used to have argc/argv. Use clicon_argv_get() instead + */ +int +clixon_plugin_datastore_upgrade(clicon_handle h, + char *db, + cxobj *xt, + modstate_diff_t *msd) +{ + clixon_plugin *cp; + int i; + datastore_upgrade_t *repairfn; + + for (i = 0; i < _clixon_nplugins; i++) { + cp = &_clixon_plugins[i]; + if ((repairfn = cp->cp_api.ca_datastore_upgrade) == NULL) + continue; + if (repairfn(h, db, xt, msd) < 0) { + clicon_debug(1, "%s() failed", __FUNCTION__); + return -1; + } + } + return 0; +} /*-------------------------------------------------------------------- * RPC callbacks for both client/frontend and backend plugins. diff --git a/lib/src/clixon_proto_client.c b/lib/src/clixon_proto_client.c index bdd87f0f..552e5048 100644 --- a/lib/src/clixon_proto_client.c +++ b/lib/src/clixon_proto_client.c @@ -64,6 +64,7 @@ #include "clixon_xml.h" #include "clixon_options.h" #include "clixon_data.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_string.h" #include "clixon_xpath_ctx.h" diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 2d822638..6cb697b2 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -64,6 +64,7 @@ #include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_options.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_xml_nsctx.h" #include "clixon_xpath_ctx.h" @@ -746,6 +747,49 @@ add_namespace(cxobj *x1, /* target */ return retval; } +/*! Change namespace of XML node + * + * @param[in] x XML node + * @param[in] namespace Change to this namespace (if it does not already belong to it) + * @param[in] prefix If change, use this namespace + * @param 0 OK + * @param -1 Error + */ +int +xml_namespace_change(cxobj *x, + char *namespace, + char *prefix) +{ + int retval = -1; + cxobj *xp; + char *ns0 = NULL; /* existing namespace */ + char *prefix0 = NULL; /* existing prefix */ + + ns0 = NULL; + if (xml2ns(x, xml_prefix(x), &ns0) < 0) + goto done; + if (strcmp(ns0, namespace) == 0) + goto ok; /* Already has right namespace */ + /* Is namespace already declared? */ + if (xml2prefix(x, namespace, &prefix0) == 1){ + /* Yes it is declared and the prefix is pexists */ + if (xml_prefix_set(x, prefix0) < 0) + goto done; + } + else{ /* Namespace does not exist, add it */ + if ((xp = xml_parent(x)) == NULL){ + clicon_err(OE_XML, ENOENT, "XML node must have parent"); + goto done; + } + if (add_namespace(x, xp, prefix0, namespace) < 0) + goto done; + } + ok: + retval = 0; + done: + return retval; +} + /*! Add default values (if not set) * @param[in] xt XML tree with some node marked * @param[in] arg Ignored diff --git a/lib/src/clixon_xpath.c b/lib/src/clixon_xpath.c index 9f658c2f..385608bf 100644 --- a/lib/src/clixon_xpath.c +++ b/lib/src/clixon_xpath.c @@ -122,11 +122,11 @@ static const map_str2int xpath_tree_map[] = { static const map_str2int axis_type_map[] = { {"NaN", A_NAN}, {"ancestor", A_ANCESTOR}, - {"ancestor-or-selgf", A_ANCESTOR_OR_SELF}, + {"ancestor-or-self", A_ANCESTOR_OR_SELF}, {"attribute", A_ATTRIBUTE}, {"child", A_CHILD}, {"descendant", A_DESCENDANT}, - {"descendeant-or-self", A_DESCENDANT_OR_SELF}, + {"descendant-or-self", A_DESCENDANT_OR_SELF}, {"following", A_FOLLOWING}, {"following-sibling", A_FOLLOWING_SIBLING}, {"namespace", A_NAMESPACE}, diff --git a/lib/src/clixon_xpath_eval.c b/lib/src/clixon_xpath_eval.c index dd3c0965..1c3a7b77 100644 --- a/lib/src/clixon_xpath_eval.c +++ b/lib/src/clixon_xpath_eval.c @@ -362,8 +362,23 @@ xp_eval_step(xp_ctx *xc0, } ctx_nodeset_replace(xc, vec, veclen); break; - case A_DESCENDANT: case A_DESCENDANT_OR_SELF: + for (i=0; ixc_size; i++){ + xv = xc->xc_nodeset[i]; + if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0) + goto done; + } + for (i=0; ixc_nodeset, &xc->xc_size) < 0) + goto done; + } + if (vec){ + free(vec); + vec = NULL; + } + break; + case A_DESCENDANT: for (i=0; ixc_size; i++){ xv = xc->xc_nodeset[i]; if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0) diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 09e6a822..9807c62a 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -87,6 +87,7 @@ #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_data.h" #include "clixon_options.h" diff --git a/lib/src/clixon_yang_module.c b/lib/src/clixon_yang_module.c index 4210fa6a..2888fbd3 100644 --- a/lib/src/clixon_yang_module.c +++ b/lib/src/clixon_yang_module.c @@ -74,10 +74,10 @@ #include "clixon_xpath.h" #include "clixon_options.h" #include "clixon_data.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_netconf_lib.h" #include "clixon_xml_map.h" -#include "clixon_yang_module.h" #include "clixon_yang_internal.h" /* internal */ modstate_diff_t * @@ -97,6 +97,8 @@ modstate_diff_free(modstate_diff_t *md) { if (md == NULL) return 0; + if (md->md_set_id) + free(md->md_set_id); if (md->md_del) xml_free(md->md_del); if (md->md_mod) diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index 0cd50346..fb59b3e4 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -94,6 +94,7 @@ #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" +#include "clixon_yang_module.h" #include "clixon_plugin.h" #include "clixon_options.h" #include "clixon_yang.h" diff --git a/test/test_datastore_repair.sh b/test/test_datastore_repair.sh new file mode 100755 index 00000000..640e9b48 --- /dev/null +++ b/test/test_datastore_repair.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Test of the general-purpose (raw) upgrade mechanism. +# Input is a startup db without mod-state info. +# It has wrong namespace bindings and needs to remove some nodes +# Output is a valid config woith correct namespaces and removed nods +# The code for this is in the main example backend plugin. + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +cfg=$dir/conf_yang.xml +fyangA=$dir/A.yang +fyangB=$dir/B.yang + +# Create configuration +cat < $cfg + + $cfg + ietf-netconf:startup + /usr/local/share/clixon + $dir + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/lib/example/backend + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + true + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + +EOF + +# Yang module A (base) +cat < $fyangA +module A{ + prefix a; + revision 2020-02-11; + namespace "urn:example:a"; + container x { + container y { + } + } + list remove_me { + key k; + leaf k { + type string; + } + } +} +EOF + +# Yang module B (augments A) +cat < $fyangB +module B{ + prefix b; + revision 2020-02-11; + namespace "urn:example:b"; + import A { + prefix "a"; + } + augment "/a:x/ngrt:y" { + container z { + leaf w { + type string; + } + } + } +} +EOF + +# permission kludges +sudo touch $dir/startup_db +sudo chmod 666 $dir/startup_db + +# This is how it should look after repair, using prefixes +AFTER=$(cat <foo +EOF +) + +testrun(){ + new "test params: -f $cfg -- -U" + # Bring your own backend + if [ $BE -ne 0 ]; then + # kill old backend (if any) + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s startup -f $cfg -- -U" + start_backend -s startup -f $cfg -- -U + + new "waiting" + wait_backend + fi + + new "netconf get config" + expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$AFTER]]>]]>$" + + if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + fi + +} # end testrun + +# Create startup db of "old" db with incorrect augment namespace tagging +# without modstate +cat < $dir/startup_db + + + + + foo + + + + This node is obsolete + this too + +EOF + +new "general-purpose upgrade without modstate" +testrun + +rm -rf $dir