experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'datastore-repair' into release-4.3

Browse source
This commit is contained in:
Olof hagsand 2020-02-15 14:31:38 +01:00
commit 7fd1751302
18 changed files with 415 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

18
apps/backend/backend_commit.c
View file

@ -197,14 +197,16 @@ startup_common(clicon_handle h,
xt = NULL;
goto ok;
}
if (msd){
/* Here xt is old syntax */
if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0)
goto done;
if (ret == 0)
goto fail;
/* Here xt is new syntax */
}
/* Here xt is old syntax */
/* General purpose datastore upgrade */
if (clixon_plugin_datastore_upgrade(h, db, xt, msd) < 0)
goto done;
/* Module-specific upgrade callbacks */
if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0)
goto done;
if (ret == 0)
goto fail;
if ((yspec = clicon_dbspec_yang(h)) == NULL){
clicon_err(OE_YANG, 0, "Yang spec not set");
goto done;

132
example/main/example_backend.c
View file

@ -71,11 +71,15 @@ static int _reset = 0;
*/
static int _state = 0;
/*! Variable to control upgrade callbacks.
/*! Variable to control module-specific upgrade callbacks.
* If set, call test-case for upgrading ietf-interfaces, otherwise call
* auto-upgrade
*/
static int _upgrade = 0;
static int _module_upgrade = 0;
/*! Variable to control general-purpose upgrade callbacks.
*/
static int _general_upgrade = 0;
/*! Variable to control transaction logging (for debug)
* If set, call syslog for every transaction callback
@ -410,6 +414,116 @@ example_extension(clicon_handle h,
return retval;
}
/* Here follows code for general-purpose datastore upgrade
* Nodes affected are identified by paths.
* In this example nodes' namespaces are changed, or they are removed altogether
*/
/* Rename the namespaces of these paths.
* That is, paths (on the left) should get namespaces (to the right)
*/
static const map_str2str namespace_map[] = {
{"/a:x/a:y/a:z/descendant-or-self::node()", "urn:example:b"},
/* add more paths to be renamed here */
{NULL, NULL}
};
/* Remove these paths */
static const char *remove_map[] = {
"/a:remove_me",
/* add more paths to be deleted here */
NULL
};
/*! General-purpose datastore upgrade callback called once on startup
*
* Gets called on startup after initial XML parsing, but before module-specific upgrades
* and before validation.
* @param[in] h Clicon handle
* @param[in] db Name of datastore, eg "running", "startup" or "tmp"
* @param[in] xt XML tree. Upgrade this "in place"
* @param[in] msd Info on datastore module-state, if any
* @retval -1 Error
* @retval 0 OK
*/
int
example_upgrade(clicon_handle h,
char *db,
cxobj *xt,
modstate_diff_t *msd)
{
int retval = -1;
cvec *nsc = NULL; /* Canonical namespace */
yang_stmt *yspec;
const struct map_str2str *ms; /* map iterator */
cxobj **xvec = NULL; /* vector of result nodes */
size_t xlen;
int i;
const char **pp;
if (_general_upgrade == 0)
goto ok;
if (strcmp(db, "startup") != 0) /* skip other than startup datastore */
goto ok;
if (msd && msd->md_status) /* skip if there is proper module-state in datastore */
goto ok;
yspec = clicon_dbspec_yang(h); /* Get all yangs */
/* Get canonical namespaces for using "normalized" prefixes */
if (xml_nsctx_yangspec(yspec, &nsc) < 0)
goto done;
/* 1. Remove paths */
for (pp = remove_map; *pp; ++pp){
/* Find all nodes matching n */
if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, *pp) < 0)
goto done;
/* Remove them */
/* Loop through all nodes matching mypath and change theoir namespace */
for (i=0; i<xlen; i++){
if (xml_purge(xvec[i]) < 0)
goto done;
}
if (xvec){
free(xvec);
xvec = NULL;
}
}
/* 2. Rename namespaces of the paths declared in the namespace map
*/
for (ms = &namespace_map[0]; ms->ms_s0; ms++){
char *mypath;
char *mynamespace;
char *myprefix = NULL;
mypath = ms->ms_s0;
mynamespace = ms->ms_s1;
if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){
clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map",
mynamespace);
goto done;
}
/* Find all nodes matching mypath */
if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0)
goto done;
/* Loop through all nodes matching mypath and change theoir namespace */
for (i=0; i<xlen; i++){
/* Change namespace of this node (using myprefix) */
if (xml_namespace_change(xvec[i], mynamespace, myprefix) < 0)
goto done;
}
if (xvec){
free(xvec);
xvec = NULL;
}
}
ok:
retval = 0;
done:
if (xvec)
free(xvec);
if (nsc)
cvec_free(nsc);
return retval;
}
/*! Testcase upgrade function moving interfaces-state to interfaces
* @param[in] h Clicon handle
* @param[in] xn XML tree to be updated
@ -684,7 +798,8 @@ static clixon_plugin_api api = {
.ca_trans_commit=main_commit, /* trans commit */
.ca_trans_revert=main_revert, /* trans revert */
.ca_trans_end=main_end, /* trans end */
.ca_trans_abort=main_abort /* trans abort */
.ca_trans_abort=main_abort, /* trans abort */
.ca_datastore_upgrade=example_upgrade /* general-purpose upgrade. */
};
/*! Backend plugin initialization
@ -709,7 +824,7 @@ clixon_plugin_init(clicon_handle h)
goto done;
opterr = 0;
optind = 1;
while ((c = getopt(argc, argv, "rsut:")) != -1)
while ((c = getopt(argc, argv, "rsuUt:")) != -1)
switch (c) {
case 'r':
_reset = 1;
@ -717,8 +832,11 @@ clixon_plugin_init(clicon_handle h)
case 's':
_state = 1;
break;
case 'u':
_upgrade = 1;
case 'u': /* module-specific upgrade */
_module_upgrade = 1;
break;
case 'U': /* general-purpose upgrade */
_general_upgrade = 1;
break;
case 't': /* transaction log */
_transaction_log = 1;
@ -776,7 +894,7 @@ clixon_plugin_init(clicon_handle h)
/* Upgrade callback: if you start the backend with -- -u you will get the
* test interface example. Otherwise the auto-upgrade feature is enabled.
*/
if (_upgrade){
if (_module_upgrade == 1){
if (upgrade_callback_register(h, upgrade_2016, "urn:example:interfaces", 20140508, 20160101, NULL) < 0)
goto done;
if (upgrade_callback_register(h, upgrade_2018, "urn:example:interfaces", 20160101, 20180220, NULL) < 0)

19
lib/clixon/clixon_plugin.h
View file

@ -159,6 +159,19 @@ typedef int (trans_cb_t)(clicon_handle h, transaction_data td);
*/
typedef char *(cli_prompthook_t)(clicon_handle, char *mode);
/*! General-purpose datastore upgrade callback called once on startup
*
* Gets called on startup after initial XML parsing, but before module-specific upgrades
* and before validation.
* @param[in] h Clicon handle
* @param[in] db Name of datastore, eg "running", "startup" or "tmp"
* @param[in] xt XML tree. Upgrade this "in place"
* @param[in] msd Info on datastore module-state, if any
* @retval -1 Error
* @retval 0 OK
*/
typedef int (datastore_upgrade_t)(clicon_handle h, char *db, cxobj *xt, modstate_diff_t *msd);
/*! Startup status for use in startup-callback
* Note that for STARTUP_ERR and _INVALID, running runs in failsafe mode
* and startup contains the erroneous or invalid database.
@ -206,8 +219,8 @@ struct clixon_plugin_api{
trans_cb_t *cb_trans_revert; /* Transaction revert */
trans_cb_t *cb_trans_end; /* Transaction completed */
trans_cb_t *cb_trans_abort; /* Transaction aborted */
datastore_upgrade_t *cb_datastore_upgrade; /* General-purpose datastore upgrade */
} cau_backend;
} u;
};
/* Access fields */
@ -224,6 +237,8 @@ struct clixon_plugin_api{
#define ca_trans_revert u.cau_backend.cb_trans_revert
#define ca_trans_end u.cau_backend.cb_trans_end
#define ca_trans_abort u.cau_backend.cb_trans_abort
#define ca_datastore_upgrade u.cau_backend.cb_datastore_upgrade
/*
* Macros
@ -271,6 +286,8 @@ int clixon_plugin_auth(clicon_handle h, void *arg);
int clixon_plugin_extension(clicon_handle h, yang_stmt *yext, yang_stmt *ys);
int clixon_plugin_datastore_upgrade(clicon_handle h, char *db, cxobj *xt, modstate_diff_t *msd);
/* rpc callback API */
int rpc_callback_register(clicon_handle h, clicon_rpc_cb cb, void *arg, char *namespace, char *name);
int rpc_callback_delete_all(clicon_handle h);

10
lib/clixon/clixon_string.h
View file

@ -36,7 +36,7 @@
#ifndef _CLIXON_STRING_H_
#define _CLIXON_STRING_H_
/* Struct used to map between int and strings. Typically used to map between
/*! Struct used to map between int and strings. Typically used to map between
* values and their names. Note NULL terminated
* Example:
* @code
@ -55,6 +55,14 @@ struct map_str2int{
};
typedef struct map_str2int map_str2int;
/*! Struct used to map between two strings.
*/
struct map_str2str{
char *ms_s0;
char *ms_s1;
};
typedef struct map_str2str map_str2str;
/*! A malloc version that aligns on 4 bytes. To avoid warning from valgrind */
#define align4(s) (((s)/4)*4 + 4)

1
lib/clixon/clixon_xml_map.h
View file

@ -56,6 +56,7 @@ int xml_diff(yang_stmt *yspec, cxobj *x0, cxobj *x1,
cxobj ***changed_x0, cxobj ***changed_x1, size_t *changedlen);
int xml_tree_prune_flagged_sub(cxobj *xt, int flag, int test, int *upmark);
int xml_tree_prune_flagged(cxobj *xt, int flag, int test);
int xml_namespace_change(cxobj *x, char *namespace, char *prefix);
int xml_default(cxobj *x, void *arg);
int xml_sanity(cxobj *x, void *arg);
int xml_non_config_data(cxobj *xt, void *arg);

6
lib/clixon/clixon_yang_module.h
View file

@ -51,8 +51,10 @@
* This is in state of flux so it needs to be contained and easily changed.
*/
typedef struct {
cxobj *md_del; /* yang module state deletes */
cxobj *md_mod; /* yang module state modifications */
int md_status; /* 0 if no module-state in a datastore, 1 if there is */
char *md_set_id; /* server-specific identifier */
cxobj *md_del; /* yang module state deletes */
cxobj *md_mod; /* yang module state modifications */
} modstate_diff_t;
/*

1
lib/src/clixon_data.c
View file

@ -67,6 +67,7 @@
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_xml_sort.h"
#include "clixon_yang_module.h"
#include "clixon_options.h"
#include "clixon_plugin.h"
#include "clixon_xpath_ctx.h"

3
lib/src/clixon_datastore.c
View file

@ -69,12 +69,11 @@
#include "clixon_file.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_options.h"
#include "clixon_data.h"
#include "clixon_yang_module.h"
#include "clixon_datastore.h"
#include "clixon_datastore_write.h"
#include "clixon_datastore_read.h"

9
lib/src/clixon_datastore_read.c
View file

@ -245,7 +245,8 @@ text_read_modstate(clicon_handle h,
if ((xmodst = xml_find_type(xt, NULL, "modules-state", CX_ELMNT)) == NULL){
/* 1) There is no modules-state info in the file */
}
else if (xmcache && msd){
else if (xmcache && msd){ /* There is module state. */
msd->md_status = 1;
/* Create diff trees */
if (xml_parse_string("<modules-state xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"/>", yspec, &msd->md_del) < 0)
goto done;
@ -258,6 +259,12 @@ text_read_modstate(clicon_handle h,
/* 3) For each module state m in the file */
while ((xm = xml_child_each(xmodst, xm, CX_ELMNT)) != NULL) {
if (strcmp(xml_name(xm), "module-set-id") == 0){
if (xml_body(xm) && (msd->md_set_id = strdup(xml_body(xm))) == NULL){
clicon_err(OE_UNIX, errno, "strdup");
goto done;
}
}
if (strcmp(xml_name(xm), "module"))
continue; /* ignore other tags, such as module-set-id */
if ((name = xml_find_body(xm, "name")) == NULL)

35
lib/src/clixon_plugin.c
View file

@ -59,6 +59,7 @@
#include "clixon_handle.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
/* List of plugins XXX
@ -467,6 +468,40 @@ clixon_plugin_extension(clicon_handle h,
}
return retval;
}
/*! Call plugingeneral-purpose datastore upgrade in all plugins
*
* @param[in] h Clicon handle
* @param[in] db Name of datastore, eg "running", "startup" or "tmp"
* @param[in] xt XML tree. Upgrade this "in place"
* @param[in] msd Module-state diff, info on datastore module-state
* @retval -1 Error
* @retval 0 OK
* Upgrade datastore on load before or as an alternative to module-specific upgrading mechanism
* @param[in] h Clicon handle
* Call plugin start functions (if defined)
* @note Start functions used to have argc/argv. Use clicon_argv_get() instead
*/
int
clixon_plugin_datastore_upgrade(clicon_handle h,
char *db,
cxobj *xt,
modstate_diff_t *msd)
{
clixon_plugin *cp;
int i;
datastore_upgrade_t *repairfn;
for (i = 0; i < _clixon_nplugins; i++) {
cp = &_clixon_plugins[i];
if ((repairfn = cp->cp_api.ca_datastore_upgrade) == NULL)
continue;
if (repairfn(h, db, xt, msd) < 0) {
clicon_debug(1, "%s() failed", __FUNCTION__);
return -1;
}
}
return 0;
}
/*--------------------------------------------------------------------
* RPC callbacks for both client/frontend and backend plugins.

1
lib/src/clixon_proto_client.c
View file

@ -64,6 +64,7 @@
#include "clixon_xml.h"
#include "clixon_options.h"
#include "clixon_data.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_string.h"
#include "clixon_xpath_ctx.h"

44
lib/src/clixon_xml_map.c
View file

@ -64,6 +64,7 @@
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_options.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_xml_nsctx.h"
#include "clixon_xpath_ctx.h"
@ -746,6 +747,49 @@ add_namespace(cxobj *x1, /* target */
return retval;
}
/*! Change namespace of XML node
*
* @param[in] x XML node
* @param[in] namespace Change to this namespace (if it does not already belong to it)
* @param[in] prefix If change, use this namespace
* @param 0 OK
* @param -1 Error
*/
int
xml_namespace_change(cxobj *x,
char *namespace,
char *prefix)
{
int retval = -1;
cxobj *xp;
char *ns0 = NULL; /* existing namespace */
char *prefix0 = NULL; /* existing prefix */
ns0 = NULL;
if (xml2ns(x, xml_prefix(x), &ns0) < 0)
goto done;
if (strcmp(ns0, namespace) == 0)
goto ok; /* Already has right namespace */
/* Is namespace already declared? */
if (xml2prefix(x, namespace, &prefix0) == 1){
/* Yes it is declared and the prefix is pexists */
if (xml_prefix_set(x, prefix0) < 0)
goto done;
}
else{ /* Namespace does not exist, add it */
if ((xp = xml_parent(x)) == NULL){
clicon_err(OE_XML, ENOENT, "XML node must have parent");
goto done;
}
if (add_namespace(x, xp, prefix0, namespace) < 0)
goto done;
}
ok:
retval = 0;
done:
return retval;
}
/*! Add default values (if not set)
* @param[in] xt XML tree with some node marked
* @param[in] arg Ignored

4
lib/src/clixon_xpath.c
View file

@ -122,11 +122,11 @@ static const map_str2int xpath_tree_map[] = {
static const map_str2int axis_type_map[] = {
{"NaN", A_NAN},
{"ancestor", A_ANCESTOR},
{"ancestor-or-selgf", A_ANCESTOR_OR_SELF},
{"ancestor-or-self", A_ANCESTOR_OR_SELF},
{"attribute", A_ATTRIBUTE},
{"child", A_CHILD},
{"descendant", A_DESCENDANT},
{"descendeant-or-self", A_DESCENDANT_OR_SELF},
{"descendant-or-self", A_DESCENDANT_OR_SELF},
{"following", A_FOLLOWING},
{"following-sibling", A_FOLLOWING_SIBLING},
{"namespace", A_NAMESPACE},

17
lib/src/clixon_xpath_eval.c
View file

@ -362,8 +362,23 @@ xp_eval_step(xp_ctx *xc0,
}
ctx_nodeset_replace(xc, vec, veclen);
break;
case A_DESCENDANT:
case A_DESCENDANT_OR_SELF:
for (i=0; i<xc->xc_size; i++){
xv = xc->xc_nodeset[i];
if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0)
goto done;
}
for (i=0; i<veclen; i++){
x = vec[i];
if (cxvec_append(x, &xc->xc_nodeset, &xc->xc_size) < 0)
goto done;
}
if (vec){
free(vec);
vec = NULL;
}
break;
case A_DESCENDANT:
for (i=0; i<xc->xc_size; i++){
xv = xc->xc_nodeset[i];
if (nodetest_recursive(xv, xs->xs_c0, CX_ELMNT, 0x0, nsc, localonly, &vec, &veclen) < 0)

1
lib/src/clixon_yang.c
View file

@ -87,6 +87,7 @@
#include "clixon_yang.h"
#include "clixon_hash.h"
#include "clixon_xml.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_data.h"
#include "clixon_options.h"

4
lib/src/clixon_yang_module.c
View file

@ -74,10 +74,10 @@
#include "clixon_xpath.h"
#include "clixon_options.h"
#include "clixon_data.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_netconf_lib.h"
#include "clixon_xml_map.h"
#include "clixon_yang_module.h"
#include "clixon_yang_internal.h" /* internal */
modstate_diff_t *
@ -97,6 +97,8 @@ modstate_diff_free(modstate_diff_t *md)
{
if (md == NULL)
return 0;
if (md->md_set_id)
free(md->md_set_id);
if (md->md_del)
xml_free(md->md_del);
if (md->md_mod)

1
lib/src/clixon_yang_type.c
View file

@ -94,6 +94,7 @@
#include "clixon_yang.h"
#include "clixon_hash.h"
#include "clixon_xml.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_options.h"
#include "clixon_yang.h"

135
test/test_datastore_repair.sh Executable file
View file

@ -0,0 +1,135 @@
#!/usr/bin/env bash
# Test of the general-purpose (raw) upgrade mechanism.
# Input is a startup db without mod-state info.
# It has wrong namespace bindings and needs to remove some nodes
# Output is a valid config woith correct namespaces and removed nods
# The code for this is in the main example backend plugin.
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
APPNAME=example
cfg=$dir/conf_yang.xml
fyangA=$dir/A.yang
fyangB=$dir/B.yang
# Create configuration
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
<CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
<CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
<CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
<CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
</clixon-config>
EOF
# Yang module A (base)
cat <<EOF > $fyangA
module A{
prefix a;
revision 2020-02-11;
namespace "urn:example:a";
container x {
container y {
}
}
list remove_me {
key k;
leaf k {
type string;
}
}
}
EOF
# Yang module B (augments A)
cat <<EOF > $fyangB
module B{
prefix b;
revision 2020-02-11;
namespace "urn:example:b";
import A {
prefix "a";
}
augment "/a:x/ngrt:y" {
container z {
leaf w {
type string;
}
}
}
}
EOF
# permission kludges
sudo touch $dir/startup_db
sudo chmod 666 $dir/startup_db
# This is how it should look after repair, using prefixes
AFTER=$(cat <<EOF
<x xmlns="urn:example:a"><y><b:z xmlns:b="urn:example:b"><b:w>foo</b:w></b:z></y></x>
EOF
)
testrun(){
new "test params: -f $cfg -- -U"
# Bring your own backend
if [ $BE -ne 0 ]; then
# kill old backend (if any)
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s startup -f $cfg -- -U"
start_backend -s startup -f $cfg -- -U
new "waiting"
wait_backend
fi
new "netconf get config"
expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data>$AFTER</data></rpc-reply>]]>]]>$"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
} # end testrun
# Create startup db of "old" db with incorrect augment namespace tagging
# without modstate
cat <<EOF > $dir/startup_db
<config>
<x xmlns="urn:example:a">
<y>
<z>
<w>foo</w>
</z>
</y>
</x>
<remove_me xmlns="urn:example:a"><k>This node is obsolete</k></remove_me>
<remove_me xmlns="urn:example:a"><k>this too</k></remove_me>
</config>
EOF
new "general-purpose upgrade without modstate"
testrun
rm -rf $dir