datastore repair test branch

apps/backend/backend_commit.c

@@ -197,14 +197,14 @@ startup_common(clicon_handle       h,
 	xt = NULL;
 	goto ok;
     }
-    if (msd){
+    if (clixon_plugin_xmldb_repair(h, db, xt) < 0)
-	/* Here xt is old syntax */
+	goto done;
-	if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0)
+    /* Here xt is old syntax */
-	    goto done;
+    if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0)
-	if (ret == 0)
+	goto done;
-	    goto fail;
+    if (ret == 0)
-	/* Here xt is new syntax */
+	goto fail;
-    }
+
     if ((yspec = clicon_dbspec_yang(h)) == NULL){
 	clicon_err(OE_YANG, 0, "Yang spec not set");
 	goto done;

example/main/example_backend.c

@@ -173,6 +173,98 @@ main_abort(clicon_handle    h,
     return 0;
 }
 
+struct map_str2str{
+    char         *ms_path;
+    char         *ms_ns;
+};
+static const struct map_str2str path_namespace_map[] = {
+    {"/a:x/a:y/a:z/a:w", "urn:example:b"},
+    {"/a:x/a:y/a:z",     "urn:example:b"},
+    {NULL,                          NULL}
+};
+
+static int
+main_repair_one(cxobj *xt,
+		char  *mypath,
+		char  *mynamespace,
+		cvec  *nsc)
+{
+    int        retval = -1;
+    cxobj    **xvec = NULL;
+    size_t     xlen; 
+    char      *myprefix = NULL;
+    int        i;
+    cxobj     *x;
+    char      *namespace;
+    char      *pexist = NULL; /* Existing prefix */
+    
+    if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, mypath) < 0) 
+	goto done;
+    if (xml_nsctx_get_prefix(nsc, mynamespace, &myprefix) == 0){
+	clicon_err(OE_XML, ENOENT, "Namespace %s not found in canonical namespace map",
+		   mynamespace);
+	goto done;
+    }
+    for (i=0; i<xlen; i++){
+	x = xvec[i];
+	namespace = NULL;
+	if (xml2ns(x, xml_prefix(x), &namespace) < 0)
+	    goto done;
+	if (strcmp(namespace, mynamespace) == 0)
+	    continue; /* After this point namespace is not correct */
+	/* Is namespace already declared? */
+	if (xml2prefix(x, mynamespace, &pexist) == 1){
+	    /* Yes it is declared and the prefix is pexists */
+	    if (xml_prefix_set(x, pexist) < 0)
+		goto done;
+	}
+	else{ /* Namespace does not exist, add it */
+	    if (add_namespace(x, xml_parent(x), myprefix, mynamespace) < 0)
+		goto done;
+	}
+    }
+    retval = 0;
+ done:
+    if (xvec)
+	free(xvec);
+    return retval;
+}
+
+/*! Datastore repair callback 
+ * Gets called on startup after initial XML parsing, but before upgrading and before validation
+ * @param[in] h    Clicon handle
+ * @param[in] db   Name of datastore, eg "running"
+ * @param[in] xt   XML tree (to repair)
+ * @retval   -1    Error
+ * @retval    0    OK
+ */
+int
+main_repair(clicon_handle h,
+	    char         *db,
+	    cxobj        *xt)
+{
+    int        retval = -1;
+    cvec      *nsc = NULL; /* Canonical namespace */
+    yang_stmt *yspec;
+    const struct map_str2str *ms;
+
+    if (strcmp(db, "startup") != 0) /* skip other than startup */
+	goto ok;
+    yspec = clicon_dbspec_yang(h);     /* Get all yangs */
+    /* Get canonical namespace */
+    if (xml_nsctx_yangspec(yspec, &nsc) < 0)
+	goto done;
+    for (ms = &path_namespace_map[0]; ms->ms_path; ms++)
+	if (main_repair_one(xt, ms->ms_path, ms->ms_ns, nsc) < 0)
+	    goto done;
+ ok:
+    retval = 0;
+ done:
+    if (nsc)
+	cvec_free(nsc);
+    return retval;
+}
+
 /*! Routing example notification timer handler. Here is where the periodic action is 
  */
 static int
@@ -684,7 +776,8 @@ static clixon_plugin_api api = {
     .ca_trans_commit=main_commit,           /* trans commit */
     .ca_trans_revert=main_revert,           /* trans revert */
     .ca_trans_end=main_end,                 /* trans end */
-    .ca_trans_abort=main_abort              /* trans abort */
+    .ca_trans_abort=main_abort,              /* trans abort */
+    .ca_xmldb_repair=main_repair            /* xmldb repair. */
 };
 
 /*! Backend plugin initialization

lib/clixon/clixon_plugin.h

@@ -159,6 +159,16 @@ typedef int (trans_cb_t)(clicon_handle h, transaction_data td);
  */
 typedef char *(cli_prompthook_t)(clicon_handle, char *mode);
 
+/*! Datastore repair callback 
+ * Gets called on startup after initial XML parsing, but before upgrading and before validation
+ * @param[in] h    Clicon handle
+ * @param[in] db   Name of datastore, eg "running"
+ * @param[in] xt   XML tree. Repair this "in place"
+ * @retval   -1    Error
+ * @retval    0    OK
+ */
+typedef int (xmldb_repair_t)(clicon_handle h, char *db, cxobj *xt);
+
 /*! Startup status for use in startup-callback
  * Note that for STARTUP_ERR and _INVALID, running runs in failsafe mode
  * and startup contains the erroneous or invalid database.
@@ -206,8 +216,9 @@ struct clixon_plugin_api{
 	    trans_cb_t       *cb_trans_revert;   /* Transaction revert */
 	    trans_cb_t       *cb_trans_end;	 /* Transaction completed  */
     	    trans_cb_t       *cb_trans_abort;	 /* Transaction aborted */    
-	} cau_backend;
+	    xmldb_repair_t   *cb_xmldb_repair;	 /* Repair datastore on load */
 
+	} cau_backend;
     } u;
 };
 /* Access fields */
@@ -224,6 +235,8 @@ struct clixon_plugin_api{
 #define ca_trans_revert   u.cau_backend.cb_trans_revert
 #define ca_trans_end      u.cau_backend.cb_trans_end
 #define ca_trans_abort    u.cau_backend.cb_trans_abort
+#define ca_xmldb_repair   u.cau_backend.cb_xmldb_repair
+
 
 /*
  * Macros
@@ -271,6 +284,8 @@ int clixon_plugin_auth(clicon_handle h, void *arg);
 
 int clixon_plugin_extension(clicon_handle h, yang_stmt *yext, yang_stmt *ys);
 
+int clixon_plugin_xmldb_repair(clicon_handle h, char *db, cxobj *xt);
+
 /* rpc callback API */
 int rpc_callback_register(clicon_handle h, clicon_rpc_cb cb, void *arg, char *namespace, char *name);
 int rpc_callback_delete_all(clicon_handle h);

lib/clixon/clixon_xml_map.h

@@ -56,6 +56,7 @@ int xml_diff(yang_stmt *yspec, cxobj *x0, cxobj *x1,
 	     cxobj ***changed_x0, cxobj ***changed_x1, size_t *changedlen);
 int xml_tree_prune_flagged_sub(cxobj *xt, int flag, int test, int *upmark);
 int xml_tree_prune_flagged(cxobj *xt, int flag, int test);
+int add_namespace(cxobj *x1, cxobj *x1p, char *prefix1, char *namespace);
 int xml_default(cxobj *x, void  *arg);
 int xml_sanity(cxobj *x, void  *arg);
 int xml_non_config_data(cxobj *xt, void *arg);

lib/src/clixon_plugin.c

@@ -467,6 +467,33 @@ clixon_plugin_extension(clicon_handle h,
     }
     return retval;
 }
+/*! Call plugin module repair in all plugins
+ *
+ * Repair datastore on load before or as an alternative to the upgrading mechanism
+ * @param[in]  h       Clicon handle
+ * Call plugin start functions (if defined)
+ * @note  Start functions used to have argc/argv. Use clicon_argv_get() instead
+ */
+int
+clixon_plugin_xmldb_repair(clicon_handle h,
+			   char         *db,
+			   cxobj        *xt)
+{
+    clixon_plugin  *cp;
+    int             i;
+    xmldb_repair_t *repairfn;
+    
+    for (i = 0; i < _clixon_nplugins; i++) {
+	cp = &_clixon_plugins[i];
+	if ((repairfn = cp->cp_api.ca_xmldb_repair) == NULL)
+	    continue;
+	if (repairfn(h, db, xt) < 0) {
+	    clicon_debug(1, "%s() failed", __FUNCTION__);
+	    return -1;
+	}
+    }
+    return 0;
+}
 
 /*--------------------------------------------------------------------
  * RPC callbacks for both client/frontend and backend plugins.

lib/src/clixon_xml_map.c

@@ -708,7 +708,7 @@ xml_tree_prune_flagged(cxobj *xt,
 
 /*! Add prefix:namespace pair to xml node, set cache, prefix, etc
  */
-static int
+int
 add_namespace(cxobj *x1, /* target */
 	      cxobj *x1p,
 	      char  *prefix1,

test/test_datastore_repair.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# Keep a vector of {xpath, namespace} pairs, match xpath in parsed XML and check if symbols belong to namespace
+# If not, set that namespace.
+# This is a primitive upgrade mechanism if th emore general upgrade mechanism can not be used
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyangA=$dir/A.yang
+fyangB=$dir/B.yang
+
+# Create configuration
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_MODSTATE>true</CLICON_XMLDB_MODSTATE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+</clixon-config>
+EOF
+
+# Yang module A (base)
+cat <<EOF > $fyangA
+module A{
+  prefix a;
+  revision 2020-02-11;
+  namespace "urn:example:a";
+  container x {
+    container y {
+    }
+  }
+}
+EOF
+
+# Yang module B (augments A)
+cat <<EOF > $fyangB
+module B{
+  prefix b;
+  revision 2020-02-11;
+  namespace "urn:example:b";
+  import A {
+     prefix "a";
+  }
+  augment "/a:x/ngrt:y" {
+    container z {
+      leaf w {
+        type string;
+      }
+    }
+  }
+}
+EOF
+
+# permission kludges
+sudo touch $dir/startup_db
+sudo chmod 666 $dir/startup_db
+# Create startup db of "old" db with incorrect augment namespace tagging
+cat <<EOF > $dir/startup_db
+<config>
+   <x xmlns="urn:example:a">
+     <y>
+        <z>
+          <w>foo</w>
+        </z>
+     </y>
+   </x>
+</config>
+EOF
+
+# This is how it should look after repair, using prefixes
+AFTER=$(cat <<EOF
+<x xmlns="urn:example:a"><y><b:z xmlns:b="urn:example:b"><b:w>foo</b:w></b:z></y></x>
+EOF
+)
+
+# Or using default:
+# <config><x xmlns="urn:example:a"><y><z xmlns="urn:example:b"><w>foo</w></z></y></x></config>
+
+new "test params: -f $cfg"
+# Bring your own backend
+if [ $BE -ne 0 ]; then
+    # kill old backend (if any)
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+    new "start backend -s startup -f $cfg"
+    start_backend -s startup -f $cfg
+
+    new "waiting"
+    wait_backend
+fi
+
+new "netconf get config"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data>$AFTER</data></rpc-reply>]]>]]>$"
+
+new "Kill backend"
+# Check if premature kill
+pid=$(pgrep -u root -f clixon_backend)
+if [ -z "$pid" ]; then
+    err "backend already dead"
+fi
+# kill backend
+stop_backend -f $cfg
+
+#rm -rf $dir