Multi-db: Moved top-level datastore from (eg) running_db to running.d/0.xml

CHANGELOG.md

@@ -35,6 +35,12 @@ Expected: June 2024
 ### API changes on existing protocol/config features
 Users may have to change how they access the system
 
+* If `CLICON_XMLDB_MULTI` is set, datastores are stored in a new directory
+   * Previously: `CLICON_XMLDB_DIR/<db>_db`
+   * New: `CLICON_XMLDB_DIR/<db>d/`
+   * In particular, the top-level is moved from `<db>_db` to `<db>.d/0.xml`
+   * Backward-compatible:
+     * If backend is started with `-s startup` or `-s running` then `<db>_db` is read if `<db>.d/0.xml` is not found
 * Openssl mandatory for all configs, not only restconf
 
 ### Corrected Bugs

apps/backend/backend_main.c

@@ -921,6 +921,13 @@ main(int    argc,
         goto done;
     }
 
+    /* Multi-upgrade: If <db>.d/0.xml does not exist, but <db>_d does, copy <db>_db to <db>.d/0.xml */
+    if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){
+        if (xmldb_multi_upgrade(h, "running") < 0)
+            goto done;
+        if (xmldb_multi_upgrade(h, "startup") < 0)
+            goto done;
+    }
     /* Init running db if it is not there
      * change running_startup -> running or startup depending on if running exists or not
      */

lib/clixon/clixon_datastore.h

@@ -103,5 +103,6 @@ int xmldb_volatile_set(clixon_handle h, const char *db, int value);
 int xmldb_print(clixon_handle h, FILE *f);
 int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix);
 int xmldb_populate(clixon_handle h, const char *db);
+int xmldb_multi_upgrade(clixon_handle h, const char *db);
 
 #endif /* _CLIXON_DATASTORE_H */

lib/src/clixon_datastore.c

@@ -125,6 +125,52 @@ clicon_db_elmnt_set(clixon_handle h,
     return 0;
 }
 
+/*! Translate from symbolic database name to actual filename in file-system
+ *
+ * Internal function for explicit XMLDB_MULTI use or not
+ * @param[in]   h        Clixon handle
+ * @param[in]   db       Symbolic database name, eg "candidate", "running"
+ * @param[in]   multi    Use multi/split datastores, see CLICON_XMLDB_MULTI
+ * @param[out]  filename Filename. Unallocate after use with free()
+ * @retval      0        OK
+ * @retval     -1        Error
+ */
+static int
+xmldb_db2file1(clixon_handle  h,
+              const char    *db,
+               int           multi,
+              char         **filename)
+{
+    int   retval = -1;
+    cbuf *cb = NULL;
+    char *dir;
+
+    if ((cb = cbuf_new()) == NULL){
+        clixon_err(OE_XML, errno, "cbuf_new");
+        goto done;
+    }
+    if ((dir = clicon_xmldb_dir(h)) == NULL){
+        clixon_err(OE_XML, errno, "CLICON_XMLDB_DIR not set");
+        goto done;
+    }
+    /* Multi: write (root) to: <db>.d/0.xml
+     * Classic: write to: <db>_db
+     */
+    if (multi)
+        cprintf(cb, "%s/%s.d/0.xml", dir, db); /* Hardcoded to XML, XXX: JSON? */
+    else
+        cprintf(cb, "%s/%s_db", dir, db);
+    if ((*filename = strdup4(cbuf_get(cb))) == NULL){
+        clixon_err(OE_UNIX, errno, "strdup");
+        goto done;
+    }
+    retval = 0;
+ done:
+    if (cb)
+        cbuf_free(cb);
+    return retval;
+}
+
 /*! Translate from symbolic database name to actual filename in file-system
  *
  * @param[in]   h        Clixon handle
@@ -142,28 +188,7 @@ xmldb_db2file(clixon_handle  h,
               const char    *db,
               char         **filename)
 {
-    int   retval = -1;
+    return xmldb_db2file1(h, db, clicon_option_bool(h, "CLICON_XMLDB_MULTI"), filename);
-    cbuf *cb = NULL;
-    char *dir;
-
-    if ((cb = cbuf_new()) == NULL){
-        clixon_err(OE_XML, errno, "cbuf_new");
-        goto done;
-    }
-    if ((dir = clicon_xmldb_dir(h)) == NULL){
-        clixon_err(OE_XML, errno, "CLICON_XMLDB_DIR not set");
-        goto done;
-    }
-    cprintf(cb, "%s/%s_db", dir, db);
-    if ((*filename = strdup4(cbuf_get(cb))) == NULL){
-        clixon_err(OE_UNIX, errno, "strdup");
-        goto done;
-    }
-    retval = 0;
- done:
-    if (cb)
-        cbuf_free(cb);
-    return retval;
 }
 
 /*! Translate from symbolic database name to sub-directory of configure sub-files, no checks
@@ -253,6 +278,7 @@ xmldb_disconnect(clixon_handle h)
     return retval;
 }
 
+
 /*! Copy datastore from db1 to db2
  *
  * May include copying datastore directory structure
@@ -337,8 +363,6 @@ xmldb_copy(clixon_handle h,
     if (clicon_file_copy(fromfile, tofile) < 0)
         goto done;
     if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")) {
-
-
         if (xmldb_db2subdir(h, from, &fromdir) < 0)
             goto done;
         if (xmldb_db2subdir(h, to, &todir) < 0)
@@ -627,13 +651,13 @@ int
 xmldb_create(clixon_handle h,
              const char   *db)
 {
-    int                 retval = -1;
+    int         retval = -1;
-    char               *filename = NULL;
+    char       *filename = NULL;
-    int                 fd = -1;
+    int         fd = -1;
-    db_elmnt           *de = NULL;
+    db_elmnt   *de = NULL;
-    cxobj              *xt = NULL;
+    cxobj      *xt = NULL;
-    char        *subdir = NULL;
+    char       *subdir = NULL;
-    struct stat  st = {0,};
+    struct stat st = {0,};
 
     clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "%s", db);
     if ((de = clicon_db_elmnt_get(h, db)) != NULL){
@@ -642,12 +666,6 @@ xmldb_create(clixon_handle h,
             de->de_xml = NULL;
         }
     }
-    if (xmldb_db2file(h, db, &filename) < 0)
-        goto done;
-    if ((fd = open(filename, O_CREAT|O_WRONLY, S_IRWXU)) == -1) {
-        clixon_err(OE_UNIX, errno, "open(%s)", filename);
-        goto done;
-    }
     if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){
         if (xmldb_db2subdir(h, db, &subdir) < 0)
             goto done;
@@ -658,6 +676,12 @@ xmldb_create(clixon_handle h,
             }
         }
     }
+    if (xmldb_db2file(h, db, &filename) < 0)
+        goto done;
+    if ((fd = open(filename, O_CREAT|O_WRONLY, S_IRWXU)) == -1) {
+        clixon_err(OE_UNIX, errno, "open(%s)", filename);
+        goto done;
+    }
     retval = 0;
  done:
     clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval);
@@ -958,3 +982,43 @@ xmldb_populate(clixon_handle h,
  done:
     return retval;
 }
+
+/*! Upgrade datastore from original non-multi to multi/split mode
+ *
+ * This is for upgrading the datastores on startup using CLICON_XMLDB_MULTI
+ * (1) If <db>.d/0.xml does not exist AND
+ * (2) <db>_db does exist and is a regular file
+ * (3) THEN copy file from <db>_db to <db>.d/0.xml
+ * @param[in]  h   Clixon handle
+ * @param[in]  db  Datastore
+ */
+int
+xmldb_multi_upgrade(clixon_handle h,
+                    const char   *db)
+{
+    int         retval = -1;
+    char       *fromfile = NULL;
+    char       *tofile = NULL;
+    struct stat st = {0,};
+
+    if (xmldb_db2file1(h, db, 1, &tofile) < 0)
+        goto done;
+    if (stat(tofile, &st) < 0 && errno == ENOENT) {
+        /* db.d/0.xml does not exist */
+        if (xmldb_create(h, db) < 0)
+            goto done;
+        if (xmldb_db2file1(h, db, 0, &fromfile) < 0)
+            goto done;
+        if (stat(fromfile, &st) == 0 && S_ISREG(st.st_mode)){
+            if (clicon_file_copy(fromfile, tofile) < 0)
+                goto done;
+        }
+    }
+    retval = 0;
+ done:
+    if (fromfile)
+        free(fromfile);
+    if (tofile)
+        free(tofile);
+    return retval;
+}

test/test_datastore_multi.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Datastore split test, eg x_db has x.d/ directory with subdirs# ALso test cache bevahour, that unmodified
+# Datastore split test, eg x_db has x.d/ directory with subdirs
+# Also test cache bevahour, that unmodified
 # subteres are not touched
 # For now subdirs only enabled for mointpoints, so this test is with mountpoints as well
 
@@ -152,7 +153,7 @@ function check_db()
     dbname=$1
     subfile=$2
 
-    sudo chmod o+r $dir/${dbname}_db
+    sudo chmod o+r $dir/${dbname}.d/0.xml
     sudo chmod o+r $dir/${dbname}.d/$subfile
 
     sudo rm -f $dir/x_db
@@ -167,9 +168,11 @@ function check_db()
 </config>
 EOF
     new "Check ${dbname}_db"
-    ret=$(diff $dir/x_db $dir/${dbname}_db)
+    #    ret=$(diff $dir/x_db $dir/${dbname}_db)
+    ret=$(diff $dir/x_db $dir/${dbname}.d/0.xml)
     if [ $? -ne 0 ]; then
-        err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
+        #        err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
+        err "$(cat $dir/x_db)" "$(cat $dir/${dbname}.d/0.xml)"
     fi
     cat <<EOF > $dir/x_subfile
 <mount1 xmlns="urn:example:mount1">
@@ -343,6 +346,9 @@ if [ $BE -ne 0 ]; then
     if [ $? -ne 0 ]; then
         err
     fi
+fi
+
+if [ $BE -ne 0 ]; then
     new "start backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1"
     start_backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1
 fi
@@ -361,6 +367,30 @@ if [ $BE -ne 0 ]; then
     stop_backend -f $cfg
 fi
 
+# move running.d/0.xml to running_db to trigger upgrade
+sudo mv $dir/running.d/0.xml $dir/running_db
+
+new "Check backward compatible: if running.0/0.xml is not found read running_db on startup"
+if [ $BE -ne 0 ]; then
+    new "start backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1"
+    start_backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1
+fi
+
+new "Check running after restart"
+check_db running ${subfilename}
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u root -f clixon_backend)
+    if [ -z "$pid" ]; then
+        err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+
 sudo rm -rf $dir
 
 unset dbname