Merge branch 'master' into fix-editorconfig

.editorconfig

@@ -3,3 +3,4 @@ root = true
 [*.{c,h}]
 indent_size = 4
 indent_style = space
+

CHANGELOG.md

@@ -40,6 +40,19 @@
 ## 6.0.0
 Expected: End of 2022
 
+### New features
+
+* Confirmed-commit capability
+  * Standards
+    * RFC 4741 "NETCONF Configuration Protocol": Section 8.4
+    * RFC 6241 "Network Configuration Protocol (NETCONF)": Section 8.4
+  * Features
+    * `:confirmed-commit:1.0` capability
+    * `:confirmed-commit:1.1` capability
+  * Added support for relevant arguments to CLI commit
+    * See [example_cli.cli](https://github.com/clicon/clixon/blob/master/example/main/example_cli.cli)
+  * See [Netconf Confirmed Commit Capability](https://github.com/clicon/clixon/issues/255)
+
 ### API changes on existing protocol/config features
 
 Users may have to change how they access the system
@@ -53,9 +66,29 @@ Developers may need to change their code
 
 * C API changes
   * Added `defaults` parameter to `clicon_rpc_get_pageable_list()`
+  * `clicon_rpc_commit()` and `cli_commit`
+    * Added `confirmed`, `cancel`, `timeout`, `persist-id`, and `persist-id-val` parameters to
+  * `clicon_rpc_commit()` and `clicon_rpc_validate`
+    * Added three-value return.
+    * Code need to be changed from: checking for `<0` to `<1` to keep same semantics
+	
+### Minor features
+
+* [Code formatting: Change indentation style to space](https://github.com/clicon/clixon/issues/379)
+  * Applies to all c/h/y/l/sh files and .editorconfig
+* Added warning if modstate is not present in datastore if `CLICON_XMLDB_MODSTATE` is set.
 
 ### Corrected Bugs
 
+* Fixed: [Non-obvious behavior of clixon_snmp after snmpset command when transaction validation returns an error](https://github.com/clicon/clixon/issues/375)
+  * Fixed by validating writes on ACTION instead of COMMIT since libnetsnmp seems not to accept commit errors
+* Fixed: [YANG when condition evaluated as false combined with a mandatory leaf does not work](https://github.com/clicon/clixon/issues/380)
+* Fixed: [Trying to change the "read-only" node through snmpset](https://github.com/clicon/clixon/issues/376)
+* Fixed: [Trying to change the "config false" node through snmpset](https://github.com/clicon/clixon/issues/377)
+  * Fixed by returning `SNMP_ERR_NOTWRITABLE` when trying to reserve object
+* Fixed: [Non-obvious behavior of clixon_snmp after snmpset command when transaction validation returns an error](https://github.com/clicon/clixon/issues/375)
+* Fixed: [clixon_snmp module crashes on snmpwalk command](https://github.com/clicon/clixon/issues/378)
+* Fixed: [unneeded trailing zero character on SNMP strings](https://github.com/clicon/clixon/issues/367)
 * Fixed: [message-id present on netconf app "hello"](https://github.com/clicon/clixon/issues/369)
 * Fixed: [SNMP "smiv2" yang extension doesn't work on augmented nodes](https://github.com/clicon/clixon/issues/366)
 

apps/backend/Makefile.in

@@ -100,6 +100,7 @@ APPOBJ  = $(APPSRC:.c=.o)
 LIBSRC	= clixon_backend_transaction.c
 LIBSRC += clixon_backend_handle.c
 LIBSRC += backend_commit.c
+LIBSRC += backend_confirm.c
 LIBSRC += backend_plugin.c
 LIBOBJ	= $(LIBSRC:.c=.o)
 

apps/backend/backend_client.c

@@ -159,6 +159,8 @@ release_all_dbs(clicon_handle h,
  * Finally actually remove client struct in handle
  * @param[in]  h   Clicon handle
  * @param[in]  ce  Client handle
+ * @retval     -1  Error (fatal)
+ * @retval      0  Ok
  * @see backend_client_delete for actual deallocation of client entry struct
  */
 int
@@ -168,6 +170,33 @@ backend_client_rm(clicon_handle        h,
     struct client_entry  *c;
     struct client_entry  *c0;
     struct client_entry **ce_prev;
+    uint32_t              myid = ce->ce_id;
+    yang_stmt            *yspec;
+    int                   retval = -1;
+
+    /* If the confirmed-commit feature is enabled, rollback any ephemeral commit originated by this client */
+    if ((yspec = clicon_dbspec_yang(h)) == NULL) {
+        clicon_err(OE_YANG, ENOENT, "No yang spec");
+        goto done;
+    }
+
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit")) {
+        if (confirmed_commit_state_get(h) == EPHEMERAL) {
+            /* See if this client is the origin */
+            clicon_debug(1, "session_id: %u, confirmed_commit.session_id: %u", ce->ce_id, confirmed_commit_session_id_get(h));
+
+            if (myid == confirmed_commit_session_id_get(h)) {
+                clicon_debug(1, "ok, rolling back");
+                clicon_log(LOG_NOTICE, "a client with an active ephemeral confirmed-commit has disconnected; rolling back");
+
+                /* do_rollback errors are logged internally and there is no client to report errors to, so errors are
+                 * ignored here.
+                 */
+                cancel_rollback_event(h);
+                do_rollback(h, NULL);
+            }
+        }
+    }
 
     clicon_debug(1, "%s", __FUNCTION__);
     /* for all streams: XXX better to do it top-level? */
@@ -187,7 +216,10 @@ backend_client_rm(clicon_handle        h,
         }
         ce_prev = &c->ce_next;
     }
-    return backend_client_delete(h, ce); /* actually purge it */
+    retval = backend_client_delete(h, ce); /* actually purge it */
+
+    done:
+        return retval;
 }
 
 /*! Get clixon per datastore stats
@@ -423,7 +455,35 @@ from_client_edit_config(clicon_handle h,
         autocommit = 1;
     /* If autocommit option is set or requested by client */
     if (clicon_autocommit(h) || autocommit) {
-	if ((ret = candidate_commit(h, "candidate", cbret)) < 0){ /* Assume validation fail, nofatal */
+        /* if this is from a restconf client ...
+         *      and, if there is an existing ephemeral commit, set is_valid_confirming_commit=1 such that
+         *          candidate_commit will apply the configuration per RFC 8040 1.4:
+         *              If a confirmed commit procedure is
+         *              in progress by any NETCONF client, then any new commit will act as
+         *              the confirming commit.
+         *      and, if there is an existing persistent commit, netconf_operation_failed with "in-use", so
+         *          that the restconf server will return "409 Conflict" per RFC 8040 1.4:
+         *              If the NETCONF server is expecting a
+         *              "persist-id" parameter to complete the confirmed commit procedure,
+         *              then the RESTCONF edit operation MUST fail with a "409 Conflict"
+         *              status-line.  The error-tag "in-use" is used in this case.
+         */
+        if (if_feature(yspec, "ietf-netconf", "confirmed-commit")) {
+            switch (confirmed_commit_state_get(h)){
+            case INACTIVE:
+                break;
+            case PERSISTENT:
+                if (netconf_in_use(cbret, "application", "Persistent commit is ongoing")< 0)
+                    goto done;
+                goto ok;
+                break;
+            case EPHEMERAL:
+            case ROLLBACK:
+                cancel_confirmed_commit(h);
+                break;
+            }
+        }
+        if ((ret = candidate_commit(h, NULL, "candidate", cbret)) < 0){ /* Assume validation fail, nofatal */
             if (netconf_operation_failed(cbret, "application", clicon_err_reason)< 0)
                 goto done;
             xmldb_copy(h, "running", "candidate");
@@ -656,9 +716,15 @@ from_client_lock(clicon_handle h,
     struct client_entry *ce = (struct client_entry *)arg;
     uint32_t             id = ce->ce_id;
     uint32_t             iddb;
+    uint32_t             otherid;
     char                *db;
     cbuf                *cbx = NULL; /* Assist cbuf */
+    yang_stmt           *yspec;
 
+    if ((yspec =  clicon_dbspec_yang(h)) == NULL){
+        clicon_err(OE_YANG, ENOENT, "No yang spec9");
+        goto done;
+    }
     if ((db = netconf_db_find(xe, "target")) == NULL){
         if (netconf_missing_element(cbret, "protocol", "target", NULL) < 0)
             goto done;
@@ -694,6 +760,20 @@ from_client_lock(clicon_handle h,
             goto done;
         goto ok;
     }
+    /* 3) The target configuration is <running>, and another NETCONF
+     *    session has an ongoing confirmed commi
+     */
+    if (strcmp(db, "running") == 0 &&
+        if_feature(yspec, "ietf-netconf", "confirmed-commit") &&
+        confirmed_commit_state_get(h) != INACTIVE){
+        if ((otherid = confirmed_commit_session_id_get(h)) != 0){
+            cprintf(cbx, "<session-id>%u</session-id>", otherid);
+            if (netconf_lock_denied(cbret, cbuf_get(cbx),
+                                    "Operation failed, another session has an ongoing confirmed commit") < 0)
+                goto done;
+            goto ok;
+        }
+    }
     if (xmldb_lock(h, db, id) < 0)
         goto done;
      /* user callback */

apps/backend/backend_client.h

@@ -51,7 +51,7 @@ struct client_entry{
     int                   ce_nr;      /* Client number (for dbg/tracing) */
     int                   ce_stat_in; /* Nr of received msgs from client */
     int                   ce_stat_out;/* Nr of sent msgs to client */
-    int                   ce_id;      /* Session id */
+    uint32_t              ce_id;      /* Session id, accessor functions: clicon_session_id_get/set */
     char                 *ce_username;/* Translated from peer user cred */
     clicon_handle         ce_handle;  /* clicon config handle (all clients have same?) */
 };

apps/backend/backend_commit.c

@@ -32,7 +32,7 @@
   the terms of any one of the Apache License version 2 or the GPL.
 
   ***** END LICENSE BLOCK *****
-
+  Commit and validate
  */
 
 #ifdef HAVE_CONFIG_H
@@ -200,6 +200,11 @@ startup_common(clicon_handle       h,
         if (xmldb_get0(h, db, YB_NONE, NULL, "/", 0, &xt, msdiff, &xerr) < 0)
             goto done;
     }
+    if (msdiff && msdiff->md_status == 0){ // Possibly check for CLICON_XMLDB_MODSTATE
+        clicon_log(LOG_WARNING, "Modstate expected in startup datastore but not found\n"
+                   "This may indicate that the datastore is not initialized corrrectly, such as copy/pasted.\n"
+                   "It may also be normal bootstrapping since module state will be written on next datastore save");
+    }
     if ((yspec = clicon_dbspec_yang(h)) == NULL){
         clicon_err(OE_YANG, 0, "Yang spec not set");
         goto done;
@@ -629,19 +634,24 @@ candidate_validate(clicon_handle h,
     goto done;
 }
 
+
 /*! Do a diff between candidate and running, then start a commit transaction
  *
  * The code reverts changes if the commit fails. But if the revert
  * fails, we just ignore the errors and proceed. Maybe we should
  * do something more drastic?
  * @param[in]  h          Clicon handle
+ * @param[in]  xe         Request: <rpc><xn></rpc>  (or NULL)
+ * @param[in]  session_id Client session id, only if xe
  * @param[in]  db         A candidate database, not necessarily "candidate"
+ * @param[out] cbret      Return xml tree, eg <rpc-reply>..., <rpc-error.. 
  * @retval    -1          Error - or validation failed 
  * @retval     0          Validation failed (with cbret set)
  * @retval     1          Validation OK       
  */
 int
 candidate_commit(clicon_handle h, 
+                 cxobj        *xe,
                  char         *db,
                  cbuf         *cbret)
 {
@@ -649,6 +659,7 @@ candidate_commit(clicon_handle h,
     transaction_data_t *td = NULL;
     int                 ret;
     cxobj              *xret = NULL;
+    yang_stmt          *yspec;
 
      /* 1. Start transaction */
     if ((td = transaction_new()) == NULL)
@@ -659,6 +670,28 @@ candidate_commit(clicon_handle h,
      */
     if ((ret = validate_common(h, db, td, &xret)) < 0)
         goto done;
+
+    /* If the confirmed-commit feature is enabled, execute phase 2:
+     *  - If a valid confirming-commit, cancel the rollback event
+     *  - If a new confirmed-commit, schedule a new rollback event, otherwise
+     *  - delete the rollback database
+     *
+     * Unless, however, this invocation of candidate_commit() was by way of a
+     * rollback event, in which case the timers are already cancelled and the
+     * caller will cleanup the rollback database.  All that must be done here is
+     * to activate it.
+     */
+    if ((yspec = clicon_dbspec_yang(h)) == NULL) {
+        clicon_err(OE_YANG, ENOENT, "No yang spec");
+        goto done;
+    }
+
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit")
+        && confirmed_commit_state_get(h) != ROLLBACK
+        && xe != NULL){
+        if (handle_confirmed_commit(h, xe) < 0)
+            goto done;
+    }
     if (ret == 0){
         if (clixon_xml2cbuf(cbret, xret, 0, 0, -1, 0) < 0)
             goto done;
@@ -728,6 +761,19 @@ candidate_commit(clicon_handle h,
  *  configuration datastore that are actually different and only check
  *  "create", "update", and "delete" access permissions for this set of
  *  nodes, which could be empty.
+ *
+ * Handling of the first phase of confirmed-commit:
+ * First, it must be determined if the given <commit> RPC constitutes a "confirming-commit", roughly meaning:
+ *   1) it was issued in the same session as a prior confirmed-commit
+ *   2) it bears a <persist-id> element matching the <persist> element that accompanied the prior confirmed-commit
+ *
+ *   If it is a valid "confirming-commit" and this RPC does not bear another <confirmed/> element, then the
+ *   confirmed-commit is complete, the rollback event can be cancelled and the rollback database deleted.
+ *
+ *   No further action is necessary as the candidate configuration was already copied to the running configuration.
+ *
+ *   If the RPC does bear another <confirmed/> element, that will be handled in phase two, from within the
+ *   candidate_commit() method.
  */
 int
 from_client_commit(clicon_handle h,
@@ -742,6 +788,19 @@ from_client_commit(clicon_handle h,
     uint32_t             iddb;
     cbuf                *cbx = NULL; /* Assist cbuf */
     int                  ret;
+    yang_stmt           *yspec;
+
+    if ((yspec = clicon_dbspec_yang(h)) == NULL) {
+        clicon_err(OE_YANG, ENOENT, "No yang spec");
+        goto done;
+    }
+
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit")) {
+        if ((ret = from_client_confirmed_commit(h, xe, myid, cbret)) < 0)
+            goto done;
+        if (ret == 0)
+            goto ok;
+    }
 
     /* Check if target locked by other client */
     iddb = xmldb_islocked(h, "running");
@@ -750,12 +809,11 @@ from_client_commit(clicon_handle h,
             clicon_err(OE_XML, errno, "cbuf_new");
             goto done;
         }       
-	cprintf(cbx, "<session-id>%u</session-id>", iddb);
+        if (netconf_in_use(cbret, "protocol", "Operation failed, lock is already held") < 0)
-	if (netconf_in_use(cbret, cbuf_get(cbx), "Operation failed, lock is already held") < 0)
             goto done;
         goto ok;
     }
-    if ((ret = candidate_commit(h, "candidate", cbret)) < 0){ /* Assume validation fail, nofatal */
+    if ((ret = candidate_commit(h, xe, "candidate", cbret)) < 0){ /* Assume validation fail, nofatal */
         clicon_debug(1, "Commit candidate failed");
         if (ret < 0)
             if (netconf_operation_failed(cbret, "application", clicon_err_reason)< 0)
@@ -824,33 +882,6 @@ from_client_discard_changes(clicon_handle h,
     return retval; /* may be zero if we ignoring errors from commit */
 }
 
-/*! Cancel an ongoing confirmed commit.
- * If the confirmed commit is persistent, the parameter 'persist-id' must be
- * given, and it must match the value of the 'persist' parameter.
- *
- * @param[in]  h       Clicon handle 
- * @param[in]  xe      Request: <rpc><xn></rpc> 
- * @param[out] cbret   Return xml tree, eg <rpc-reply>..., <rpc-error.. 
- * @param[in]  arg     client-entry
- * @param[in]  regarg  User argument given at rpc_callback_register() 
- * @retval  0  OK. This may indicate both ok and err msg back to client
- * @retval     0       OK
- * @retval    -1       Error
- * @see RFC 6241 Sec 8.4
- */
-int
-from_client_cancel_commit(clicon_handle h,
-			  cxobj        *xe,
-			  cbuf         *cbret,
-			  void         *arg,
-			  void         *regarg)
-{
-    int retval = -1;
-    retval = 0;
-    // done:
-    return retval;
-}
-
 /*! Validates the contents of the specified configuration.
  * @param[in]  h       Clicon handle 
  * @param[in]  xe      Request: <rpc><xn></rpc> 
@@ -1011,3 +1042,58 @@ from_client_restart_one(clicon_handle h,
     goto done;
 }
 
+/*! Reset running and start in failsafe mode. If no failsafe then quit.
+ * 
+ * param[in] h     Clixon handle
+ * param[in] phase Debug string
+  Typically done when startup status is not OK so
+
+failsafe      ----------------------+
+                            reset    \ commit
+running                   ----|-------+---------------> RUNNING FAILSAFE
+                           \
+tmp                         |---------------------->
+ */
+int
+load_failsafe(clicon_handle h,
+              char         *phase)
+{
+    int   retval = -1;
+    int   ret;
+    char *db = "failsafe";
+    cbuf *cbret = NULL;
+
+    phase = phase == NULL ? "(unknown)" : phase;
+
+    if ((cbret = cbuf_new()) == NULL){
+        clicon_err(OE_XML, errno, "cbuf_new");
+        goto done;
+    }
+    if ((ret = xmldb_exists(h, db)) < 0)
+        goto done;
+    if (ret == 0){ /* No it does not exist, fail */
+        clicon_err(OE_DB, 0, "%s failed and no Failsafe database found, exiting", phase);
+        goto done;
+    }
+    /* Copy original running to tmp as backup (restore if error) */
+    if (xmldb_copy(h, "running", "tmp") < 0)
+        goto done;
+    if (xmldb_db_reset(h, "running") < 0)
+        goto done;
+    ret = candidate_commit(h, NULL, db, cbret);
+    if (ret != 1)
+        if (xmldb_copy(h, "tmp", "running") < 0)
+            goto done;
+    if (ret < 0)
+        goto done;
+    if (ret == 0){
+        clicon_err(OE_DB, 0, "%s failed, Failsafe database validation failed %s", phase, cbuf_get(cbret));
+        goto done;
+    }
+    clicon_log(LOG_NOTICE, "%s failed, Failsafe database loaded ", phase);
+    retval = 0;
+    done:
+    if (cbret)
+        cbuf_free(cbret);
+    return retval;
+}

apps/backend/backend_confirm.c

@@ -0,0 +1,809 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
+  Copyright (C) 2017-2019 Olof Hagsand
+  Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+  Commit-confirm
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <time.h>
+#include <pwd.h>
+#include <syslog.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+
+/* cligen */
+#include <cligen/cligen.h>
+
+/* clicon */
+#include <clixon/clixon.h>
+
+#include "clixon_backend_transaction.h"
+#include "clixon_backend_plugin.h"
+#include "backend_handle.h"
+#include "clixon_backend_commit.h"
+#include "backend_client.h"
+
+/* 
+ * Local types 
+ */
+/* A struct to store the information necessary for tracking the status and relevant details of
+ * one or more overlapping confirmed-commit events.
+ */
+struct confirmed_commit {
+    enum confirmed_commit_state cc_state;
+    char       *cc_persist_id;       /* a value given by a client in the confirmed-commit */
+    uint32_t    cc_session_id;       /* the session_id of the client that gave no <persist> value */
+    int        (*cc_fn)(int, void*); /* function pointer for rollback event (rollback_fn()) */
+    void        *cc_arg;             /* clicon_handle that will be passed to rollback_fn() */
+};
+
+int
+confirmed_commit_init(clicon_handle h)
+{
+    int                      retval = -1;
+    struct confirmed_commit *cc = NULL;
+
+    if ((cc = calloc(1, sizeof(*cc))) == NULL){
+        clicon_err(OE_UNIX, errno, "calloc");
+        goto done;
+    }
+    cc->cc_state = INACTIVE;
+    if (clicon_ptr_set(h, "confirmed-commit-struct", cc) < 0)
+        goto done;
+    retval = 0;
+ done:
+    return retval;
+}
+
+/*! If confirm commit persist-id exists, free it
+ * @param[in] h  Clixon handle
+ * @retval    0  OK
+ */
+int
+confirmed_commit_free(clicon_handle h)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    if (cc != NULL){
+        if (cc->cc_persist_id != NULL)
+            free (cc->cc_persist_id);
+        free(cc);
+    }
+    clicon_ptr_del(h, "confirmed-commit-struct");
+    return 0;
+}
+
+/*
+ * Accessor functions
+ */
+enum confirmed_commit_state
+confirmed_commit_state_get(clicon_handle h)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    return cc->cc_state;
+}
+
+static int
+confirmed_commit_state_set(clicon_handle h,
+                           enum confirmed_commit_state state)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    cc->cc_state = state;
+    return 0;
+}
+
+char *
+confirmed_commit_persist_id_get(clicon_handle h)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    return cc->cc_persist_id;
+}
+
+static int
+confirmed_commit_persist_id_set(clicon_handle h,
+                                char         *persist_id)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    if (cc->cc_persist_id)
+        free(cc->cc_persist_id);
+    if (persist_id){
+        if ((cc->cc_persist_id = strdup4(persist_id)) == NULL){
+            clicon_err(OE_UNIX, errno, "strdup4");
+            return -1;
+        }
+    }
+    else
+        cc->cc_persist_id = NULL;
+    return 0;
+}
+
+uint32_t
+confirmed_commit_session_id_get(clicon_handle h)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    return cc->cc_session_id;
+}
+
+static int
+confirmed_commit_session_id_set(clicon_handle h,
+                                uint32_t      session_id)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    cc->cc_session_id = session_id;
+    return 0;
+}
+
+static int
+confirmed_commit_fn_arg_get(clicon_handle h,
+                            int        (**fn)(int, void*),
+                            void        **arg)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    *fn = cc->cc_fn;
+    *arg = cc->cc_arg;
+    return 0;
+}
+
+static int
+confirmed_commit_fn_arg_set(clicon_handle h,
+                            int        (*fn)(int, void*),
+                            void        *arg)
+{
+    struct confirmed_commit *cc = NULL;
+    
+    clicon_ptr_get(h, "confirmed-commit-struct", (void**)&cc);
+    cc->cc_fn = fn;
+    cc->cc_arg = arg;
+    return 0;
+}
+
+/*! Return if confirmed tag found
+ * @param[in]  xe  Commit rpc xml
+ * @retval     1   Confirmed tag exists
+ * @retval     0   Confirmed tag does not exist
+ */
+static int
+xe_confirmed(cxobj *xe)
+{
+    return (xml_find_type(xe, NULL, "confirmed", CX_ELMNT) != NULL) ? 1 : 0;
+}
+
+/*! Return if persist exists and its string value field
+ * @param[in]  xe  Commit rpc xml
+ * @param[out] str Pointer to persist
+ * @retval     1   Persist field exists
+ * @retval     0   Persist field  does not exist
+ */
+static int
+xe_persist(cxobj *xe,
+           char **str)
+{
+    cxobj *xml;
+
+    if ((xml = xml_find_type(xe, NULL, "persist", CX_ELMNT)) != NULL){
+        *str = xml_body(xml);
+        return 1;
+    }
+    *str = NULL;
+    return 0;
+}
+
+/*! Return if persist-id exists and its string value
+ *
+ * @param[in]  xe  Commit rpc xml
+ * @param[out] str Pointer to persist-id
+ * @retval     1   Persist-id exists
+ * @retval     0   Persist-id does not exist
+ */
+static int
+xe_persist_id(cxobj *xe,
+              char **str)
+{
+    cxobj *xml;
+
+    if ((xml = xml_find_type(xe, NULL, "persist-id", CX_ELMNT)) != NULL){
+        *str = xml_body(xml);
+        return 1;
+    }
+    *str = NULL;
+    return 0;
+}
+
+/*! Return timeout
+ * @param[in]  xe  Commit rpc xml
+ * @retval     sec Timeout in seconds, can be 0 if no timeout exists or is zero
+ */
+static unsigned int
+xe_timeout(cxobj *xe)
+{
+    cxobj *xml;
+    char  *str;
+    
+    if ((xml = xml_find_type(xe, NULL, "confirm-timeout", CX_ELMNT)) != NULL &&
+        (str = xml_body(xml)) != NULL)
+        return strtoul(str, NULL, 10);
+    return 0;
+}
+
+/*! Cancel a scheduled rollback as previously registered by schedule_rollback_event()
+ *
+ * @param[in]   h       Clixon handle
+ * @retval      0       Rollback event successfully cancelled
+ * @retval      -1      No Rollback event was found
+ */
+int
+cancel_rollback_event(clicon_handle h)
+{
+    int   retval;
+    int (*fn)(int, void*) = NULL;
+    void *arg = NULL;
+
+    confirmed_commit_fn_arg_get(h, &fn, &arg);
+    if ((retval = clixon_event_unreg_timeout(fn, arg)) == 0) {
+        clicon_log(LOG_INFO, "a scheduled rollback event has been cancelled");
+    } else {
+        clicon_log(LOG_WARNING, "the specified scheduled rollback event was not found");
+    }
+
+    return retval;
+}
+
+/*! Apply the rollback configuration upon expiration of the confirm-timeout
+ *
+ * @param[in]   fd      a dummy argument per the event callback semantics
+ * @param[in]   arg     a void pointer to a clicon_handle
+ * @retval      0       the rollback was successful
+ * @retval      -1      the rollback failed
+ * @see                 do_rollback()
+ */
+static int
+rollback_fn(int  fd,
+            void *arg)
+{
+    clicon_handle h = arg;
+
+    clicon_log(LOG_CRIT, "a confirming-commit was not received before the confirm-timeout expired; rolling back");
+
+    return do_rollback(h, NULL);
+}
+
+/*! Schedule a rollback in case no confirming-commit is received before the confirm-timeout
+ *
+ * @param[in]   h       a clicon handle
+ * @param[in]   timeout a uint32 representing the number of seconds before the rollback event should fire
+ *
+ * @retval      0       Rollback event successfully scheduled
+ * @retval      -1      Rollback event was not scheduled
+ */
+static int
+schedule_rollback_event(clicon_handle h,
+                        uint32_t      timeout)
+{
+    int retval = -1;
+
+    // register a new scheduled event
+    struct timeval t, t1;
+    if (gettimeofday(&t, NULL) < 0) {
+        clicon_err(OE_UNIX, 0, "failed to get time of day: %s", strerror(errno));
+        goto done;
+    };
+    t1.tv_sec = timeout; t1.tv_usec = 0;
+    timeradd(&t, &t1, &t);
+
+    /* The confirmed-commit is either:
+     * - ephemeral, and the client requesting the new confirmed-commit is on the same session, OR
+     * - persistent, and the client provided the persist-id in the new confirmed-commit
+     */
+
+    /* remember the function pointer and args so the confirming-commit can cancel the rollback */
+    confirmed_commit_fn_arg_set(h, rollback_fn, h);
+    if (clixon_event_reg_timeout(t, rollback_fn, h, "rollback after timeout") < 0) {
+        /* error is logged in called function */
+        goto done;
+    };
+
+    retval = 0;
+
+    done:
+    return retval;
+}
+
+/*! Cancel a confirming commit by removing rollback, and free state
+ * @param[in]  h
+ * @param[out] cbret
+ * @retval     0      OK
+ */
+int
+cancel_confirmed_commit(clicon_handle h)
+{
+    cancel_rollback_event(h);
+
+    if (confirmed_commit_state_get(h) == PERSISTENT &&
+        confirmed_commit_persist_id_get(h) != NULL) {
+        confirmed_commit_persist_id_set(h, NULL);
+    }
+
+    confirmed_commit_state_set(h, INACTIVE);
+
+    if (xmldb_delete(h, "rollback") < 0)
+        clicon_err(OE_DB, 0, "Error deleting the rollback configuration");
+    return 0;
+}
+
+/*! Determine if the present commit RPC invocation constitutes a valid "confirming-commit".
+ *
+ * To be considered a valid confirming-commit, the <commit/> must either:
+ *   1) be presented without a <persist-id> value, and on the same session as a prior confirmed-commit that itself was
+ *      without a <persist> value, OR
+ *   2) be presented with a <persist-id> value that matches the <persist> value accompanying the prior confirmed-commit
+ *
+ * @param[in]   h       Clicon handle
+ * @param[in]   xe       Request: <rpc><xn></rpc> 
+ * @param[in]   myid    current client session-id
+ * @retval      1       The confirming-commit is valid
+ * @retval      0       The confirming-commit is not valid
+ * @retval      -1      Error
+ */
+static int
+check_valid_confirming_commit(clicon_handle h,
+                              cxobj        *xe,
+                              uint32_t      myid)
+{
+    int retval = -1;
+    char *persist_id = NULL;
+
+    if (xe == NULL){
+        clicon_err(OE_CFG, EINVAL, "xe is NULL");
+        goto done;
+    }
+    switch (confirmed_commit_state_get(h)) {
+        case PERSISTENT:
+            if (xe_persist_id(xe, &persist_id)) {
+                if (clicon_strcmp(persist_id, confirmed_commit_persist_id_get(h)) == 0) {
+                    /* the RPC included a <persist-id> matching the prior confirming-commit's <persist> */
+                    break; // valid
+                }
+                else {
+                    clicon_log(LOG_INFO,
+                               "a persistent confirmed-commit is in progress but the client issued a "
+                               "confirming-commit with an incorrect persist-id");
+                    goto invalid;
+                }
+            } else {
+                clicon_log(LOG_INFO,
+                           "a persistent confirmed-commit is in progress but the client issued a confirming-commit"
+                           "without a persist-id");
+                goto invalid;
+            }
+        case EPHEMERAL:
+            if (myid == confirmed_commit_session_id_get(h)) {
+                /* the RPC lacked a <persist-id>, the prior confirming-commit lacked <persist>, and both were issued
+                 * on the same session.
+                 */
+                break; // valid
+            }
+            clicon_log(LOG_DEBUG, "an ephemeral confirmed-commit is in progress, but there confirming-commit was"
+                                  "not issued on the same session as the confirmed-commit");
+            goto invalid;
+        default:
+            clicon_debug(1, "commit-confirmed state !? %d", confirmed_commit_state_get(h));
+            goto invalid;
+    }
+    retval = 1; // valid
+ done:
+    return retval;
+ invalid:
+    retval = 0;
+    goto done;
+}
+
+/*! Handle the second phase of confirmed-commit processing.
+ *
+ * In the first phase, the proper action was taken in the case of a valid confirming-commit, but no subsequent
+ * confirmed-commit.
+ *
+ * In the second phase, the action taken is to handle both confirming- and confirmed-commit by creating the
+ * rollback database as required, then deleting it once the sequence is complete.
+ *
+ * @param[in]   h          Clicon handle
+ * @param[in]   xe         Commit rpc xml or NULL
+ * @retval      0          OK
+ * @retval      -1         Error
+ */
+int
+handle_confirmed_commit(clicon_handle h,
+                        cxobj        *xe)
+{
+    int           retval = -1;
+    uint32_t      session_id;
+    char         *persist;
+    unsigned long confirm_timeout = 0L;
+    int           cc_valid;
+    int           db_exists;
+
+    if (xe == NULL){
+        clicon_err(OE_CFG, EINVAL, "xe is NULL");
+        goto done;
+    }
+    if (clicon_session_id_get(h, &session_id) < 0) {
+        clicon_err(OE_DAEMON, 0,
+                   "an ephemeral confirmed-commit was issued, but the session-id could not be determined");
+        goto done;
+    };
+    /* The case of a valid confirming-commit is also handled in the first phase, but only if there is no subsequent
+     * confirmed-commit.  It is tested again here as the case of a valid confirming-commit *with* a subsequent
+     * confirmed-commit must be handled once the transaction has begun and after all the plugins' validate callbacks
+     * have been called.
+     */
+    cc_valid = check_valid_confirming_commit(h, xe, session_id);
+    if (cc_valid) {
+        if (cancel_rollback_event(h) < 0) {
+            clicon_err(OE_DAEMON, 0, "A valid confirming-commit was received, but the corresponding rollback event was not found");
+        }
+
+        if (confirmed_commit_state_get(h) == PERSISTENT &&
+            confirmed_commit_persist_id_get(h) != NULL) {
+            confirmed_commit_persist_id_set(h, NULL);
+        }
+
+        confirmed_commit_state_set(h, INACTIVE);
+    }
+
+    /* Now, determine if there is a subsequent confirmed-commit */
+    if (xe_confirmed(xe)){
+        /* There is, get it's confirm-timeout value, which will default per the yang schema if not client-specified */
+        /* Clixon also pre-validates input according to the schema, so bounds checking here is redundant */
+        confirm_timeout = xe_timeout(xe);
+        if (xe_persist(xe, &persist)){
+            if (persist == NULL) {
+                confirmed_commit_persist_id_set(h, NULL);
+            }
+            else if (confirmed_commit_persist_id_set(h, persist) < 0){
+                goto done;
+            }
+
+            /* The client has passed <persist>; the confirming-commit MUST now be accompanied by a matching
+             * <persist-id>
+             */
+            confirmed_commit_state_set(h, PERSISTENT);
+            clicon_log(LOG_INFO,
+                       "a persistent confirmed-commit has been requested with persist id of '%s' and a timeout of %lu seconds",
+                       confirmed_commit_persist_id_get(h), confirm_timeout);
+        }
+
+        else {
+            /* The client did not pass a value for <persist> and therefore any subsequent confirming-commit must be
+             * issued within the same session.
+             */
+            confirmed_commit_session_id_set(h, session_id);
+            confirmed_commit_state_set(h, EPHEMERAL);
+
+            clicon_log(LOG_INFO,
+                       "an ephemeral confirmed-commit has been requested by session-id %u and a timeout of %lu seconds",
+                       confirmed_commit_session_id_get(h),
+                       confirm_timeout);
+        }
+
+        /* The confirmed-commits and confirming-commits can overlap; the rollback database is created at the beginning
+         * of such a sequence and deleted at the end; hence its absence implies this is the first of a sequence. **
+         *
+         *
+         * |    edit
+         * |    | confirmed-commit
+         * |    | copy t=0 running to rollback
+         * |    | | edit
+         * |    | | | both
+         * |    | | | | edit
+         * |    | | | | | both
+         * |    | | | | | | confirming-commit
+         * |    | | | | | | | delete rollback
+         * +----|-|-|-|-|-|-|-|---------------
+         * t=0  1 2 3 4 5 6 7 8
+         *
+         * edit = edit of the candidate configuration
+         * both = both a confirmed-commit and confirming-commit in the same RPC
+         *
+         * As shown, the rollback database created at t=2 is comprised of the running database from t=0
+         * Thus, if there is a rollback event at t=7, the t=0 configuration will be committed.
+         *
+         *  ** the rollback database may be present at system startup if there was a crash during a confirmed-commit;
+         *     in the case the system is configured to startup from running and the rollback database is present, the
+         *     rollback database will be committed to running and then deleted.  If the system is configured to use a
+         *     startup configuration instead, any present rollback database will be deleted.
+         *
+         */
+
+        db_exists = xmldb_exists(h, "rollback");
+        if (db_exists == -1) {
+            clicon_err(OE_DAEMON, 0, "there was an error while checking existence of the rollback database");
+            goto done;
+        } else if (db_exists == 0) {
+            // db does not yet exists
+            if (xmldb_copy(h, "running", "rollback") < 0) {
+                clicon_err(OE_DAEMON, 0, "there was an error while copying the running configuration to rollback database.");
+                goto done;
+            };
+        }
+
+        if (schedule_rollback_event(h, confirm_timeout) < 0) {
+            clicon_err(OE_DAEMON, 0, "the rollback event could not be scheduled");
+            goto done;
+        };
+
+    }
+    else {
+        /* There was no subsequent confirmed-commit, meaning this is the end of the confirmed/confirming sequence;
+         * The new configuration is already committed to running and the rollback database can now be deleted
+         */
+        if (xmldb_delete(h, "rollback") < 0) {
+            clicon_err(OE_DB, 0, "Error deleting the rollback configuration");
+            goto done;
+        }
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
+/*! Do a rollback of the running configuration to the state prior to initiation of a confirmed-commit
+ *
+ * The "running" configuration prior to the first confirmed-commit was stored in another database named "rollback".
+ * Here, it is committed as if it is the candidate configuration.
+ *
+ * Execution has arrived here because do_rollback() was called by one of:
+ *  1. backend_client_rm()          (client disconnected and confirmed-commit is ephemeral)
+ *  2. from_client_cancel_commit()  (invoked either by netconf client, or CLI)
+ *  3. rollback_fn()                (invoked by expiration of the rollback event timer)
+ *
+ * @param[in]   h       Clicon handle
+ * @retval      -1      Error
+ * @retval      0       Success
+ * @see                 backend_client_rm()
+ * @see                 from_client_cancel_commit()
+ * @see                 rollback_fn()
+ */
+int
+do_rollback(clicon_handle h,
+            uint8_t      *errs)
+{
+    int     retval = -1;
+    uint8_t errstate = 0;
+    cbuf   *cbret;
+
+    if ((cbret = cbuf_new()) == NULL) {
+        clicon_err(OE_DAEMON, 0, "rollback was not performed. (cbuf_new: %s)", strerror(errno));
+        /* the rollback_db won't be deleted, so one can try recovery by:
+         *   load rollback running
+         *   restart the backend, which will try to load the rollback_db, and delete it if successful
+         *     (otherwise it will load the failsafe)
+         */
+        clicon_log(LOG_CRIT, "An error occurred during rollback and the rollback_db wasn't deleted.");
+        errstate |= ROLLBACK_NOT_APPLIED | ROLLBACK_DB_NOT_DELETED;
+        goto done;
+    }
+
+    if (confirmed_commit_state_get(h) == PERSISTENT &&
+        confirmed_commit_persist_id_get(h) != NULL) {
+        confirmed_commit_persist_id_set(h, NULL);
+    }
+    confirmed_commit_state_set(h, ROLLBACK);
+    if (candidate_commit(h, NULL, "rollback", cbret) < 0) { /* Assume validation fail, nofatal */
+        /* theoretically, this should never error, since the rollback database was previously active and therefore
+         * had itself been previously and successfully committed.
+         */
+        clicon_log(LOG_CRIT, "An error occurred committing the rollback database.");
+        errstate |= ROLLBACK_NOT_APPLIED;
+
+        /* Rename the errored rollback database */
+        if (xmldb_rename(h, "rollback", NULL, ".error") < 0) {
+            clicon_log(LOG_CRIT, "An error occurred renaming the rollback database.");
+            errstate |= ROLLBACK_DB_NOT_DELETED;
+        }
+
+        /* Attempt to load the failsafe config */
+
+        if (load_failsafe(h, "Rollback") < 0) {
+            clicon_log(LOG_CRIT, "An error occurred committing the failsafe database.  Exiting.");
+            /* Invoke our own signal handler to exit */
+            raise(SIGINT);
+
+            /* should never make it here */
+        }
+
+        errstate |= ROLLBACK_FAILSAFE_APPLIED;
+        goto done;
+    }
+    cbuf_free(cbret);
+
+    if (xmldb_delete(h, "rollback") < 0) {
+        clicon_log(LOG_WARNING, "A rollback occurred but the rollback_db wasn't deleted.");
+        errstate |= ROLLBACK_DB_NOT_DELETED;
+        goto done;
+    };
+    retval = 0;
+ done:
+    confirmed_commit_state_set(h, INACTIVE);
+    if (errs)
+        *errs = errstate;
+    return retval;
+}
+
+/*! Cancel an ongoing confirmed commit.
+ * If the confirmed commit is persistent, the parameter 'persist-id' must be
+ * given, and it must match the value of the 'persist' parameter.
+ * If the confirmed-commit is ephemeral, the 'persist-id' must not be given and both the confirmed-commit and the
+ * cancel-commit must originate from the same session.
+ *
+ * @param[in]  h       Clicon handle 
+ * @param[in]  xe      Request: <rpc><xn></rpc> 
+ * @param[out] cbret   Return xml tree, eg <rpc-reply>..., <rpc-error.. 
+ * @param[in]  arg     client-entry
+ * @param[in]  regarg  User argument given at rpc_callback_register() 
+ * @retval     0       OK. This may indicate both ok and err msg back to client
+ * @retval    -1       Error
+ * @see RFC 6241 Sec 8.4
+ */
+int
+from_client_cancel_commit(clicon_handle h,
+                          cxobj        *xe,
+                          cbuf         *cbret,
+                          void         *arg,
+                          void         *regarg)
+{
+    cxobj   *persist_id_xml;
+    char    *persist_id = NULL;
+    uint32_t cur_session_id;
+    int      retval = -1;
+    int      rollback = 0;
+
+    if ((persist_id_xml = xml_find_type(xe, NULL, "persist-id", CX_ELMNT)) != NULL) {
+        /* persist == persist_id == NULL is legal */
+        persist_id = xml_body(persist_id_xml);
+    }
+    switch(confirmed_commit_state_get(h)) {
+        case EPHEMERAL:
+            if (persist_id_xml != NULL) {
+                if (netconf_invalid_value(cbret, "protocol", "current confirmed-commit is not persistent") < 0)
+                    goto done;
+            }
+            else if (clicon_session_id_get(h, &cur_session_id) < 0) {
+                if (netconf_invalid_value(cbret, "application", "session-id was not set") < 0)
+                    goto done;
+            }
+            else if (cur_session_id != confirmed_commit_session_id_get(h)) {
+                if (netconf_invalid_value(cbret, "protocol", "confirming-commit must be given within session that gave the confirmed-commit") < 0)
+                    goto done;
+            }
+            else
+                rollback++;
+            break;
+        case PERSISTENT:
+            if (persist_id_xml == NULL) {
+                if (netconf_invalid_value(cbret, "protocol", "persist-id is required") < 0)
+                    goto done;
+            }
+            else if (clicon_strcmp(persist_id, confirmed_commit_persist_id_get(h)) != 0){
+                if (netconf_invalid_value(cbret, "application", "a confirmed-commit with the given persist-id was not found") < 0)
+                    goto done;
+            }
+            else
+                rollback++;
+            break;
+        case INACTIVE:
+            if (netconf_invalid_value(cbret, "application", "no confirmed-commit is in progress") < 0)
+                goto done;
+            break;
+        default:
+            if (netconf_invalid_value(cbret, "application", "server error") < 0)
+                goto done;
+            break;
+    }
+    /* all invalid conditions jump to done: and valid code paths jump to or fall through to here. */
+    if (rollback){
+        cancel_rollback_event(h);
+        if (do_rollback(h, NULL) < 0)
+            goto done;
+        cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
+        clicon_log(LOG_INFO, "a confirmed-commit has been cancelled by client request");
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
+/*! Incoming commit handler for confirmed commit
+ * @param[in]  h     Clicon handle
+ * @param[in]  xe    Request: <rpc><xn></rpc> 
+ * @param[in]  myid  Client-id
+ * @param[out] cbret Return xml tree
+ * @retval     1     OK
+ * @retval     0     OK, dont proceed with commit
+ * @retval    -1     Error
+ */
+int
+from_client_confirmed_commit(clicon_handle h,
+                             cxobj        *xe,
+                             uint32_t      myid,
+                             cbuf         *cbret)
+{
+    int retval = -1;
+    int cc_valid;
+
+    if ((cc_valid = check_valid_confirming_commit(h, xe, myid)) < 0)
+        goto done;
+    /* If <confirmed/> is *not* present, this will conclude the confirmed-commit, so cancel the rollback. */
+    if (!xe_confirmed(xe) && cc_valid) {
+        cancel_confirmed_commit(h);
+        cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
+        goto dontcommit;
+    }
+    retval = 1;
+ done:
+    return retval;
+ dontcommit:
+    retval = 0;
+    goto done;
+}

apps/backend/backend_main.c

@@ -124,6 +124,7 @@ backend_terminate(clicon_handle h)
         xml_free(x);
     if ((x = clicon_conf_xml(h)) != NULL)
         xml_free(x);
+    confirmed_commit_free(h);
     stream_publish_exit();
     /* Delete all plugins, RPC callbacks, and upgrade callbacks */
     clixon_plugin_module_exit(h);
@@ -279,7 +280,8 @@ xmldb_drop_priv(clicon_handle h,
  */
 static int
 check_drop_priv(clicon_handle h,
-		gid_t         gid)
+                gid_t         gid,
+                yang_stmt    *yspec)
 {
     int              retval = -1;
     uid_t            uid;
@@ -287,7 +289,6 @@ check_drop_priv(clicon_handle h,
     enum priv_mode_t priv_mode = PM_NONE;
     char            *backend_user = NULL;
 
-
     /* Get privileges mode (for dropping privileges) */
     if ((priv_mode = clicon_backend_privileges_mode(h)) == PM_NONE)
         goto ok;
@@ -324,12 +325,20 @@ check_drop_priv(clicon_handle h,
             goto done;
     if (xmldb_drop_priv(h, "candidate", newuid, gid) < 0)
         goto done;
+    if (if_feature(yspec, "ietf-netconf", "startup")) {
         if (xmldb_exists(h, "startup") != 1)
             if (xmldb_create(h, "startup") < 0)
                 goto done;
         if (xmldb_drop_priv(h, "startup", newuid, gid) < 0)
             goto done;
-
+    }
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit")) {
+        if (xmldb_exists(h, "rollback") != 1)
+            if (xmldb_create(h, "rollback") < 0)
+                goto done;
+        if (xmldb_drop_priv(h, "rollback", newuid, gid) < 0)
+            goto done;
+    }
     if (setgid(gid) == -1) {
         clicon_err(OE_DAEMON, errno, "setgid %d", gid);
         goto done;
@@ -846,7 +855,11 @@ main(int    argc,
     if (clicon_option_bool(h, "CLICON_XML_CHANGELOG"))
         if (clixon_xml_changelog_init(h) < 0)
             goto done;
-    
+    /* Init commit confirmed */
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit")) {
+        if (confirmed_commit_init(h) < 0)
+            goto done;
+    }
     /* Save modules state of the backend (server). Compare with startup XML */
     if (startup_module_state(h, yspec) < 0)
         goto done;
@@ -883,6 +896,8 @@ main(int    argc,
     }
     switch (startup_mode){
     case SM_INIT: /* Scratch running and start from empty */
+        /* Delete any rollback database, if it exists */
+        xmldb_delete(h, "rollback");
         /* [Delete and] create running db */
         if (xmldb_db_reset(h, "running") < 0)
             goto done;
@@ -956,7 +971,7 @@ main(int    argc,
     if (status != STARTUP_OK){
         if (cbuf_len(cbret))
             clicon_log(LOG_NOTICE, "%s: %u %s", __PROGRAM__, getpid(), cbuf_get(cbret));
-	if (startup_failsafe(h) < 0){
+        if (load_failsafe(h, "Startup") < 0){
             goto done;
         }
         status = STARTUP_OK;
@@ -1034,7 +1049,7 @@ main(int    argc,
         clicon_option_dump(h, dbg);
     /* Depending on configure setting, privileges may be dropped here after
      * initializations */
-    if (check_drop_priv(h, gid) < 0)
+    if (check_drop_priv(h, gid, yspec) < 0)
         goto done;
 
     /* Start session-id for clients */

apps/backend/backend_startup.c

@@ -133,7 +133,9 @@ startup_mode_startup(clicon_handle        h,
                      cbuf                *cbret)
 {
     int        retval = -1;
-    int         ret;
+    int        ret = 0;
+    int        rollback_exists;
+    yang_stmt *yspec = clicon_dbspec_yang(h);
     
     if (strcmp(db, "running")==0){
         clicon_err(OE_FATAL, 0, "Invalid startup db: %s", db);
@@ -144,10 +146,54 @@ startup_mode_startup(clicon_handle        h,
         if (xmldb_create(h, db) < 0) /* diff */
             return -1;
     }
+
+    /* When a confirming-commit is issued, the confirmed-commit timeout
+     * callback is removed and then the rollback database is deleted.
+     *
+     * The presence of a rollback database means that before the rollback
+     * database was deleted, either clixon_backend crashed or the machine
+     * rebooted.
+     */
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit")) {
+        if ((rollback_exists = xmldb_exists(h, "rollback")) < 0) {
+            clicon_err(OE_DAEMON, 0, "Error checking for the existence of the rollback database");
+            goto done;
+        }
+        if (rollback_exists == 1) {
+            ret = startup_commit(h, "rollback", cbret);
+            switch(ret) {
+                case -1:
+                case 0:
+                    /* validation failed, cbret set */
+                    if ((ret = startup_commit(h, "failsafe", cbret)) < 0)
+                        goto fail;
+
+                    /* Rename the errored rollback database so that it is not tried on a subsequent startup */
+                    xmldb_rename(h, db, NULL, ".error");
+                    goto ok;
+                case 1:
+                    /* validation ok */
+                    xmldb_delete(h, "rollback");
+                    goto ok;
+                default:
+                    /* Unexpected response */
+                    goto fail;
+            }
+        }
+        else {
             if ((ret = startup_commit(h, db, cbret)) < 0)
                 goto done;
             if (ret == 0)
                 goto fail;
+        }
+    }
+    else {
+        if ((ret = startup_commit(h, db, cbret)) < 0)
+            goto done;
+        if (ret == 0)
+            goto fail;
+    }
+ ok:
     retval = 1;
  done:
     return retval;
@@ -293,56 +339,6 @@ startup_extraxml(clicon_handle        h,
     goto done;
 }
 
-/*! Reset running and start in failsafe mode. If no failsafe then quit.
-  Typically done when startup status is not OK so
-
-failsafe      ----------------------+
-                            reset    \ commit
-running                   ----|-------+---------------> RUNNING FAILSAFE
-                           \
-tmp                         |----------------------> 
- */
-int
-startup_failsafe(clicon_handle h)
-{
-    int   retval = -1;
-    int   ret;
-    char *db = "failsafe";
-    cbuf *cbret = NULL;
-
-    if ((cbret = cbuf_new()) == NULL){
-	clicon_err(OE_XML, errno, "cbuf_new");
-	goto done;
-    }
-    if ((ret = xmldb_exists(h, db)) < 0)
-	goto done;
-    if (ret == 0){ /* No it does not exist, fail */
-	clicon_err(OE_DB, 0, "Startup failed and no Failsafe database found, exiting");
-	goto done;
-    }
-    /* Copy original running to tmp as backup (restore if error) */
-    if (xmldb_copy(h, "running", "tmp") < 0)
-	goto done;
-    if (xmldb_db_reset(h, "running") < 0)
-	goto done;
-    ret = candidate_commit(h, db, cbret);
-    if (ret != 1)
-	if (xmldb_copy(h, "tmp", "running") < 0)
-	    goto done;
-    if (ret < 0)
-	goto done;
-    if (ret == 0){
-	clicon_err(OE_DB, 0, "Startup failed, Failsafe database validation failed %s", cbuf_get(cbret));
-	goto done;
-    }
-    clicon_log(LOG_NOTICE, "Startup failed, Failsafe database loaded ");
-    retval = 0;
- done:
-    if (cbret)
-	cbuf_free(cbret);
-    return retval;
-}
-
 /*! Init modules state of the backend (server). To compare with startup XML
  * Set the modules state as setopt to the datastore module.
  * Only if CLICON_XMLDB_MODSTATE is enabled

apps/backend/backend_startup.h

@@ -44,7 +44,6 @@
  */ 
 int startup_mode_startup(clicon_handle h, char *db, cbuf *cbret);
 int startup_extraxml(clicon_handle h, char *file, cbuf *cbret);
-int startup_failsafe(clicon_handle h);
 int startup_module_state(clicon_handle h, yang_stmt *yspec);
 
 #endif  /* _BACKEND_STARTUP_H_ */

apps/backend/clixon_backend_commit.h

@@ -39,18 +39,44 @@
 #ifndef _CLIXON_BACKEND_COMMIT_H_
 #define _CLIXON_BACKEND_COMMIT_H_
 
+#define ROLLBACK_NOT_APPLIED        1
+#define ROLLBACK_DB_NOT_DELETED     2
+#define ROLLBACK_FAILSAFE_APPLIED   4
+
+#define COMMIT_NOT_CONFIRMED "Commit was not confirmed; automatic rollback complete."
+
+enum confirmed_commit_state {
+    INACTIVE,       // a confirmed-commit is not in progress
+    PERSISTENT,     // a confirmed-commit is in progress and a persist value was given
+    EPHEMERAL,      // a confirmed-commit is in progress and a persist value was not given
+    ROLLBACK
+};
+
 /*
  * Prototypes
  */
+/* backend_confirm.c */
+int confirmed_commit_init(clicon_handle h);
+int confirmed_commit_free(clicon_handle h);
+enum confirmed_commit_state confirmed_commit_state_get(clicon_handle h);
+uint32_t confirmed_commit_session_id_get(clicon_handle h);
+int cancel_rollback_event(clicon_handle h);
+int cancel_confirmed_commit(clicon_handle h);
+int handle_confirmed_commit(clicon_handle h, cxobj *xe);
+int do_rollback(clicon_handle h, uint8_t *errs);
+int from_client_cancel_commit(clicon_handle h,  cxobj *xe, cbuf *cbret, void *arg, void *regarg);
+int from_client_confirmed_commit(clicon_handle h, cxobj *xe, uint32_t myid, cbuf *cbret);
+
+/* backend_commit.c */
 int startup_validate(clicon_handle h, char *db, cxobj **xtr, cbuf *cbret);
 int startup_commit(clicon_handle h, char *db, cbuf *cbret);
 int candidate_validate(clicon_handle h, char *db, cbuf *cbret);
-int candidate_commit(clicon_handle h, char *db, cbuf *cbret);
+int candidate_commit(clicon_handle h, cxobj *xe, char *db, cbuf *cbret);
 
 int from_client_commit(clicon_handle h, cxobj *xe, cbuf *cbret, void *arg, void *regarg);
 int from_client_discard_changes(clicon_handle h, cxobj *xe, cbuf *cbret, void *arg, void *regarg);
-int from_client_cancel_commit(clicon_handle h,	cxobj *xe, cbuf *cbret, void *arg, void *regarg);
 int from_client_validate(clicon_handle h, cxobj *xe, cbuf *cbret, void *arg, void *regarg);
 int from_client_restart_one(clicon_handle h, clixon_plugin_t *cp, cbuf *cbret);
+int load_failsafe(clicon_handle h, char *phase);
 
 #endif  /* _CLIXON_BACKEND_COMMIT_H_ */

apps/cli/cli_common.c

@@ -648,8 +648,23 @@ cli_commit(clicon_handle h,
             cvec         *argv)
 {
     int            retval = -1;
+    uint32_t       timeout = 0; /* any non-zero value means "confirmed-commit" */
+    cg_var        *timeout_var;
+    char          *persist = NULL;
+    char          *persist_id = NULL;
 
-    if ((retval = clicon_rpc_commit(h)) < 0)
+    int confirmed = (cvec_find_str(vars, "confirmed") != NULL);
+    int cancel = (cvec_find_str(vars, "cancel") != NULL);
+
+    if ((timeout_var = cvec_find(vars, "timeout")) != NULL) {
+        timeout = cv_uint32_get(timeout_var);
+        clicon_debug(1, "commit confirmed with timeout %ul", timeout);
+    }
+
+    persist = cvec_find_str(vars, "persist-val");
+    persist_id = cvec_find_str(vars, "persist-id-val");
+    
+    if (clicon_rpc_commit(h, confirmed, cancel, timeout, persist, persist_id) < 1)
         goto done;
     retval = 0;
   done:
@@ -665,7 +680,7 @@ cli_validate(clicon_handle h,
 {
     int     retval = -1;
 
-    if ((retval = clicon_rpc_validate(h, "candidate")) < 0)
+    if (clicon_rpc_validate(h, "candidate") < 1)
         goto done;
     retval = 0;
  done:

apps/snmp/snmp_handler.c

@@ -44,7 +44,6 @@
 #include <errno.h>
 #include <sys/types.h>
 #include <signal.h>
-#include <assert.h>
 
 /* net-snmp */
 #include <net-snmp/net-snmp-config.h>
@@ -146,13 +145,14 @@ snmp_scalar_return(cxobj                      *xs,
     char                  *reason = NULL;
     netsnmp_variable_list *requestvb = request->requestvb;
     int                    ret;
+    char                  *body = NULL;
 
     /* SMI default value, How is this different from yang defaults?
      */
     if (yang_extension_value(ys, "defval", IETF_YANG_SMIV2_NS, NULL, &defaultval) < 0)
         goto done;
-    if (xs != NULL){
+    if (xs != NULL && (body = xml_body(xs)) != NULL){
-	if ((ret = type_xml2snmp_pre(xml_body(xs), ys, &xmlstr)) < 0)
+        if ((ret = type_xml2snmp_pre(body, ys, &xmlstr)) < 0) // XXX <---
             goto done;
         if (ret == 0){
             if ((ret = netsnmp_request_set_error(request, SNMP_ERR_WRONGVALUE)) != SNMPERR_SUCCESS){
@@ -242,6 +242,7 @@ snmp_scalar_get(clicon_handle               h,
     char   *reason = NULL;
     netsnmp_variable_list *requestvb = request->requestvb;
     cxobj  *xcache = NULL;
+    char   *body = NULL;
 
     clicon_debug(1, "%s", __FUNCTION__);
     /* Prepare backend call by constructing namespace context */
@@ -270,9 +271,8 @@ snmp_scalar_get(clicon_handle               h,
      * 1. From XML to SNMP string, there is a special case for enumeration, and for default value
      * 2. From SNMP string to SNMP binary value which invloves parsing
      */
-    if (x != NULL){
+    if (x != NULL && (body = xml_body(x)) != NULL){
-	assert(xml_spec(x) == ys);
+        if ((ret = type_xml2snmp_pre(body, ys, &xmlstr)) < 0)
-	if ((ret = type_xml2snmp_pre(xml_body(x), ys, &xmlstr)) < 0)
             goto done;
         if (ret == 0){
             if ((ret = netsnmp_request_set_error(request, SNMP_ERR_WRONGVALUE)) != SNMPERR_SUCCESS){
@@ -729,9 +729,13 @@ clixon_snmp_scalar_handler1(netsnmp_mib_handler          *handler,
             goto done;
         break;
     case MODE_GETNEXT:      /* 161 */
-	assert(0); // Not seen?
         break;
     case MODE_SET_RESERVE1: /* 0 */
+        if (!yang_config_ancestor(sh->sh_ys) ||
+            nhreg->modes == HANDLER_CAN_RONLY){
+            netsnmp_request_set_error(request, SNMP_ERR_NOTWRITABLE);
+            goto done;
+        }
         /* Translate from YANG ys leaf type to SNMP asn1.1 type ids (not value), also cvtype */
         if (type_yang2asn1(sh->sh_ys, &asn1_type, 0) < 0)
             goto done;
@@ -748,10 +752,30 @@ clixon_snmp_scalar_handler1(netsnmp_mib_handler          *handler,
     case MODE_SET_ACTION:   /* 2 */
         if (snmp_scalar_set(sh->sh_h, sh->sh_ys, NULL, NULL, reqinfo, request) < 0)
             goto done;
+        /*
+         * There does not seem to be a separate validation action and commit does not 
+         * return an error.
+         * Therefore validation is done here directly as well as discard if it fails.
+         */
+        if ((ret = clicon_rpc_validate(sh->sh_h, "candidate")) < 0)
+            goto done;
+        if (ret == 0){
+            clicon_rpc_discard_changes(sh->sh_h);
+            netsnmp_request_set_error(request, SNMP_ERR_COMMITFAILED);
+            goto done;
+        }
         break;
     case MODE_SET_COMMIT:   /* 3 */
-	if (clicon_rpc_commit(sh->sh_h) < 0)
+        if ((ret = clicon_rpc_commit(sh->sh_h, 0, 0, 0, NULL, NULL)) < 0)
             goto done;
+        if (ret == 0){
+            /* Note that error given in commit is not propagated to the snmp client,
+             * therefore validation is in the ACTION instead
+             */
+            clicon_rpc_discard_changes(sh->sh_h);
+            netsnmp_request_set_error(request, SNMP_ERR_COMMITFAILED);
+            goto done;
+        }
         break;
     case MODE_SET_FREE:     /* 4 */
         break;
@@ -763,6 +787,7 @@ clixon_snmp_scalar_handler1(netsnmp_mib_handler          *handler,
  ok:
     retval = SNMP_ERR_NOERROR;
  done:
+    clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     return retval;
 }
 
@@ -922,8 +947,9 @@ snmp_table_get(clicon_handle               h,
  * @param[in]  oidslen  OID length of scalar
  * @param[in]  reqinfo  Agent transaction request structure
  * @param[in]  request  The netsnmp request info structure.
+ * @param[out] err      Error code if failed (retval = 0)
  * @retval     -1       Error
- * @retval     0        Object not found
+ * @retval     0        Failed, err set
  * @retval     1        OK
  */
 static int
@@ -932,7 +958,8 @@ snmp_table_set(clicon_handle               h,
                oid                        *oids,
                size_t                      oidslen,
                netsnmp_agent_request_info *reqinfo,
-	       netsnmp_request_info       *request)
+               netsnmp_request_info       *request,
+               int                        *err)
 {
     int        retval = -1;
     oid        oidt[MAX_OID_LEN] = {0,}; /* Table / list oid */
@@ -995,8 +1022,27 @@ snmp_table_set(clicon_handle               h,
     }
     if (ys == NULL){
         /* No leaf with matching OID */
+        *err = SNMP_NOSUCHOBJECT;
         goto fail;
     }
+    if (!yang_config_ancestor(ys)){
+        *err = SNMP_ERR_NOTWRITABLE;
+        goto fail;
+    }
+    {
+        char *modes_str = NULL;
+        int   modes;
+
+        if (yang_extension_value(ys, "max-access", IETF_YANG_SMIV2_NS, NULL, &modes_str) < 0)
+            goto done;
+        if (modes_str){
+            modes = snmp_access_str2int(modes_str);
+            if (modes == HANDLER_CAN_RONLY){
+                *err = SNMP_ERR_NOTWRITABLE;
+                goto fail;
+            }
+        }
+    }
     if (type_yang2asn1(ys, &asn1_type, 0) < 0)
         goto done;
     requestvb = request->requestvb;
@@ -1033,6 +1079,7 @@ snmp_table_set(clicon_handle               h,
     }
     if (oidilen != 0){
         clicon_err(OE_YANG, 0, "Expected oidlen 0 but is %zu", oidilen);
+        *err = SNMP_NOSUCHOBJECT;
         goto fail;
     }
     if (yrowst){
@@ -1242,6 +1289,7 @@ clixon_snmp_table_handler1(netsnmp_mib_handler          *handler,
     cbuf                   *cb = NULL;
     int                     ret;
     netsnmp_variable_list  *requestvb;
+    int                     err = 0;
     
     clicon_debug(2, "%s", __FUNCTION__);
     if ((ret = snmp_common_handler(handler, nhreg, reqinfo, request, 1, &sh)) < 0)
@@ -1275,7 +1323,6 @@ clixon_snmp_table_handler1(netsnmp_mib_handler          *handler,
                                       reqinfo, request)) < 0)
             goto done;
         if (ret == 0){
-	    //	    if ((ret = netsnmp_request_set_error(request, SNMP_NOSUCHOBJECT)) != SNMPERR_SUCCESS){
             if ((ret = netsnmp_request_set_error(request, SNMP_NOSUCHOBJECT)) != SNMPERR_SUCCESS){
 
                 clicon_err(OE_SNMP, ret, "netsnmp_request_set_error");
@@ -1285,6 +1332,10 @@ clixon_snmp_table_handler1(netsnmp_mib_handler          *handler,
         }
         break;
     case MODE_SET_RESERVE1: // 0
+        if (!yang_config_ancestor(sh->sh_ys)){
+            netsnmp_request_set_error(request, SNMP_ERR_NOTWRITABLE);
+            goto done;;     
+        }
         // Check types: compare type in requestvb to yang type (or do later)
         break;
     case MODE_SET_RESERVE2: // 1
@@ -1292,19 +1343,36 @@ clixon_snmp_table_handler1(netsnmp_mib_handler          *handler,
     case MODE_SET_ACTION:   // 2
         if ((ret = snmp_table_set(sh->sh_h, sh->sh_ys,
                                   requestvb->name, requestvb->name_length,
-				  reqinfo, request)) < 0)
+                                  reqinfo, request, &err)) < 0)
             goto done;
         if (ret == 0){
-	    if ((ret = netsnmp_request_set_error(request, SNMP_NOSUCHINSTANCE)) != SNMPERR_SUCCESS){
+            if ((ret = netsnmp_request_set_error(request, err)) != SNMPERR_SUCCESS){
                 clicon_err(OE_SNMP, ret, "netsnmp_request_set_error");
                 goto done;
             }
             clicon_debug(1, "%s Nosuchinstance", __FUNCTION__);
         }
+        /*
+         * There does not seem to be a separate validation action and commit does not 
+         * return an error.
+         * Therefore validation is done here directly as well as discard if it fails.
+         */
+        if ((ret = clicon_rpc_validate(sh->sh_h, "candidate")) < 0)
+            goto done;
+        if (ret == 0){
+            clicon_rpc_discard_changes(sh->sh_h);
+            netsnmp_request_set_error(request, SNMP_ERR_COMMITFAILED);
+            goto done;
+        }
         break;
     case MODE_SET_COMMIT:   // 3
-	if (clicon_rpc_commit(sh->sh_h) < 0)
+        if ((ret = clicon_rpc_commit(sh->sh_h, 0, 0, 0, NULL, NULL)) < 0)
             goto done;
+        if (ret == 0){
+            clicon_rpc_discard_changes(sh->sh_h);
+            netsnmp_request_set_error(request, SNMP_ERR_COMMITFAILED);
+            goto done;
+        }
         break;
     case MODE_SET_FREE:     // 4
         break;

apps/snmp/snmp_lib.c

@@ -457,7 +457,9 @@ type_yang2asn1(yang_stmt    *ys,
         if (yang_extension_value(yrp, "display-hint", IETF_YANG_SMIV2_NS, NULL, &display_hint) < 0)
             goto done;  
         /* RFC2578/2579 but maybe all strings with display-hint should use this, eg exist>0? */
-	if (display_hint && strcmp(display_hint, "255t")==0)
+        if (display_hint &&
+            (strcmp(display_hint, "255a")==0 ||
+             strcmp(display_hint, "255t")==0))
             at = CLIXON_ASN_FIXED_STRING;
     }
     if (asn1_type)
@@ -631,8 +633,8 @@ type_xml2snmp_pre(char      *xmlstr0,
     char      *str = NULL;
     int        ret;
 
-    if (xmlstr1 == NULL){
+    if (xmlstr0 == NULL || xmlstr1 == NULL){
-	clicon_err(OE_UNIX, EINVAL, "xmlstr1");
+        clicon_err(OE_UNIX, EINVAL, "xmlstr0/1 is NULL");
         goto done;
     }
     /* Get yang type of leaf and trasnslate to ASN.1 */

apps/snmp/snmp_main.c

@@ -198,7 +198,7 @@ clixon_snmp_input_cb(int   s,
     clicon_handle  h = (clicon_handle)arg;
     int            ret;
 
-    clicon_debug(1, "%s %d", __FUNCTION__, s);
+    clicon_debug(2, "%s %d", __FUNCTION__, s);
     FD_ZERO(&readfds);
     FD_SET(s, &readfds);
     (void)snmp_read(&readfds);
@@ -309,7 +309,7 @@ usage(clicon_handle h,
     fprintf(stderr, "usage:%s\n"
             "where options are\n"
             "\t-h\t\tHelp\n"
-	    "\t-D <level>\tDebug level\n"
+            "\t-D <level>\tDebug level (>1 for extensive libnetsnmp debug)\n"
             "\t-f <file>\tConfiguration file (mandatory)\n"
             "\t-l (e|o|s|f<file>) Log on std(e)rr, std(o)ut, (s)yslog(default), (f)ile\n"
             "\t-z\t\tKill other %s daemon and exit\n"
@@ -384,7 +384,9 @@ main(int    argc,
      */
     clicon_log_init(__PROGRAM__, dbg?LOG_DEBUG:LOG_INFO, logdst); 
     clicon_debug_init(dbg, NULL); 
-
+    /* This is netsnmplib debugging which is quite extensive + only if compiled w debug */
+    if (dbg > 1)
+        snmp_set_do_debugging(1);
     /*
      * Register error category and error/log callbacks for netsnmp special error handling
      */

doc/DEVELOP.md

@@ -47,15 +47,14 @@ myfn(int           par1,
     ms = NULL;
 ```
 Notes:
-1. the return type of the function and all qualifers on first line (`static int`)
+1. The return type of the function and all qualifers on first line (`static int`)
-2. function name and first parameter on second line, thereafter each parameter on own line
+2. Function name and first parameter on second line, thereafter each parameter on own line
 3. Each parameter indented to match the "longest" (`my_structure`)
 4. Pointer declarations written: `type *p`, not: `type* p`.
 5. All local variables in a function declared at top of function, not inline with C-statements.
 6. Local variables can be initialized with scalars or constants, not eg malloc or functions with return values that need to be  checked for errors
 7. There is a single empty line between local variable declarations and the first function statement.
 
-
 Function signatures are declared in include files or in forward declaration using "one-line" syntax, unless very long:
 ```
 static int myfn(int par1, my_structure *par2);
@@ -123,6 +122,28 @@ files, there is only a single level of include file dependencies.
 
 The drawback is that the same include file may need to be repeated in many .c files.
 
+### Structs
+
+Struct fields should have a prefix to distinguish them from other struct fields. The prefix should use an abbreviation of the struct name.
+
+Example:
+```
+struct my_struct{
+  int   ms_foo;
+  char *ms_string[42];
+}
+```
+where `ms_` is the prefix and is an abbreviation of `my_struct`.
+
+### Global variables
+
+Try to avoid global variables.
+
+If you absolutely need one, try to contain it as static within a
+single C-file, ie do not declare it extern and use it elsewhere. 
+
+Also, always prepend a global variable with `_`, underscore.
+
 ## How to work in git
 
 Clixon uses semantic versioning (https://semver.org).

doc/Doxyfile.graphs

@@ -743,7 +743,7 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT     = ../lib/src/ ../lib/clicon/ ../apps/cli/ ../apps/backend/ ../apps/restconf/ ../apps/netconf ../apps/dbctrl ../datastore ../datastore/text
+INPUT     = ../lib/src/ ../lib/clicon/ ../apps/cli/ ../apps/backend/ ../apps/restconf/ ../apps/netconf ../apps/snmp
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

doc/Makefile.in

@@ -37,7 +37,6 @@ datarootdir	= @datarootdir@
 docdir		= @docdir@
 pdflatex	= @PDFLATEX@
 
-
 SUBDIRS = 
 
 .PHONY: clean all doc $(SUBDIRS)

example/main/example.xml.in

@@ -1,6 +1,7 @@
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>/usr/local/etc/example.xml</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_FEATURE>ietf-netconf:confirmed-commit</CLICON_FEATURE>
   <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE>
   <CLICON_FEATURE>clixon-restconf:fcgi</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>

example/main/example_cli.cli

@@ -48,7 +48,15 @@ delete("Delete a configuration item") {
       all("Delete whole candidate configuration"), delete_all("candidate");
 }
 validate("Validate changes"), cli_validate();
-commit("Commit the changes"), cli_commit();
+commit("Commit the changes"), cli_commit(); {
+  [persist-id("Specify the 'persist' value of a previous confirmed-commit") <persist-id-val:string show:"string">("The 'persist' value of the persistent confirmed-commit")], cli_commit(); {
+    <cancel:string keyword:cancel>("Cancel an ongoing confirmed-commit"), cli_commit();
+    <confirmed:string keyword:confirmed>("Require a confirming commit") {
+       [persist("Make this confirmed-commit persistent") <persist-val:string show:"string">("The value that must be provided as 'persist-id' in the confirming-commit or cancel-commit")]
+       [<timeout:uint32 range[1:4294967295] show:"1..4294967295">("The rollback timeout in seconds")], cli_commit();
+    }
+  }
+}
 quit("Quit"), cli_quit();
 
 debug("Debugging parts of the system"){

include/clixon_config.h.in

@@ -27,6 +27,9 @@
 /* Define to 1 if you have the <cligen/cligen.h> header file. */
 #undef HAVE_CLIGEN_CLIGEN_H
 
+/* Define to 1 if you have the <curl.h> header file. */
+#undef HAVE_CURL_H
+
 /* Define to 1 if you have the `getpeereid' function. */
 #undef HAVE_GETPEEREID
 

lib/clixon/clixon_data.h

@@ -66,6 +66,7 @@ int clicon_data_get(clicon_handle h, const char *name, char **val);
 int clicon_data_set(clicon_handle h, const char *name, char *val);
 int clicon_data_del(clicon_handle h, const char *name);
 
+/* Get generic clixon data on the form <name>=<ptr> where <ptr> is void* */
 int clicon_ptr_get(clicon_handle h, const char *name, void **ptr);
 int clicon_ptr_set(clicon_handle h, const char *name, void *ptr);
 int clicon_ptr_del(clicon_handle h, const char *name);

lib/clixon/clixon_datastore.h

@@ -76,5 +76,6 @@ int xmldb_modified_set(clicon_handle h, const char *db, int value);
 int xmldb_empty_get(clicon_handle h, const char *db);
 int xmldb_dump(clicon_handle h, FILE *f, cxobj *xt);
 int xmldb_print(clicon_handle h, FILE *f);
+int xmldb_rename(clicon_handle h, const char *db, const char *newdb, const char *suffix);
 
 #endif /* _CLIXON_DATASTORE_H */

lib/clixon/clixon_log.h

@@ -58,6 +58,7 @@ int clicon_log_file(char *filename);
 int clicon_log_string_limit_set(size_t sz);
 size_t clicon_log_string_limit_get(void);
 int clicon_get_logflags(void);
+int clicon_log_str(int level, char *msg);
 int clicon_log(int level, const char *format, ...) __attribute__ ((format (printf, 2, 3)));
 int clicon_debug(int dbglevel, const char *format, ...) __attribute__ ((format (printf, 2, 3)));
 int clicon_debug_init(int dbglevel, FILE *f);

lib/clixon/clixon_proto_client.h

@@ -63,7 +63,7 @@ int clicon_rpc_get_pageable_list(clicon_handle h, char *datastore, char *xpath,
 int clicon_rpc_close_session(clicon_handle h);
 int clicon_rpc_kill_session(clicon_handle h, uint32_t session_id);
 int clicon_rpc_validate(clicon_handle h, char *db);
-int clicon_rpc_commit(clicon_handle h);
+int clicon_rpc_commit(clicon_handle h, int confirmed, int cancel, uint32_t timeout, char *persist, char *persist_id);
 int clicon_rpc_discard_changes(clicon_handle h);
 int clicon_rpc_create_subscription(clicon_handle h, char *stream, char *filter, int *s);
 int clicon_rpc_debug(clicon_handle h, int level);

lib/clixon/clixon_xml_map.h

@@ -75,6 +75,7 @@ int yang_enum2valstr(yang_stmt *ytype, char *enumstr, char **valstr);
 int yang_enum_int_value(cxobj *node, int32_t *val);
 int xml_copy_marked(cxobj *x0, cxobj *x1);
 int yang_check_when_xpath(cxobj *xn, cxobj *xp, yang_stmt *yn, int *hit, int *nrp, char **xpathp);
+int yang_xml_mandatory(cxobj *xt, yang_stmt *ys);
 int xml_rpc_isaction(cxobj *xn);
 int xml_find_action(cxobj *xn, int top, cxobj **xap);
 int purge_tagged_nodes(cxobj *xn, char *ns, char *name, char *value, int keepnode);

lib/clixon/clixon_yang.h

@@ -272,7 +272,6 @@ int        yang_apply(yang_stmt *yn, enum rfc_6020 key, yang_applyfn_t fn, int f
 int        yang_datanode(yang_stmt *ys);
 int        yang_abs_schema_nodeid(yang_stmt *ys, char *schema_nodeid, yang_stmt **yres);
 int        yang_desc_schema_nodeid(yang_stmt *yn, char *schema_nodeid, yang_stmt **yres);
-int        yang_mandatory(yang_stmt *ys);
 int        yang_config(yang_stmt *ys);
 int        yang_config_ancestor(yang_stmt *ys);
 int        yang_features(clicon_handle h, yang_stmt *yt);

lib/src/clixon_datastore.c

@@ -199,6 +199,7 @@ xmldb_copy(clicon_handle h,
     cxobj              *x1 = NULL;  /* from */
     cxobj              *x2 = NULL;  /* to */
 
+    clicon_debug(1, "%s %s %s", __FUNCTION__, from, to);
     /* XXX lock */
     if (clicon_datastore_cache(h) != DATASTORE_NOCACHE){
         /* Copy in-memory cache */
@@ -252,7 +253,6 @@ xmldb_copy(clicon_handle h,
     if (tofile)
         free(tofile);
     return retval;
-
 }
 
 /*! Lock database
@@ -350,12 +350,13 @@ xmldb_islocked(clicon_handle h,
     return de->de_id;
 }
 
-/*! Check if db exists 
+/*! Check if db exists or is empty
  * @param[in]  h   Clicon handle
  * @param[in]  db  Database
  * @retval -1  Error
  * @retval  0  No it does not exist
  * @retval  1  Yes it exists
+ * @note  An empty datastore is treated as not existent so that a backend after dropping priviliges can re-create it
  */
 int 
 xmldb_exists(clicon_handle h, 
@@ -365,12 +366,18 @@ xmldb_exists(clicon_handle h,
     char               *filename = NULL;
     struct stat         sb;
 
+    clicon_debug(2, "%s %s", __FUNCTION__, db);
     if (xmldb_db2file(h, db, &filename) < 0)
         goto done;
     if (lstat(filename, &sb) < 0)
         retval = 0;
+
+    else{
+        if (sb.st_size == 0)
+            retval = 0;
         else
             retval = 1;
+    }
  done:
     if (filename)
         free(filename);
@@ -404,6 +411,7 @@ xmldb_clear(clicon_handle h,
  * @param[in]  db  Database
  * @retval -1  Error
  * @retval  0  OK
+ * @note  Datastore is not actually deleted so that a backend after dropping priviliges can re-create it
  */
 int 
 xmldb_delete(clicon_handle h, 
@@ -413,6 +421,7 @@ xmldb_delete(clicon_handle h,
     char               *filename = NULL;
     struct stat         sb;
     
+    clicon_debug(2, "%s %s", __FUNCTION__, db);
     if (xmldb_clear(h, db) < 0)
         goto done;
     if (xmldb_db2file(h, db, &filename) < 0)
@@ -445,6 +454,7 @@ xmldb_create(clicon_handle h,
     db_elmnt           *de = NULL;
     cxobj              *xt = NULL;
 
+    clicon_debug(2, "%s %s", __FUNCTION__, db);
     if ((de = clicon_db_elmnt_get(h, db)) != NULL){
         if ((xt = de->de_xml) != NULL){
             xml_free(xt);
@@ -594,3 +604,48 @@ xmldb_print(clicon_handle h,
  done:
     return retval;
 }
+
+/*! Rename an XML database
+ * @param[in]  h        Clicon handle
+ * @param[in]  db       Database name
+ * @param[in]  newdb    New Database name; if NULL, then same as old
+ * @param[in]  suffix   Suffix to append to new database name
+ * @retval    -1        Error
+ * @retval     0        OK
+ * @note if newdb and suffix are null, OK is returned as it is a no-op
+ */
+int
+xmldb_rename(clicon_handle h,
+             const char    *db,
+             const char    *newdb,
+             const char    *suffix)
+{
+    int    retval = -1;
+    char  *old;
+    char  *fname = NULL;
+    cbuf  *cb = NULL;
+
+    if ((xmldb_db2file(h, db, &old)) < 0)
+        goto done;
+    if (newdb == NULL && suffix == NULL)        // no-op
+        goto done;
+    if ((cb = cbuf_new()) == NULL){
+        clicon_err(OE_XML, errno, "cbuf_new");
+        goto done;
+    }
+    cprintf(cb, "%s", newdb == NULL ? old : newdb);
+    if (suffix)
+        cprintf(cb, "%s", suffix);
+    fname = cbuf_get(cb);
+    if ((rename(old, fname)) < 0) {
+        clicon_err(OE_UNIX, errno, "rename: %s", strerror(errno));
+        goto done;
+    };
+    retval = 0;
+ done:
+    if (cb)
+        cbuf_free(cb);
+    if (old)
+        free(old);
+    return retval;
+}

lib/src/clixon_log.c

@@ -238,7 +238,7 @@ slogtime(void)
  * @note syslog makes its own filtering, but if log to stderr we do it here
  * @see  clicon_debug
  */
-static int
+int
 clicon_log_str(int           level, 
                char         *msg)
 {

lib/src/clixon_netconf_lib.c

@@ -1752,7 +1752,8 @@ netconf_content_int2str(netconf_content nr)
  *  backend, and backend may implement more modules - please consider if using
  *  library routines for detecting capabilities here. In contrast, yang module 
  * list (RFC7895) is processed by the backend.
- * @note encode bodies, see xml_chardata_encode()
+ * @note If you add new, remember to encode bodies if needed, see xml_chardata_encode()
+ * @note
  * @see yang_modules_state_get
  * @see netconf_module_load
  */
@@ -1765,6 +1766,7 @@ netconf_hello_server(clicon_handle h,
     char *module_set_id;
     char *ietf_yang_library_revision;
     char *encstr = NULL;
+    yang_stmt *yspec = clicon_dbspec_yang(h);
 
     module_set_id = clicon_option_str(h, "CLICON_MODULE_SET_ID");
     cprintf(cb, "<hello xmlns=\"%s\">", NETCONF_BASE_NAMESPACE);
@@ -1778,6 +1780,7 @@ netconf_hello_server(clicon_handle h,
     /* A peer MAY include capabilities for previous NETCONF versions, to indicate
        that it supports multiple protocol versions. */
     cprintf(cb, "<capability>%s</capability>", NETCONF_BASE_CAPABILITY_1_0);
+
     /* Check if RFC7895 loaded and revision found */
     if ((ietf_yang_library_revision = yang_modules_revision(h)) != NULL){
         if (xml_chardata_encode(&encstr, "urn:ietf:params:netconf:capability:yang-library:1.0?revision=%s&module-set-id=%s",
@@ -1790,15 +1793,26 @@ netconf_hello_server(clicon_handle h,
             encstr = NULL;
         }
     }
+    /* RFC6241 Sec 8.3.  Candidate Configuration Capability */
     cprintf(cb, "<capability>urn:ietf:params:netconf:capability:candidate:1.0</capability>");
+    /* RFC6241 Sec 8.6.  Validate Capability */
     cprintf(cb, "<capability>urn:ietf:params:netconf:capability:validate:1.1</capability>");
+    /* rfc 6241 Sec 8.7 Distinct Startup Capability */
+    if (if_feature(yspec, "ietf-netconf", "startup"))
         cprintf(cb, "<capability>urn:ietf:params:netconf:capability:startup:1.0</capability>");
+    /* RFC6241 Sec 8.9.  XPath Capability */
     cprintf(cb, "<capability>urn:ietf:params:netconf:capability:xpath:1.0</capability>");
-    cprintf(cb, "<capability>urn:ietf:params:netconf:capability:notification:1.0</capability>");
     /* rfc6243 with-defaults capability modes */
     cprintf(cb, "<capability>");
     xml_chardata_cbuf_append(cb, "urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&also-supported=report-all,trim,report-all-tagged");
     cprintf(cb, "</capability>");
+    /* RFC5277 Notification Capability */
+    cprintf(cb, "<capability>urn:ietf:params:netconf:capability:notification:1.0</capability>");
+    /* It is somewhat arbitrary why some features/capabilities are hardocded and why some are not
+     * rfc 6241 Sec 8.4 confirmed-commit capabilities */
+    if (if_feature(yspec, "ietf-netconf", "confirmed-commit"))
+        cprintf(cb, "<capability>urn:ietf:params:netconf:capability:confirmed-commit:1.1</capability>");
+
     cprintf(cb, "</capabilities>");
     if (session_id) 
         cprintf(cb, "<session-id>%lu</session-id>", (long unsigned int)session_id);

lib/src/clixon_proto_client.c

@@ -83,6 +83,10 @@
 #include "clixon_netconf_lib.h"
 #include "clixon_proto_client.h"
 
+#define PERSIST_ID_XML_FMT "<persist-id>%s</persist-id>"
+#define PERSIST_XML_FMT "<persist>%s</persist>"
+#define TIMEOUT_XML_FMT "<confirm-timeout>%u</confirm-timeout>"
+
 /*! Connect to internal netconf socket
  */
 int
@@ -1221,8 +1225,10 @@ clicon_rpc_kill_session(clicon_handle h,
 /*! Send validate request to backend daemon
  * @param[in] h        CLICON handle
  * @param[in] db       Name of database
- * @retval    0        OK
+ * @retval    1        OK
+ * @retval    0        Invalid, netconf error return, and logged to syslog
  * @retval   -1        Error and logged to syslog
+ * @note error returns are logged but not returned
  */
 int
 clicon_rpc_validate(clicon_handle h, 
@@ -1249,9 +1255,10 @@ clicon_rpc_validate(clicon_handle h,
         goto done;
     if ((xerr = xpath_first(xret, NULL, "//rpc-error")) != NULL){
         clixon_netconf_error(xerr, CLIXON_ERRSTR_VALIDATE_FAILED, NULL);
+        retval = 0;
         goto done;      
     }
-    retval = 0;
+    retval = 1;
  done:
     if (msg)
         free(msg);
@@ -1262,11 +1269,24 @@ clicon_rpc_validate(clicon_handle h,
 
 /*! Commit changes send a commit request to backend daemon
  * @param[in] h          CLICON handle
- * @retval    0        OK
+ * @param[in] confirmed  If set, send commit/confirmed
- * @retval   -1        Error and logged to syslog
+ * @param[in] cancel     If set, send cancle-commit
+ * @param[in] timeout    For confirmed, a timeout in seconds (default 600s)
+ * @param[in] persist    A persist identifier to use
+ * @param[in] persist_id If cancel or confirmed, the persist id
+ * @retval    1          OK
+ * @retval    0          Invalid, netconf error return, and logged to syslog
+ * @retval   -1          Error
+ * @see rfc6241 Sec 8.4 Confirmed Commit Capability
+ * @note error returns are logged but not returned
  */
 int
-clicon_rpc_commit(clicon_handle h)
+clicon_rpc_commit(clicon_handle h,
+                  int           confirmed,
+                  int           cancel,
+                  uint32_t      timeout,
+                  char         *persist,
+                  char         *persist_id)
 {
     int                retval = -1;
     struct clicon_msg *msg = NULL;
@@ -1274,28 +1294,83 @@ clicon_rpc_commit(clicon_handle h)
     cxobj             *xerr;
     char              *username;
     uint32_t           session_id;
+    char              *persist_id_xml = NULL;
+    char              *persist_xml = NULL;
+    char              *timeout_xml = NULL;
 
+    if (persist_id) {
+        if ((persist_id_xml = malloc(strlen(persist_id) + strlen(PERSIST_ID_XML_FMT) + 1)) == NULL) {
+            clicon_err(OE_UNIX, 0, "malloc: %s", strerror(errno));
+        }
+        sprintf(persist_id_xml, PERSIST_ID_XML_FMT, persist_id);
+    }
+
+    if (persist) {
+        if ((persist_xml = malloc(strlen(persist) + strlen(PERSIST_XML_FMT) + 1)) == NULL) {
+            clicon_err(OE_UNIX, 0, "malloc: %s", strerror(errno));
+        };
+        sprintf(persist_xml, PERSIST_XML_FMT, persist);
+    }
+
+    if (timeout > 0) {
+
+        /* timeout is a uint32_t, so max value is 2^32, a 10-digit number, we'll just always malloc for a string that
+         * may be that large rather than calculate the string length
+         */
+
+        if ((timeout_xml = malloc(10 + 1 + strlen(TIMEOUT_XML_FMT))) == NULL) {
+            clicon_err(OE_UNIX, 0, "malloc: %s", strerror(errno));
+        };
+        sprintf(timeout_xml, TIMEOUT_XML_FMT, timeout);
+    }
     if (session_id_check(h, &session_id) < 0)
         goto done;
     username = clicon_username_get(h);
-    if ((msg = clicon_msg_encode(session_id,
+    if (cancel) {
-				 "<rpc xmlns=\"%s\" username=\"%s\" %s><commit/></rpc>",
+        msg = clicon_msg_encode(session_id,
+                                "<rpc xmlns=\"%s\" username=\"%s\" %s><cancel-commit>%s</cancel-commit></rpc>",
                                 NETCONF_BASE_NAMESPACE,
-				 username?username:"",
+                                username ? username : "",
-				 NETCONF_MESSAGE_ID_ATTR)) == NULL)
+                                NETCONF_MESSAGE_ID_ATTR,
+                                persist_id ? persist_id_xml : "");
+    } else if (confirmed) {
+        msg = clicon_msg_encode(session_id,
+                                "<rpc xmlns=\"%s\" username=\"%s\" %s><commit><confirmed/>%s%s%s</commit></rpc>",
+                                NETCONF_BASE_NAMESPACE,
+                                username ? username : "",
+                                NETCONF_MESSAGE_ID_ATTR,
+                                timeout ? timeout_xml : "",
+                                persist_id ? persist_id_xml : "",
+                                persist ? persist_xml : "");
+    } else {
+        msg = clicon_msg_encode(session_id,
+                                "<rpc xmlns=\"%s\" username=\"%s\" %s><commit>%s</commit></rpc>",
+                                NETCONF_BASE_NAMESPACE,
+                                username ? username : "",
+                                NETCONF_MESSAGE_ID_ATTR,
+                                persist ? persist_xml : "");
+    }
+    if (msg == NULL)
         goto done;
     if (clicon_rpc_msg(h, msg, &xret) < 0)
         goto done;
     if ((xerr = xpath_first(xret, NULL, "//rpc-error")) != NULL){
         clixon_netconf_error(xerr, CLIXON_ERRSTR_COMMIT_FAILED, NULL);
+        retval = 0;
         goto done;
     }
-    retval = 0;
+    retval = 1;
  done:
     if (xret)
         xml_free(xret);
     if (msg)
         free(msg);
+    if (persist_id_xml)
+        free(persist_id_xml);
+    if (persist_xml)
+        free(persist_xml);
+    if (timeout_xml)
+        free(timeout_xml);
     return retval;
 }
 
@@ -1479,7 +1554,7 @@ clicon_rpc_restconf_debug(clicon_handle h,
         clicon_err(OE_XML, 0, "rpc error"); /* XXX extract info from rpc-error */
         goto done;
     }
-    if ((retval = clicon_rpc_commit(h)) < 0)
+    if ((retval = clicon_rpc_commit(h, 0, 0, 0, NULL, NULL)) < 1)
         goto done;
  done:
     if (msg)

lib/src/clixon_validate.c

@@ -613,16 +613,20 @@ check_list_key(cxobj     *xt,
  * @retval    -1     Error
  */
 static int
-choice_mandatory_check(yang_stmt *ycase,
+choice_mandatory_check(cxobj     *xt,
+                       yang_stmt *ycase,
                        cxobj    **xret)
 {
     int        retval = -1;
     yang_stmt *yc = NULL;
     cbuf      *cb = NULL;
     int        fail = 0;
+    int        ret;
 
     while ((yc = yn_each(ycase, yc)) != NULL) {
-	if (yang_mandatory(yc)){
+        if ((ret = yang_xml_mandatory(xt, yc)) < 0)
+            goto done;
+        if (ret == 1){
             if (yang_flag_get(yc, YANG_FLAG_MARK))
                 yang_flag_reset(yc, YANG_FLAG_MARK);
             else if (fail == 0){
@@ -728,16 +732,20 @@ check_mandatory_case(cxobj     *xt,
         if ((y = xml_spec(x)) != NULL &&
             yang_ancestor_child(y, yc, &ym, &ycnew) != 0 &&
             yang_keyword_get(ycnew) == Y_CASE){
-	    if (ym && yang_mandatory(ym)){
+            if (ym){
+                if ((ret = yang_xml_mandatory(xt, ym)) < 0)
+                    goto done;
+                if (ret == 1){
                 if (yang_flag_get(ym, YANG_FLAG_MARK) != 0){
                     clicon_debug(1, "%s Already marked, shouldnt happen", __FUNCTION__);
                 }
                 yang_flag_set(ym, YANG_FLAG_MARK);
                 }
+            }
             if (ycase != NULL){
                 if (ycnew != ycase){ /* End of case, new case */
                     /* Check and clear marked mandatory */
-		    if ((ret = choice_mandatory_check(ycase, xret)) < 0)
+                    if ((ret = choice_mandatory_check(xt, ycase, xret)) < 0)
                         goto done;
                     if (ret == 0)
                         goto fail;
@@ -749,7 +757,7 @@ check_mandatory_case(cxobj     *xt,
         }
         else if (ycase != NULL){ /* End of case */
             /* Check and clear marked mandatory */
-	    if ((ret = choice_mandatory_check(ycase, xret)) < 0)
+            if ((ret = choice_mandatory_check(xt, ycase, xret)) < 0)
                 goto done;
             if (ret == 0)
                 goto fail;
@@ -757,7 +765,7 @@ check_mandatory_case(cxobj     *xt,
         }
     }
     if (ycase){
-	if ((ret = choice_mandatory_check(ycase, xret)) < 0)
+        if ((ret = choice_mandatory_check(xt, ycase, xret)) < 0)
             goto done;
         if (ret == 0)
             goto fail;
@@ -806,7 +814,7 @@ check_mandatory(cxobj     *xt,
     while ((yc = yn_each(yt, yc)) != NULL) {
         /* Choice is more complex because of choice/case structure and possibly hierarchical */
         if (yang_keyword_get(yc) == Y_CHOICE){ 
-	    if (yang_mandatory(yc)){
+            if (yang_xml_mandatory(xt, yc)){
                 x = NULL;
                 while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
                     if ((y = xml_spec(x)) != NULL &&
@@ -829,7 +837,9 @@ check_mandatory(cxobj     *xt,
             if (ret == 0)
                 goto fail;
         }
-	if (!yang_mandatory(yc)) /* Rest of yangs are immediate children */
+        if ((ret = yang_xml_mandatory(xt, yc)) < 0) /* Rest of yangs are immediate children */
+            goto done;
+        if (ret == 0)
             continue;
         switch (yang_keyword_get(yc)){
         case Y_CONTAINER:
@@ -1130,6 +1140,22 @@ xml_yang_validate_all(clicon_handle h,
         goto fail;
     }
     if (yang_config(yt) != 0){
+        if (yang_check_when_xpath(xt, xml_parent(xt), yt, &hit, &nr, &xpath) < 0)
+            goto done;
+        if (hit && nr == 0){
+            if ((cb = cbuf_new()) == NULL){
+                clicon_err(OE_UNIX, errno, "cbuf_new");
+                goto done;
+            }
+            cprintf(cb, "Failed WHEN condition of %s in module %s (WHEN xpath is %s)",
+                    xml_name(xt),
+                    yang_argument_get(ys_module(yt)),
+                    xpath);
+            if (xret && netconf_operation_failed_xml(xret, "application", 
+                                                     cbuf_get(cb)) < 0)
+                goto done;
+            goto fail;
+        }
         if ((ret = check_mandatory(xt, yt, xret)) < 0)
             goto done;
         if (ret == 0)
@@ -1204,22 +1230,6 @@ xml_yang_validate_all(clicon_handle h,
                 nsc = NULL;
             }
         }
-	if (yang_check_when_xpath(xt, xml_parent(xt), yt, &hit, &nr, &xpath) < 0)
-	    goto done;
-	if (hit && nr == 0){
-	    if ((cb = cbuf_new()) == NULL){
-		clicon_err(OE_UNIX, errno, "cbuf_new");
-		goto done;
-	    }
-	    cprintf(cb, "Failed WHEN condition of %s in module %s (WHEN xpath is %s)",
-		    xml_name(xt),
-		    yang_argument_get(ys_module(yt)),
-		    xpath);
-	    if (xret && netconf_operation_failed_xml(xret, "application", 
-						     cbuf_get(cb)) < 0)
-		goto done;
-	    goto fail;
-	}
     }
     x = NULL;
     while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {

lib/src/clixon_validate_minmax.c

@@ -658,7 +658,6 @@ xml_yang_minmax_recurse(cxobj  *xt,
     
     yt = xml_spec(xt); /* If yt == NULL, then no gap-analysis is done */
     while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL){
-	clicon_debug(1, "%s x:%s", __FUNCTION__, xml_name(x));
         if ((y = xml_spec(x)) == NULL)
             continue;
         keyw = yang_keyword_get(y);

lib/src/clixon_xml_map.c

@@ -2089,10 +2089,14 @@ xml_copy_marked(cxobj *x0,
 
 /*! Check when condition 
  * 
- * @param[in]   h    Clixon handle
  * @param[in]   xn     XML node, can be NULL, in which case it is added as dummy under xp
  * @param[in]   xp     XML parent
- * @param[in]   ys   Yang node
+ * @param[in]   yn     Yang node
+ * @param[out]  hit    when statement found
+ * @param[out]  nrp    1: when stmt evaluates to true
+ * @param[out]  xpathp when stmts xpath
+ * @retval      0      OK
+ * @retval      -1     Error
  * First variants of WHEN: Augmented and uses when using special info in node
  * Second variant of when, actual "when" sub-node RFC 7950 Sec 7.21.5. Can only be one.
  */
@@ -2155,6 +2159,77 @@ yang_check_when_xpath(cxobj        *xn,
     return retval;
 }
 
+/*! Check if node is (recursively) mandatory also checking when conditional
+ *
+ * @param[in]  xn  Optional XML node
+ * @param[in]  xp  XML parent
+ * @param[in]  ys  YANG node
+ * @retval     1   Recursively contains a mandatory node
+ * @retval     0   Does not contain a mandatory node
+ * @retval    -1   Error
+ * @see RFC7950 Sec 3:
+ *   o  mandatory node: A mandatory node is one of:
+ *      1)  A leaf, choice, anydata, or anyxml node with a "mandatory"
+ *         statement with the value "true".
+ *      2) # see below
+ *      3)  A container node without a "presence" statement and that has at
+ *         least one mandatory node as a child.
+ */
+int
+yang_xml_mandatory(cxobj     *xt,
+                   yang_stmt *ys)
+{
+    int           retval = -1;
+    yang_stmt    *ym;
+    cg_var       *cv;
+    enum rfc_6020 keyw;
+    cxobj        *xs = NULL;
+    int           ret;
+    yang_stmt    *yc;
+    int           hit;
+    int           nr;
+
+    /* Create dummy xs if not exist */
+    if ((xs = xml_new(yang_argument_get(ys), xt, CX_ELMNT)) == NULL)
+        goto done;
+    xml_spec_set(xs, ys);
+    if (yang_check_when_xpath(xs, xt, ys, &hit, &nr, NULL) < 0)
+        goto done;
+    if (hit && !nr){
+        retval = 0;
+        goto done;
+    }
+    keyw = yang_keyword_get(ys);
+    if (keyw == Y_LEAF || keyw == Y_CHOICE || keyw == Y_ANYDATA || keyw == Y_ANYXML){
+        if ((ym = yang_find(ys, Y_MANDATORY, NULL)) != NULL){
+            if ((cv = yang_cv_get(ym)) != NULL){ /* shouldnt happen */
+                retval = cv_bool_get(cv);
+                goto done;
+            }
+        }
+    }
+    /* 3) A container node without a "presence" statement and that has at
+     *    least one mandatory node as a child. */
+    else if (keyw == Y_CONTAINER && 
+             yang_find(ys, Y_PRESENCE, NULL) == NULL){
+        yc = NULL;
+        while ((yc = yn_each(ys, yc)) != NULL) {
+            if ((ret = yang_xml_mandatory(xs, yc)) < 0)
+                goto done;
+            if (ret == 1)
+                goto mandatory;
+        }
+    }
+    retval = 0; /* Does not contain mandatory node */
+ done:
+    if (xs != NULL)
+        xml_purge(xs);
+    return retval;
+ mandatory:
+    retval = 1;
+    goto done;
+}
+
 /*! Is XML node (ie under <rpc>) an action, ie name action and belong to YANG_XML_NAMESPACE?
  * @param[in]   xn   XML node
  * @retval      1    Yes, an action

lib/src/clixon_yang.c

@@ -3213,65 +3213,6 @@ yang_desc_schema_nodeid(yang_stmt    *yn,
     return retval;
 }
 
-/*! Check if this leaf is mandatory or not
- * Note: one can cache this value in ys_cvec instead of functionally evaluating it.
- * @retval 1 yang statement is leaf and it has a mandatory sub-stmt with value true
- * @retval 0 The negation of conditions for return value 1.
- * @see RFC7950 Sec 3:
- *   o  mandatory node: A mandatory node is one of:
- *      1)  A leaf, choice, anydata, or anyxml node with a "mandatory"
- *         statement with the value "true".
- *      2) # see below
- *      3)  A container node without a "presence" statement and that has at
- *         least one mandatory node as a child.
- *
- * @note There is also this statement 
- *      2)  A list or leaf-list node with a "min-elements" statement with a
- *         value greater than zero.
- * which we ignore here since:
- *      (a) it does not consider the XML siblings and  therefore returns false positives
- *      (b) where the actual check is catched by check_list_unique_minmax()
- */
-int
-yang_mandatory(yang_stmt *ys)
-{
-    yang_stmt *ym;
-    cg_var    *cv;
-
-    /* 1) A leaf, choice, anydata, or anyxml node with a "mandatory"
-     *    statement with the value "true". */
-    if (ys->ys_keyword == Y_LEAF || ys->ys_keyword == Y_CHOICE ||
-	ys->ys_keyword == Y_ANYDATA || ys->ys_keyword == Y_ANYXML){
-	if ((ym = yang_find(ys, Y_MANDATORY, NULL)) != NULL){
-	    if ((cv = yang_cv_get(ym)) != NULL) /* shouldnt happen */
-		return cv_bool_get(cv);
-	}
-    }
-#if 0 /* See note above */
-    /* 2) A list or leaf-list node with a "min-elements" statement with a
-     *    value greater than zero. */
-    else if (ys->ys_keyword == Y_LIST || ys->ys_keyword == Y_LEAF_LIST){
-	if ((ym = yang_find(ys, Y_MIN_ELEMENTS, NULL)) != NULL){
-	    cv = yang_cv_get(ym);
-	    return cv_uint32_get(cv) > 0; /* Not true if XML considered */
-	}
-    }
-#endif
-    /* 3) A container node without a "presence" statement and that has at
-     *    least one mandatory node as a child. */
-    else if (ys->ys_keyword == Y_CONTAINER && 
-	     yang_find(ys, Y_PRESENCE, NULL) == NULL){
-	yang_stmt *yc;
-	int i;
-	for (i=0; i<ys->ys_len; i++){
-	    yc = ys->ys_stmt[i];
-	    if (yang_mandatory(yc))
-		return 1;
-	}
-    }
-    return 0;
-}
-
 /*! Return config state of this node
  * @param[in] ys  Yang statement
  * @retval    0   If node has a config sub-statement and it is false

test/lib.sh

@@ -98,6 +98,9 @@ DEFAULTNS="$DEFAULTONLY message-id=\"42\""
 # Minimal hello message as a prelude to netconf rpcs
 DEFAULTHELLO="<?xml version=\"1.0\" encoding=\"UTF-8\"?><hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.0</capability><capability>urn:ietf:params:netconf:base:1.1</capability></capabilities></hello>]]>]]>"
 
+# Minimal hello message that excludes 1.1 capability, in the case EOM style framing is needed
+HELLONO11="<?xml version=\"1.0\" encoding=\"UTF-8\"?><hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.0</capability></capabilities></hello>]]>]]>"
+
 # XXX cannot get this to work for all combinations of nc/netcat fcgi/native
 # But leave it here for debugging where netcat works properly
 if [ -n "$(type netcat 2> /dev/null)" ]; then
@@ -569,6 +572,7 @@ function wait_backend(){
 # Start restconf daemon
 # @see wait_restconf
 function start_restconf(){
+    STTYSETTINGS=`stty -g`
     # Start in background 
     echo "sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $*"
     sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $* </dev/null &>/dev/null &
@@ -625,6 +629,8 @@ function wait_restconf(){
     if [ $valgrindtest -eq 3 ]; then 
         sleep 2 # some problems with valgrind
     fi
+
+  stty $STTYSETTINGS
 }
 
 # Wait for restconf to stop 

test/test_api.sh

@@ -228,7 +228,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
  fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_augment_state.sh

@@ -144,7 +144,8 @@ if [ $BE -ne 0 ]; then
     new "start backend -s init -f $cfg -- -sS $fstate"
     start_backend -s init -f $cfg -- -sS $fstate
 fi
-new "waiting"
+
+new "wait backend"
 wait_backend
 
 #-----------------------------

test/test_augment_trans.sh

@@ -109,7 +109,8 @@ if [ $BE -ne 0 ]; then
     new "start backend -s init -f $cfg"
     start_backend -s init -f $cfg
 fi
-new "waiting"
+
+new "wait backend"
 wait_backend
 
 new "get-config empty"

test/test_client.sh

@@ -138,7 +138,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
  fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_confirmed_commit.sh

@@ -0,0 +1,475 @@
+#!/usr/bin/env bash
+# Netconf confirm commit capability
+# See RFC 6241 Section 8.4 and RFC 8040 Section 1.4
+# TODO:
+# - privileges drop
+# - lock check
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+tmp=$dir/tmp.x
+fyang=$dir/clixon-example.yang
+
+# Backend user for priv drop, otherwise root
+USER=${BUSER}
+
+# Define default restconfig config: RESTCONFIG
+RESTCONFIG=$(restconf_config none false)
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_FEATURE>ietf-netconf:confirmed-commit</CLICON_FEATURE>
+  <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE> <!-- Use auth-type=none -->
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>$dir/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+  <CLICON_BACKEND_USER>$USER</CLICON_BACKEND_USER>
+  <CLICON_BACKEND_PRIVILEGES>drop_perm</CLICON_BACKEND_PRIVILEGES>
+  $RESTCONFIG
+</clixon-config>
+EOF
+
+cat <<EOF > $fyang
+module clixon-example{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix ex;
+
+    /* Generic config data */
+    container table{
+        list parameter{
+            key name;
+            leaf name{
+                type string;
+            }
+        }
+    }
+}
+EOF
+
+function data() {
+  if [[ "$1" == "" ]]
+  then
+    echo "<data/>"
+  else
+    echo "<data>$1</data>"
+  fi
+}
+
+# Pipe stdin to command and also do chunked framing (netconf 1.1)
+# Arguments:
+# - Command
+# - expected command return value (0 if OK)
+# - stdin input1  This is NOT encoded, eg preamble/hello
+# - stdin input2  This gets chunked encoding
+# - expect1 stdout outcome, can be partial and contain regexps
+# - expect2 stdout outcome This gets chunked encoding, must be complete netconf message
+# Use this if you want regex eg  ^foo$
+
+function rpc() {
+  expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS>$1</rpc>" "" "<rpc-reply $DEFAULTNS>$2</rpc-reply>"
+}
+
+function commit() {
+  new "commit $1"
+  if [[ "$1" == "" ]]
+  then
+    rpc "<commit/>" "<ok/>"
+  else
+    rpc "<commit>$1</commit>" "<ok/>"
+  fi
+}
+
+function edit_config() {
+  TARGET="$1"
+  CONFIG="$2"
+  new "edit-config $1 $2"
+  rpc "<edit-config><target><$TARGET/></target><config>$CONFIG</config></edit-config>" "<ok/>"
+}
+
+function assert_config_equals() {
+  TARGET="$1"
+  EXPECTED="$2"
+#  new "get-config: $TARGET"
+  rpc "<get-config><source><$TARGET/></source></get-config>" "$(data "$EXPECTED")"
+}
+
+# delete all
+function reset() {
+  rpc "<edit-config><target><candidate/></target><default-operation>none</default-operation><config operation=\"delete\"/></edit-config>" "<ok/>"
+  commit
+  assert_config_equals "candidate" ""
+  assert_config_equals "running" ""
+}
+
+CANDIDATE_PATH="/usr/local/var/$APPNAME/candidate_db"
+RUNNING_PATH="/usr/local/var/$APPNAME/running_db"
+ROLLBACK_PATH="/usr/local/var/$APPNAME/rollback_db"
+FAILSAFE_PATH="/usr/local/var/$APPNAME/failsafe_db"
+
+
+CONFIGB="<table xmlns=\"urn:example:clixon\"><parameter><name>eth0</name></parameter></table>"
+CONFIGC="<table xmlns=\"urn:example:clixon\"><parameter><name>eth1</name></parameter></table>"
+CONFIGCONLY="<parameter xmlns=\"urn:example:clixon\"><name>eth1</name></parameter>" # restcpnf
+CONFIGBPLUSC="<table xmlns=\"urn:example:clixon\"><parameter><name>eth0</name></parameter><parameter><name>eth1</name></parameter></table>"
+FAILSAFE_CFG="<table xmlns=\"urn:example:clixon\"><parameter><name>eth99</name></parameter></table>"
+
+new "test params: -f $cfg"
+
+# Bring your own backend
+if [ $BE -ne 0 ]; then
+    # kill old backend (if any)
+    new "kill old backend"
+    stop_backend -f $cfg
+
+    new "start backend  -s init -f $cfg"
+    start_backend -s init -f $cfg
+fi
+
+new "wait backend"
+wait_backend
+
+new "Hello check confirm-commit capability"
+expecteof "$clixon_netconf -f $cfg" 0 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability></capabilities></hello>]]>]]>" "<capability>urn:ietf:params:netconf:capability:confirmed-commit:1.1</capability>" '^$'
+
+################################################################################
+new "netconf ephemeral confirmed-commit rolls back after disconnect"
+reset
+edit_config "candidate" "$CONFIGB"
+assert_config_equals "candidate" "$CONFIGB"
+commit "<confirmed/><confirm-timeout>30</confirm-timeout>"
+assert_config_equals "running" ""
+
+################################################################################
+
+new "netconf persistent confirmed-commit"
+reset
+edit_config "candidate" "$CONFIGB"
+commit "<confirmed/><persist>a</persist>"
+assert_config_equals "running" "$CONFIGB"
+edit_config "candidate" "$CONFIGC"
+commit "<confirmed/><persist>ab</persist><persist-id>a</persist-id>"
+assert_config_equals "running" "$CONFIGBPLUSC"
+
+################################################################################
+
+new "netconf cancel-commit with invalid persist-id"
+rpc "<cancel-commit><persist-id>abc</persist-id></cancel-commit>" "<rpc-error><error-type>application</error-type><error-tag>invalid-value</error-tag><error-severity>error</error-severity><error-message>a confirmed-commit with the given persist-id was not found</error-message></rpc-error>"
+
+################################################################################
+
+new "netconf cancel-commit with valid persist-id"
+rpc "<cancel-commit><persist-id>ab</persist-id></cancel-commit>" "<ok/>"
+
+################################################################################
+
+new "netconf persistent confirmed-commit with timeout"
+reset
+edit_config "candidate" "$CONFIGB"
+commit "<confirmed/><confirm-timeout>2</confirm-timeout><persist>abcd</persist>"
+assert_config_equals "running" "$CONFIGB"
+sleep 2
+assert_config_equals "running" ""
+
+################################################################################
+
+new "netconf persistent confirmed-commit with reset timeout"
+reset
+edit_config "candidate" "$CONFIGB"
+commit "<confirmed/><persist>abcde</persist><confirm-timeout>5</confirm-timeout>"
+assert_config_equals "running" "$CONFIGB"
+edit_config "candidate" "$CONFIGC"
+commit "<confirmed/><persist-id>abcde</persist-id><persist>abcdef</persist><confirm-timeout>10</confirm-timeout>"
+# prove the new timeout is active by sleeping longer than first timeout. get config, assert == B+C
+sleep 6
+assert_config_equals "running" "$CONFIGBPLUSC"
+# now sleep long enough for rollback to happen; get config, assert == A
+sleep 5
+assert_config_equals "running" ""
+
+################################################################################
+
+new "netconf persistent confirming-commit to epehemeral confirmed-commit should rollback"
+reset
+edit_config "candidate" "$CONFIGB"
+commit "<confirmed/><persist/><confirm-timeout>10</confirm-timeout>"
+assert_config_equals "running" "$CONFIGB"
+commit "<confirmed/><persist-id/>"
+assert_config_equals "running" ""
+
+################################################################################
+
+new "netconf confirming-commit for persistent confirmed-commit with empty persist value"
+reset
+edit_config "candidate" "$CONFIGB"
+commit "<confirmed/><persist/><confirm-timeout>10</confirm-timeout>"
+assert_config_equals "running" "$CONFIGB"
+commit "<persist-id/>"
+assert_config_equals "running" "$CONFIGB"
+
+################################################################################
+# TODO reconsider logic around presence/absence of rollback_db as a signal as dropping permissions may impact ability
+# to unlink and/or create that file. see clixon_datastore.c#xmldb_delete() and backend_startup.c#startup_mode_startup()
+
+new "backend loads rollback if present at startup"
+reset
+edit_config "candidate" "$CONFIGB"
+commit ""
+edit_config "candidate" "$CONFIGC"
+commit "<persist>abcdefg</persist><confirmed/>"
+assert_config_equals "running" "$CONFIGBPLUSC"
+
+new "kill old backend"
+stop_backend -f $cfg                                            # kill backend and restart
+
+new "Check $ROLLBACK_PATH"
+[ -f "$ROLLBACK_PATH" ] || err "rollback_db doesn't exist!"     # assert rollback_db exists
+
+new "start backend -s running -f $cfg"
+start_backend -s running -f $cfg
+
+new "wait backend"
+wait_backend
+
+assert_config_equals "running" "$CONFIGB"
+
+new "Check $ROLLBACK_PATH removed"
+[ -f "ROLLBACK_PATH" ] && err "rollback_db still exists!"       # assert rollback_db doesn't exist
+
+new "kill old backend"
+stop_backend -f $cfg
+
+new "start backend -s init -f $cfg"
+start_backend -s init -f $cfg
+
+################################################################################
+new "backend loads failsafe at startup if rollback present but cannot be loaded"
+
+new "wait backend"
+wait_backend
+
+reset
+
+sudo tee "$FAILSAFE_PATH" > /dev/null << EOF                    # create a failsafe database
+<config>$FAILSAFE_CFG</config>
+EOF
+edit_config "candidate" "$CONFIGC"
+
+commit "<persist>foobar</persist><confirmed/>"
+
+assert_config_equals "running" "$CONFIGC"
+
+new "kill old backend"
+stop_backend -f $cfg                                            # kill the backend
+
+sudo rm $ROLLBACK_PATH                                          # modify rollback_db so it won't commit successfully
+
+sudo tee "$ROLLBACK_PATH" > /dev/null << EOF
+<foo>
+  <bar>
+    <baz/>
+    </bar>
+</foo>
+EOF
+
+new "start backend -s running -f $cfg"
+start_backend -s running -f $cfg
+
+new "wait backend"
+wait_backend
+
+assert_config_equals "running" "$FAILSAFE_CFG"
+
+new "kill old backend"
+stop_backend -f $cfg
+
+new "start backend -s init -f $cfg"
+start_backend -s init -f $cfg -lf/tmp/clixon.log -D1
+
+new "wait backend"
+wait_backend
+
+################################################################################
+
+new "ephemeral confirmed-commit survives unrelated ephemeral session disconnect"
+reset
+edit_config "candidate" "$CONFIGB"
+assert_config_equals "candidate" "$CONFIGB"
+# start a new ephemeral confirmed commit, but keep the confirmed-commit session alive (need to put it in the background)
+# use HELLONO11 which uses older EOM framing
+sleep 60 |  cat <(echo "$HELLONO11<rpc $DEFAULTNS><commit><confirmed/><confirm-timeout>60</confirm-timeout></commit></rpc>]]>]]>") -| $clixon_netconf -qf $cfg  >> /dev/null &
+PIDS=($(jobs -l % | cut -c 6- | awk '{print $1}'))
+assert_config_equals "running" "$CONFIGB"                       # assert config twice to prove it surives disconnect
+assert_config_equals "running" "$CONFIGB"                       # of ephemeral sessions
+
+new "soft kill ${PIDS[0]}"
+kill ${PIDS[0]}                   # kill the while loop above to close STDIN on 1st
+
+################################################################################
+
+new "cli ephemeral confirmed-commit rolls back after disconnect"
+reset
+
+tmppipe=$(mktemp -u)
+mkfifo -m 600 "$tmppipe"
+
+cat << EOF | clixon_cli -f $cfg >> /dev/null &
+set table parameter eth0
+commit confirmed 60
+shell echo >> $tmppipe
+shell cat $tmppipe
+quit
+EOF
+
+cat $tmppipe >> /dev/null
+assert_config_equals "running" "$CONFIGB"
+echo >> $tmppipe
+sleep 1
+assert_config_equals "running" ""
+rm $tmppipe
+
+################################################################################
+
+new "cli persistent confirmed-commit"
+reset
+
+cat << EOF | clixon_cli -f $cfg >> /dev/null
+set table parameter eth0
+commit confirmed persist a
+quit
+EOF
+
+assert_config_equals "running" "$CONFIGB"
+
+cat << EOF | clixon_cli -f $cfg >> /dev/null
+set table parameter eth1
+commit persist-id a confirmed persist ab
+quit
+EOF
+assert_config_equals "running" "$CONFIGBPLUSC"
+
+################################################################################
+
+new "cli cancel-commit with invalid persist-id"
+expectpart "$($clixon_cli -lo -1 -f $cfg commit persist-id abc cancel)" 255 "a confirmed-commit with the given persist-id was not found"
+
+################################################################################
+
+new "cli cancel-commit with valid persist-id"
+expectpart "$($clixon_cli -lo -1 -f $cfg commit persist-id ab cancel)" 0 "^$"
+assert_config_equals "running" ""
+
+################################################################################
+
+new "cli cancel-commit with no confirmed-commit in progress"
+expectpart "$($clixon_cli -lo -1 -f $cfg commit persist-id ab cancel)" 255 "no confirmed-commit is in progress"
+
+################################################################################
+
+new "cli persistent confirmed-commit with timeout"
+reset
+cat << EOF | clixon_cli -f $cfg >> /dev/null
+set table parameter eth0
+commit confirmed persist abcd 2
+EOF
+assert_config_equals "running" "$CONFIGB"
+sleep 2
+assert_config_equals "running" ""
+
+################################################################################
+
+new "cli persistent confirmed-commit with reset timeout"
+reset
+cat << EOF | clixon_cli -f $cfg >> /dev/null
+set table parameter eth0
+commit confirmed persist abcd 5
+EOF
+
+assert_config_equals "running" "$CONFIGB"
+cat << EOF | clixon_cli -f $cfg >> /dev/null
+set table parameter eth1
+commit persist-id abcd confirmed persist abcdef 10
+EOF
+
+sleep 6
+assert_config_equals "running" "$CONFIGBPLUSC"
+# now sleep long enough for rollback to happen; get config, assert == A
+sleep 5
+assert_config_equals "running" ""
+
+# TODO test restconf receives "409 conflict" when there is a persistent confirmed-commit active
+# TODO test restconf causes confirming-commit for ephemeral confirmed-commit
+if [ $RC -ne 0 ]; then
+    new "kill old restconf daemon"
+    stop_restconf_pre
+
+    new "start restconf daemon"
+    start_restconf -f $cfg
+fi
+
+new "wait restconf"
+wait_restconf
+
+new "restconf as confirmed commit"
+reset
+edit_config "candidate" "$CONFIGB"
+assert_config_equals "candidate" "$CONFIGB"
+# use HELLONO11 which uses older EOM framing
+sleep 60 |  cat <(echo "$HELLONO11<rpc $DEFAULTNS><commit><confirmed/><confirm-timeout>60</confirm-timeout></commit></rpc>]]>]]><rpc $DEFAULTNS><commit></commit></rpc>]]>]]>") -| $clixon_netconf -qf $cfg  >> /dev/null &
+PIDS=($(jobs -l % | cut -c 6- | awk '{print $1}'))
+assert_config_equals "running" "$CONFIGB"                       # assert config twice to prove it surives disconnect
+
+new "restconf POST"
+expectpart "$(curl $CURLOPTS -X POST -H "Content-Type: application/yang-data+xml" -d "$CONFIGCONLY" $RCPROTO://localhost/restconf/data/clixon-example:table)" 0 "HTTP/$HVER 201" "location:"
+
+assert_config_equals "running" "$CONFIGBPLUSC"
+
+new "soft kill ${PIDS[0]}"
+kill ${PIDS[0]}                   # kill the while loop above to close STDIN on 1st
+
+assert_config_equals "running" "$CONFIGBPLUSC"
+
+################################################################################
+
+new "restconf persistid expect fail"
+reset
+edit_config "candidate" "$CONFIGB"
+commit "<confirmed/><persist>a</persist>"
+assert_config_equals "running" "$CONFIGB"
+
+new "restconf POST"
+expectpart "$(curl $CURLOPTS -X POST -H "Content-Type: application/yang-data+xml" -d "$CONFIGCONLY" $RCPROTO://localhost/restconf/data/clixon-example:table)" 0 # "HTTP/$HVER 409"
+
+assert_config_equals "running" "$CONFIGB"
+
+if [ $RC -ne 0 ]; then
+    new "Kill restconf daemon"
+    stop_restconf 
+fi
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u ${USER} -f clixon_backend)
+    if [ -z "$pid" ]; then
+        err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+# Set by restconf_config
+unset RESTCONFIG
+
+new "endtest"
+endtest

test/test_identity.sh

@@ -197,7 +197,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_leafref_uses_augment.sh

@@ -114,7 +114,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 new "cli set t1-con t1-con t1-a 123"

test/test_nacm.sh

@@ -123,7 +123,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_nacm_credentials.sh

@@ -135,7 +135,7 @@ EOF
         start_backend -s init -f $cfg 
     fi
     
-    new "waiting"
+    new "wait backend"
     wait_backend
 
     # First push in nacm rules via regular means

test/test_nacm_datanode.sh

@@ -226,7 +226,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_nacm_datanode_paths.sh

@@ -100,7 +100,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s startup -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
     
 if [ $RC -ne 0 ]; then

test/test_nacm_datanode_read.sh

@@ -231,7 +231,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
     
 if [ $RC -ne 0 ]; then

test/test_nacm_datanode_write.sh

@@ -227,7 +227,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
     
 if [ $RC -ne 0 ]; then

test/test_nacm_ext.sh

@@ -191,7 +191,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -s
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_nacm_module_read.sh

@@ -184,7 +184,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -s
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_nacm_protocol.sh

@@ -145,7 +145,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_netconf.sh

@@ -19,6 +19,7 @@ cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_FEATURE>ietf-netconf:confirmed-commit</CLICON_FEATURE>
   <CLICON_MODULE_SET_ID>42</CLICON_MODULE_SET_ID>
   <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
@@ -356,6 +357,30 @@ expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS>
 new "kill-session using prefix xx"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<xx:rpc xmlns:xx=\"urn:ietf:params:xml:ns:netconf:base:1.0\" xx:message-id=\"42\"><xx:kill-session><xx:session-id>44</xx:session-id></xx:kill-session></xx:rpc>" "" "<rpc-reply xmlns=\"urn:ietf:params:xml:ns:netconf:base:1.0\" xmlns:xx=\"urn:ietf:params:xml:ns:netconf:base:1.0\" xx:message-id=\"42\"><ok/></rpc-reply>"
 
+new "asynchronous lock running"
+sleep 60 |  cat <(echo "$HELLONO11<rpc $DEFAULTNS><lock><target><running/></target></lock></rpc>]]>]]>") -| $clixon_netconf -qf $cfg  >> /dev/null &
+
+PIDS=($(jobs -l % | cut -c 6- | awk '{print $1}'))
+
+new "try commit should fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>protocol</error-type><error-tag>in-use</error-tag><error-severity>error</error-severity><error-message>Operation failed, lock is already held</error-message></rpc-error></rpc-reply>"
+
+new "soft kill ${PIDS[0]}"
+kill ${PIDS[0]}                   # kill the while loop above to close STDIN on 1st
+
+new "asynchronous confirmed commit"
+sleep 60 |  cat <(echo "$HELLONO11<rpc $DEFAULTNS><commit><confirmed/><confirm-timeout>60</confirm-timeout></commit></rpc>]]>]]>") -| $clixon_netconf -qf $cfg  >> /dev/null &
+PIDS=($(jobs -l % | cut -c 6- | awk '{print $1}'))
+
+new "try lock should fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><lock><target><running/></target></lock></rpc>" "<rpc-reply $DEFAULTNS><rpc-error><error-type>protocol</error-type><error-tag>lock-denied</error-tag><error-info><session-id>[0-9]*</session-id></error-info><error-severity>error</error-severity><error-message>Operation failed, another session has an ongoing confirmed commit</error-message></rpc-error></rpc-reply>"
+
+new "soft kill ${PIDS[0]}"
+kill ${PIDS[0]}                   # kill the while loop above to close STDIN on 1st
+
+new "netconf discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
 # modify candidate, then lock, should fail.
 new "netconf edit config"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><table xmlns=\"urn:example:clixon\"><parameter><name>a</name></parameter></table></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"

test/test_netconf_filter.sh

@@ -56,7 +56,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 new "Add two entries"

test/test_netconf_hello.sh

@@ -106,7 +106,15 @@ new "Netconf snd hello with prefix"
 expecteof "$clixon_netconf -qef $cfg" 0 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><nc:hello xmlns:nc=\"urn:ietf:params:xml:ns:netconf:base:1.0\"><nc:capabilities><nc:capability>urn:ietf:params:netconf:base:1.1</nc:capability></nc:capabilities></nc:hello>]]>]]>" '^$'
 
 new "netconf snd + rcv hello"
-expecteof "$clixon_netconf -f $cfg" 0 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability></capabilities></hello>]]>]]>" "^<hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability><capability>urn:ietf:params:netconf:base:1.0</capability><capability>urn:ietf:params:netconf:capability:yang-library:1.0?revision=2019-01-04&amp;module-set-id=42</capability><capability>urn:ietf:params:netconf:capability:candidate:1.0</capability><capability>urn:ietf:params:netconf:capability:validate:1.1</capability><capability>urn:ietf:params:netconf:capability:startup:1.0</capability><capability>urn:ietf:params:netconf:capability:xpath:1.0</capability><capability>urn:ietf:params:netconf:capability:notification:1.0</capability><capability>urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&amp;also-supported=report-all,trim,report-all-tagged</capability></capabilities><session-id>[0-9]*</session-id></hello>]]>]]>$" '^$'
+expecteof "$clixon_netconf -f $cfg" 0 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability></capabilities></hello>]]>]]>" "^<hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability><capability>urn:ietf:params:netconf:base:1.0</capability>
+<capability>urn:ietf:params:netconf:capability:yang-library:1.0?revision=2019-01-04&amp;module-set-id=42</capability>
+<capability>urn:ietf:params:netconf:capability:candidate:1.0</capability>
+<capability>urn:ietf:params:netconf:capability:validate:1.1</capability>
+<capability>urn:ietf:params:netconf:capability:xpath:1.0</capability>
+<capability>urn:ietf:params:netconf:capability:notification:1.0</capability>
+<capability>urn:ietf:params:netconf:capability:startup:1.0</capability>
+<capability>urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&amp;also-supported=report-all,trim,report-all-tagged</capability>
+</capabilities><session-id>[0-9]*</session-id></hello>]]>]]>$" '^$'
 
 # Actually non-standard to reply on wrong hello with rpc-error, but may be useful
 new "Netconf snd hello with extra element"

test/test_netconf_notifications.sh

@@ -101,7 +101,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -n # create example notification stream
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 #

test/test_netconf_ssh_callhome.sh

@@ -134,7 +134,7 @@ EOF
 new "Start Listener client"
 echo "ssh -s -F $sshcfg -v -i $key -o ProxyUseFdpass=yes -o ProxyCommand=\"clixon_netconf_ssh_callhome_client -a 127.0.0.1\" . netconf"
 #-F $sshcfg
-expectpart "$(ssh -s -F $sshcfg -v -i $key -o ProxyUseFdpass=yes -o ProxyCommand="${clixon_netconf_ssh_callhome_client} -a 127.0.0.1" . netconf < $rpccmd)" 0 "<hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability><capability>urn:ietf:params:netconf:base:1.0</capability><capability>urn:ietf:params:netconf:capability:yang-library:1.0?revision=2019-01-04&amp;module-set-id=42</capability><capability>urn:ietf:params:netconf:capability:candidate:1.0</capability><capability>urn:ietf:params:netconf:capability:validate:1.1</capability><capability>urn:ietf:params:netconf:capability:startup:1.0</capability><capability>urn:ietf:params:netconf:capability:xpath:1.0</capability><capability>urn:ietf:params:netconf:capability:notification:1.0</capability><capability>urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&amp;also-supported=report-all,trim,report-all-tagged</capability></capabilities><session-id>2</session-id></hello>" "<rpc-reply $DEFAULTNS><data/></rpc-reply>"
+expectpart "$(ssh -s -F $sshcfg -v -i $key -o ProxyUseFdpass=yes -o ProxyCommand="${clixon_netconf_ssh_callhome_client} -a 127.0.0.1" . netconf < $rpccmd)" 0 "<hello $DEFAULTONLY><capabilities><capability>urn:ietf:params:netconf:base:1.1</capability><capability>urn:ietf:params:netconf:base:1.0</capability><capability>urn:ietf:params:netconf:capability:yang-library:1.0?revision=2019-01-04&amp;module-set-id=42</capability><capability>urn:ietf:params:netconf:capability:candidate:1.0</capability><capability>urn:ietf:params:netconf:capability:validate:1.1</capability><capability>urn:ietf:params:netconf:capability:xpath:1.0</capability><capability>urn:ietf:params:netconf:capability:with-defaults:1.0?basic-mode=explicit&amp;also-supported=report-all,trim,report-all-tagged</capability><capability>urn:ietf:params:netconf:capability:notification:1.0</capability></capabilities><session-id>2</session-id></hello>" "<rpc-reply $DEFAULTNS><data/></rpc-reply>"
 
 # Wait 
 wait

test/test_netconf_whitespace.sh

@@ -73,7 +73,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s startup -f $cfg 
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 new "get startup"

test/test_perf_cli.sh

@@ -83,7 +83,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -s
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 new "generate config with $perfnr list entries"

test/test_perf_mem.sh

@@ -85,7 +85,7 @@ function testrun(){
         start_backend -s startup -f $cfg
     fi
 
-    new "waiting"
+    new "wait backend"
     wait_backend
 
     pid=$(cat $pidfile)

test/test_perf_netconf.sh

@@ -87,7 +87,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -s
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 # Check this later with committed data

test/test_perf_restconf.sh

@@ -106,7 +106,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -s
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_perf_restconf_ssl.sh

@@ -122,7 +122,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -s
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_privileges.sh

@@ -96,7 +96,7 @@ function testrun(){
         err 1 $c
     fi
 
-    new "waiting"
+    new "wait backend"
     wait_backend
 
     if [ $expecterr -eq 1 ]; then

test/test_restconf_notifications.sh

@@ -149,7 +149,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg -- -n # create example notification stream
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_rpc.sh

@@ -196,7 +196,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s init -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 if [ $RC -ne 0 ]; then

test/test_snmp_entity.sh

@@ -136,6 +136,17 @@ function testinit(){
 
 function testexit(){
     stop_snmp
+
+    if [ $BE -ne 0 ]; then
+        new "Kill backend"
+        # Check if premature kill
+        pid=$(pgrep -u root -f clixon_backend)
+        if [ -z "$pid" ]; then
+            err "backend already dead"
+        fi
+        # kill backend
+        stop_backend -f $cfg
+    fi
 }
 
 ENTITY_OID=".1.3.6.1.2.1.47.1.1.1"

test/test_snmp_get.sh

@@ -132,6 +132,16 @@ function testinit(){
 
 function testexit(){
     stop_snmp
+    if [ $BE -ne 0 ]; then
+        new "Kill backend"
+        # Check if premature kill
+        pid=$(pgrep -u root -f clixon_backend)
+        if [ -z "$pid" ]; then
+            err "backend already dead"
+        fi
+        # kill backend
+        stop_backend -f $cfg
+    fi
 }
 
 new "SNMP tests"
@@ -280,6 +290,9 @@ expectpart "$($snmpget $OID19)" 0 "$OID19 = Hex-STRING: 74 65 73 74 00"
 new "Test SNMP getnext netSnmpHostName"
 expectpart "$($snmpgetnext $OID19)" 0 "$OID20 = INTEGER: 1"
 
+new "Negative test: Try to set object"
+expectpart "$($snmpset $OID1 i 4 2> /dev/null)" 2 "^$"
+
 new "Cleaning up"
 testexit
 

test/test_snmp_ifmib.sh

@@ -195,13 +195,21 @@ function testinit(){
 
 function testexit(){
     stop_snmp
+    if [ $BE -ne 0 ]; then
+        new "Kill backend"
+        # Check if premature kill
+        pid=$(pgrep -u root -f clixon_backend)
+        if [ -z "$pid" ]; then
+            err "backend already dead"
+        fi
+        # kill backend
+        stop_backend -f $cfg
+    fi
 }
 
-
 new "SNMP tests"
 testinit
 
-
 # IF-MIB::interfaces
 MIB=".1.3.6.1.2.1"
 for (( i=1; i<23; i++ )); do
@@ -370,8 +378,8 @@ expectpart "$($snmptable IF-MIB::ifTable)" 0 "Test 2" "1400" "1000" "11:22:33:44
 new "Walk the walk..."
 expectpart "$($snmpwalk IF-MIB::ifTable)" 0 "IF-MIB::ifIndex.1 = INTEGER: 1" \
            "IF-MIB::ifIndex.2 = INTEGER: 2" \
-           "IF-MIB::ifDescr.1 = STRING: Test." \
+           "IF-MIB::ifDescr.1 = STRING: Test" \
-           "IF-MIB::ifDescr.2 = STRING: Test 2." \
+           "IF-MIB::ifDescr.2 = STRING: Test 2" \
            "IF-MIB::ifType.1 = INTEGER: ethernetCsmacd(6)" \
            "IF-MIB::ifType.2 = INTEGER: ethernetCsmacd(6)" \
            "IF-MIB::ifMtu.1 = INTEGER: 1500" \

test/test_snmp_rowstatus.sh

@@ -206,6 +206,16 @@ function testrun_removeRows()
 function testexit()
 {
     stop_snmp
+    if [ $BE -ne 0 ]; then
+        new "Kill backend"
+        # Check if premature kill
+        pid=$(pgrep -u root -f clixon_backend)
+        if [ -z "$pid" ]; then
+            err "backend already dead"
+        fi
+        # kill backend
+        stop_backend -f $cfg
+    fi
 }
 
 new "SNMP tests"

test/test_snmp_set.sh

@@ -2,6 +2,7 @@
 # snmpset. This requires deviation of MIB-YANG to make write operations
 # Get default value, set new value via SNMP and check it, set new value via NETCONF and check
 # Selected types from CLIXON/IF-MIB/ENTITY mib
+# Also an incomplete commit failed test
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@@ -33,6 +34,7 @@ cat <<EOF > $cfg
   <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
   <CLICON_SOCK>$dir/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/var/tmp/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_SNMP_AGENT_SOCK>unix:$SOCK</CLICON_SNMP_AGENT_SOCK>
   <CLICON_SNMP_MIB>CLIXON-TYPES-MIB</CLICON_SNMP_MIB>
@@ -195,9 +197,10 @@ function testrun()
         echo "$snmpset $oid $set_type $value"
         expectpart "$($snmpset $oid $set_type $value)" 0 "$type:" "$value"
     else
-	echo "$snmpset $oid $set_type $value"
+        echo "$snmpset $oid $set_type $value2"
         expectpart "$($snmpset $oid $set_type $value)" 0 "$type: $value2"
     fi
+
     new "Check $name via SNMP"
     if [ "$type" == "STRING" ]; then
         expectpart "$($snmpget $oid)" 0 "$type:" "$value"
@@ -211,6 +214,17 @@ function testrun()
 
 function testexit(){
     stop_snmp
+
+    if [ $BE -ne 0 ]; then
+        new "Kill backend"
+        # Check if premature kill
+        pid=$(pgrep -u root -f clixon_backend)
+        if [ -z "$pid" ]; then
+            err "backend already dead"
+        fi
+        # kill backend
+        stop_backend -f $cfg
+    fi
 }
 
 new "SNMP tests"
@@ -225,7 +239,8 @@ testrun clixonExampleSleeper INTEGER -1 -1 -1 ${MIB}.1.2
 testrun clixonExampleString STRING foobar foobar foobar ${MIB}.1.3
 testrun ifPromiscuousMode INTEGER 1 1 true ${MIB}.1.10 # boolean
 testrun ifIpAddr IPADDRESS 1.2.3.4 1.2.3.4 1.2.3.4 ${MIB}.1.13 # InetAddress
-testrun ifPhysAddress STRING ff:ee:dd:cc:bb:aa ff:ee:dd:cc:bb:aa ff:ee:dd:cc:bb:aa ${IFMIB}.2.2.1.6.1
+# XXX It was supposed to test writing hardware address type, but it is also read-only
+#testrun ifPhysAddress STRING ff:ee:dd:cc:bb:aa ff:ee:dd:cc:bb:aa ff:ee:dd:cc:bb:aa ${IFMIB}.2.2.1.6.1
 
 # Inline testrun for rowstatus complicated logic
 name=ifStackStatus
@@ -241,6 +256,28 @@ expectpart "$($snmpget $oid)" 0 "$type: active(1)"
 new "Check $name via CLI"
 expectpart "$($clixon_cli -1 -f $cfg show config xml)" 0 "<$name>active</$name>"    
 
+# restart backend with synthetic validation failure
+# Incomplete commit failed test: the commit fails by logging but this is not actually checked
+if [ $BE -ne 0 ]; then
+    # Kill old backend and start a new one
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+        err "Failed to start backend"
+    fi
+    
+    sudo pkill -f clixon_backend
+    
+    new "Starting backend"
+    start_backend -s startup -f $cfg -- -V CLIXON-TYPES-MIB/clixonExampleScalars/clixonExampleInteger
+fi
+
+new "wait backend"
+wait_backend
+
+new "set value with error"
+expectpart "$($snmpset ${MIB}.1.1  i 4321 2>&1)" 2 "commitFailed"
+
 new "Cleaning up"
 testexit
 

test/test_snmp_system.sh

@@ -112,6 +112,8 @@ function testinit(){
         sudo killall -q clixon_snmp
         
         new "Starting clixon_snmp"
+        # XXX augmented objects seem to be registered twice: error: duplicate registration: MIB modules snmpSetSerialNo and AgentX subagent 52, session 0x562087a70e20, subsession 0x562087a820c0 (oid .1.3.6.1.6.3.1.1.6.1).
+
         start_snmp $cfg &
     fi
 
@@ -121,6 +123,16 @@ function testinit(){
 
 function testexit(){
     stop_snmp
+    if [ $BE -ne 0 ]; then
+        new "Kill backend"
+        # Check if premature kill
+        pid=$(pgrep -u root -f clixon_backend)
+        if [ -z "$pid" ]; then
+            err "backend already dead"
+        fi
+        # kill backend
+        stop_backend -f $cfg
+    fi
 }
 
 new "SNMP tests"
@@ -217,19 +229,19 @@ expectpart "$($snmptable $OID_ORTABLE)" 0 ".*Entry 2 description.*" "IF-MIB::ifT
 expectpart "$($snmptable $NAME_ORTABLE)" 0 ".*Entry 1 description.*" "IP-MIB::ip" "1:7:10:33.44"
 expectpart "$($snmptable $NAME_ORTABLE)" 0 ".*Entry 2 description.*" "IF-MIB::ifTable" "129:20:58:31.11"
 
-new "Walk the tabbles..."
+new "Walk the tables..."
-expectpart "$($snmpwalkstr system)" 0 "SNMPv2-MIB::sysDescr = STRING: System description." \
+expectpart "$($snmpwalkstr system)" 0 "SNMPv2-MIB::sysDescr = STRING: System description" \
     "SNMPv2-MIB::sysUpTime = Timeticks: (11223344) 1 day, 7:10:33.44" \
-    "SNMPv2-MIB::sysContact = STRING: clixon@clicon.com." \
+    "SNMPv2-MIB::sysContact = STRING: clixon@clicon.com" \
-    "SNMPv2-MIB::sysName = STRING: Test." \
+    "SNMPv2-MIB::sysName = STRING: Test" \
-    "SNMPv2-MIB::sysLocation = STRING: Clixon HQ." \
+    "SNMPv2-MIB::sysLocation = STRING: Clixon HQ" \
     "SNMPv2-MIB::sysServices = INTEGER: 72" \
     "SNMPv2-MIB::sysORIndex.1 = INTEGER: 1" \
     "SNMPv2-MIB::sysORIndex.2 = INTEGER: 2" \
     "SNMPv2-MIB::sysORID.1 = OID: IP-MIB::ip" \
     "SNMPv2-MIB::sysORID.2 = OID: IF-MIB::ifTable" \
-    "SNMPv2-MIB::sysORDescr.1 = STRING: Entry 1 description." \
+    "SNMPv2-MIB::sysORDescr.1 = STRING: Entry 1 description" \
-    "SNMPv2-MIB::sysORDescr.2 = STRING: Entry 2 description." \
+    "SNMPv2-MIB::sysORDescr.2 = STRING: Entry 2 description" \
     "SNMPv2-MIB::sysORUpTime.1 = Timeticks: (11223344) 1 day, 7:10:33.44" \
     "SNMPv2-MIB::sysORUpTime.2 = Timeticks: (1122111111) 129 days, 20:58:31.11"
 

test/test_startup.sh

@@ -124,7 +124,7 @@ function testrun(){
         new "start backend -f $cfg -s $mode -c $dir/extra_db"
         start_backend -s $mode -f $cfg -c $dir/extra_db
 
-	new "waiting"
+        new "wait backend"
         wait_backend
     else
         new "Restart backend as eg follows: -Ff $cfg -s $mode -c $dir/extra_db # $BETIMEOUT s"

test/test_upgrade_auto.sh

@@ -258,7 +258,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s $mode -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 new "Check running db content"

test/test_upgrade_interfaces.sh

@@ -271,7 +271,7 @@ function testrun(){
         start_backend -s startup -f $cfg -- -u
     fi
 
-    new "waiting"
+    new "wait backend"
     wait_backend
     
     new "Check running db content"

test/test_upgrade_interfaces_rfc7895.sh

@@ -268,7 +268,7 @@ function testrun(){
         start_backend -s startup -f $cfg -- -u
     fi
 
-    new "waiting"
+    new "wait backend"
     wait_backend
     
     new "Check running db content"

test/test_upgrade_repair.sh

@@ -117,7 +117,7 @@ if [ $BE -ne 0 ]; then
     start_backend -s $mode -f $cfg
 fi
 
-new "waiting"
+new "wait backend"
 wait_backend
 
 new "Check running db content is failsafe"

test/test_upgrade_simple.sh

@@ -111,11 +111,11 @@ EOF
 if [ $BE -ne 0 ]; then
     new "start backend -s running -f $cfg"
     start_backend -s running -f $cfg
-
-    new "waiting"
-    wait_backend
 fi
 
+new "wait backend"
+wait_backend
+
 new "netconf get config"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><candidate/></source></get-config></rpc>" "" "<rpc-reply $DEFAULTNS><data><hello xmlns=\"urn:example:simple\"><world/></hello></data></rpc-reply>"
 

test/test_when_mandatory.sh

@@ -0,0 +1,309 @@
+#!/usr/bin/env bash
+# Yang when conditional and mandatory
+# Testing of validation phase.
+# Variants:
+# - One level and two levels of non-presence containers
+# - when-condition true and false
+# - empty vs extra leaf in non-presence container 
+# - mandatory leaf present or not
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyang=$dir/test.yang
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+</clixon-config>
+EOF
+
+cat <<EOF > $fyang
+module $APPNAME{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
+  prefix ex;
+  identity routing-protocol {
+     description
+        "Base identity from which routing protocol identities are
+         derived.";
+  }
+  identity direct {
+       base routing-protocol;
+       description
+         "Routing pseudo-protocol which provides routes to directly
+          connected networks.";
+  }
+  identity static {
+       base routing-protocol;
+       description
+         "Static routing pseudo-protocol.";
+  }
+  list x {
+     key "type";
+     leaf type {
+        type identityref {
+           base routing-protocol;
+        }
+     }
+     container y {
+         when "../type='static'" {
+            description
+               "This container is only valid for the 'static'
+                routing protocol.";
+         }
+         leaf name{
+           type string;
+           mandatory true;
+         }
+         leaf extra{
+           type string;
+         }
+     }
+  }
+  list x2 {
+     key "type";
+     leaf type {
+        type identityref {
+           base routing-protocol;
+        }
+     }
+     container y2 {
+         when "../type='static'" {
+            description
+               "This container is only valid for the 'static'
+                routing protocol.";
+         }
+         container y3 {
+           leaf name{
+             type string;
+             mandatory true;
+           }
+           leaf extra{
+             type string;
+           }
+        }
+     }
+  }
+  list x3 {
+     key "type";
+     leaf type {
+        type identityref {
+           base routing-protocol;
+        }
+     }
+     container y2 {
+         container y3 {
+           when "../../type='static'" {
+              description
+                 "This container is only valid for the 'static'
+                  routing protocol.";
+           }
+           leaf name{
+             type string;
+             mandatory true;
+           }
+           leaf extra{
+             type string;
+           }
+        }
+     }
+  }
+}
+EOF
+
+new "test params: -f $cfg"
+
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+        err
+    fi
+    new "start backend -s init -f $cfg"
+    start_backend -s init -f $cfg
+fi
+
+new "wait backend"
+wait_backend
+
+new "First: have mandatory leaf under an empty when-conditioned ccntainer"
+
+new "when false + no name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><type>direct</type><y></y></x></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when true + no name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><type>static</type><y></y></x></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>y</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable of x in module example</error-message></rpc-error></rpc-reply>"
+
+new "when true + name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><type>static</type><y><name>a</name></y></x></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Second: have mandatory leaf under a non-empty when-conditioned ccntainer"
+
+new "when false + no name + extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><type>direct</type><y><extra>b</extra></y></x></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>operation-failed</error-tag><error-severity>error</error-severity><error-message>Failed WHEN condition of y in module example (WHEN xpath is ../type='static')</error-message></rpc-error></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when true + no name + extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><type>static</type><y><extra>b</extra></y></x></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>name</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable of y in module example</error-message></rpc-error></rpc-reply>"
+
+new "when true + name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><type>static</type><y><name>a</name><extra>b</extra></y></x></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Third: have mandatory leaf under two empty when-conditioned ccntainer"
+new "when false + no name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x2 xmlns=\"urn:example:clixon\"><type>direct</type><y2><y3/></y2></x2></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when true + no name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x2 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3/></y2></x2></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>y2</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable of x2 in module example</error-message></rpc-error></rpc-reply>"
+
+new "when true + name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x2 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3><name>a</name></y3></y2></x2></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Fourth: have mandatory leaf under two non-empty when-conditioned ccntainer"
+
+new "when false + no name + extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x2 xmlns=\"urn:example:clixon\"><type>direct</type><y2><y3><extra>b</extra></y3></y2></x2></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>operation-failed</error-tag><error-severity>error</error-severity><error-message>Failed WHEN condition of y2 in module example (WHEN xpath is ../type='static')</error-message></rpc-error></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when true + no name + extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x2 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3><extra>b</extra></y3></y2></x2></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>name</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable of y3 in module example</error-message></rpc-error></rpc-reply>"
+
+new "when true + name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x2 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3><name>a</name><extra>b</extra></y3></y2></x2></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Fifth: have mandatory leaf under one empty and one when-conditioned ccntainer"
+new "when false + no name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x3 xmlns=\"urn:example:clixon\"><type>direct</type><y2><y3/></y2></x3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when true + no name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x3 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3/></y2></x3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>y2</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable of x3 in module example</error-message></rpc-error></rpc-reply>"
+
+new "when true + name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x3 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3><name>a</name></y3></y2></x3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Sixth: have mandatory leaf under one non-empty and one when-conditioned ccntainer"
+
+new "when false + no name + extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x3 xmlns=\"urn:example:clixon\"><type>direct</type><y2><y3><extra>b</extra></y3></y2></x3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>operation-failed</error-tag><error-severity>error</error-severity><error-message>Failed WHEN condition of y3 in module example (WHEN xpath is ../../type='static')</error-message></rpc-error></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when true + no name + extra"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x3 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3><extra>b</extra></y3></y2></x3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate fail"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><rpc-error><error-type>application</error-type><error-tag>missing-element</error-tag><error-info><bad-element>name</bad-element></error-info><error-severity>error</error-severity><error-message>Mandatory variable of y3 in module example</error-message></rpc-error></rpc-reply>"
+
+new "when true + name"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><x3 xmlns=\"urn:example:clixon\"><type>static</type><y2><y3><name>a</name><extra>b</extra></y3></y2></x3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "validate ok"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "when: discard-changes"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><discard-changes/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u root -f clixon_backend)
+    if [ -z "$pid" ]; then
+        err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+rm -rf $dir
+
+new "endtest"
+endtest

test/valgrind-clixon.supp

@@ -42,3 +42,12 @@
    fun:*
    fun:OS_LibInit
 }
+{
+   supp7
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:*
+   fun:*
+   fun:*
+   fun:_dl_map_object
+}

util/clixon_util_validate.c

@@ -227,7 +227,7 @@ main(int    argc,
         goto done;
     }
     if (commit){
-	if ((ret = candidate_commit(h, database, cb)) < 0)
+        if ((ret = candidate_commit(h, NULL, database, cb)) < 0)
             goto done;
     }
     else{

yang/clixon/clixon-config@2022-03-21.yang

@@ -862,6 +862,8 @@ module clixon-config {
             description
                 "Set if all configuration changes are committed automatically 
                  on every edit change. Explicit commit commands unnecessary
+                 If confirm-commit, follow RESTCONF semantics: commit ephemeral but fail on
+                 persistent confirming commit.
                  (consider boolean)";
         }
         leaf CLICON_XMLDB_DIR {