Yang "unique" feature supported according to RFC 7950 7.8.3

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 3.10.0/4.0.0 (Upcoming)
 
 ### Major New features
+* Yang "unique" feature supported
+  * See RFC 7950 7.8.3
 * Persistent CLI history: [Preserve CLI command history across sessions. The up/down arrows](https://github.com/clicon/clixon/issues/79)
   * The design is similar to bash history:
       * The CLI loads/saves its complete history to a file on entry and exit, respectively

README.md

@@ -112,7 +112,6 @@ Clixon follows:
 However, the following YANG syntax modules are not implemented:
 - deviation
 - min/max-elements
-- unique
 - action
 - refine
 - Yang extended Xpath functions: re-match, deref, derived-from, derived-from-or-self, enum-value, bit-is-set

lib/clixon/clixon_netconf_lib.h

@@ -66,6 +66,7 @@ int netconf_operation_failed(cbuf *cb, char *type, char *message);
 int netconf_operation_failed_xml(cxobj **xret, char *type, char *message);
 int netconf_malformed_message(cbuf *cb, char *message);
 int netconf_malformed_message_xml(cxobj **xret, char *message);
+int netconf_data_not_unique(cbuf *cb, cxobj *x,	cvec *cvk);
 int netconf_trymerge(cxobj *x, yang_stmt *yspec, cxobj **xret);
 int netconf_module_load(clicon_handle h);
 char *netconf_db_find(cxobj *xn, char *name);

lib/src/clixon_netconf_lib.c

@@ -966,6 +966,48 @@ netconf_malformed_message_xml(cxobj **xret,
     return retval;
 }
 
+/*! Create Netconf data-not-unique error message according to RFC 7950 15.1
+ *
+ * A NETCONF operation would result in configuration data where a
+ *   "unique" constraint is invalidated.
+ * @param[out]  cb       CLIgen buf. Error XML is written in this buffer
+ * @param[in]   x        List element containing duplicate
+ * @param[in]   cvk      List of comonents in x that are non-unique
+ * @see RFC7950 Sec 15.1
+ */
+int
+netconf_data_not_unique(cbuf *cb,
+			cxobj *x,
+			cvec  *cvk)
+{
+    int     retval = -1;
+    cg_var *cvi = NULL; 
+    cxobj  *xi;
+    
+    if (cprintf(cb, "<rpc-reply><rpc-error>"
+		"<error-type>protocol</error-type>"
+		"<error-tag>operation-failed</error-tag>"
+		"<error-app-tag>data-not-unique</error-app-tag>"
+		"<error-severity>error</error-severity>"
+		"<error-info>") < 0)
+	goto err;
+    while ((cvi = cvec_each(cvk, cvi)) != NULL){
+	if ((xi = xml_find(x, cv_string_get(cvi))) == NULL)
+	    continue; /* ignore, shouldnt happen */
+	cprintf(cb, "<non-unique>");
+	clicon_xml2cbuf(cb, xi, 0, 0);	
+	cprintf(cb, "</non-unique>");
+    }
+    if (cprintf(cb, "</error-info></rpc-error></rpc-reply>") <0)
+	goto err;
+    retval = 0;
+ done:
+    return retval;
+ err:
+    clicon_err(OE_XML, errno, "cprintf");
+    goto done;
+}
+
 /*! Help function: merge - check yang - if error make netconf errmsg 
  * @param[in]     x       XML tree
  * @param[in]     yspec   Yang spec
@@ -1006,6 +1048,7 @@ netconf_trymerge(cxobj       *x,
     return retval;
 }
 
+
 /*! Load ietf netconf yang module and set enabled features
  * The features added are (in order):
  *   candidate (8.3)

lib/src/clixon_xml_map.c

@@ -633,6 +633,167 @@ check_mandatory(cxobj     *xt,
     goto done;
 }
 
+/*! New elemnt last in list, check if already exists if sp return -1
+ * @param[in]  vec   Vector of existing entries (new is last)
+ * @param[in]  i1    The new entry is placed at vec[i1]
+ * @param[in]  vlen  Lenght of entry
+ * @retval     0     OK, entry is unique
+ * @retval    -1     Duplicate detected
+ * @note This is currently linear complexity. It could be improved by inserting new element sorted and binary search.
+ */
+static int
+check_insert_duplicate(char **vec,
+		       int    i1,
+		       int    vlen)
+{
+    int i;
+    int v;
+    char *b;
+    
+    for (i=0; i<i1; i++){
+	for (v=0; v<vlen; v++){
+	    b = vec[i*vlen+v];
+	    if (b == NULL || strcmp(b, vec[i1*vlen+v]))
+		break;
+	}
+	if (v==vlen) /* duplicate */
+	    break;
+    }
+    return i==i1?0:-1;
+}
+
+/*! Given a list with unique constraint, detect duplicates
+ * @param[in]  x     The first element in the list (on return the last)
+ * @param[in]  xt    The parent of x
+ * @param[in]  y     Its yang spec (Y_LIST)
+ * @param[in]  yu    A yang unique spec (Y_UNIQUE)
+ * @param[out] cbret Error buffer (set w netconf error if retval == 0)
+ * @retval     1     Validation OK
+ * @retval     0     Validation failed (cbret set)
+ * @retval    -1     Error
+ * @note It would be possible to cache the vector built below
+ */
+static int
+check_unique_list(cxobj     *x, 
+		  cxobj     *xt, 
+		  yang_stmt *y,
+		  yang_stmt *yu,
+		  cbuf      *cbret)
+{
+    int       retval = -1;
+    cvec      *cvk; /* unique vector */
+    cg_var    *cvi; /* unique node name */
+    cxobj     *xi;
+    char     **vec = NULL; /* 2xmatrix */
+    int        vlen;
+    int        i;
+    int        v;
+    char      *bi;
+    
+    cvk = yang_cvec_get(yu);
+    vlen = cvec_len(cvk); /* nr of unique elements to check */
+    if ((vec = calloc(vlen*xml_child_nr(xt), sizeof(char*))) == NULL){
+	clicon_err(OE_UNIX, errno, "calloc");
+	goto done;
+    }
+    i = 0; /* x element index */
+    do {
+	cvi = NULL;
+	v = 0; /* index in each tuple */
+	while ((cvi = cvec_each(cvk, cvi)) != NULL){
+	    /* RFC7950: Sec 7.8.3.1: entries that do not have value for all
+	     * referenced leafs are not taken into account */
+	    if ((xi = xml_find(x, cv_string_get(cvi))) ==NULL)
+		break;
+	    if ((bi = xml_body(xi)) == NULL)
+		break;
+	    vec[i*vlen + v++] = bi;
+	}
+	if (cvi==NULL){
+	    /* Last element (i) is newly inserted, see if it is already there */
+	    if (check_insert_duplicate(vec, i, vlen) < 0){
+		if (netconf_data_not_unique(cbret, x, cvk) < 0)
+		    goto done;
+		goto fail;
+	    }
+	}
+	x = xml_child_each(xt, x, CX_ELMNT);
+	i++;
+    } while (x && y == xml_spec(x));  /* stop if list ends, others may follow */
+    /* It would be possible to cache vec here as an optimization */
+    retval = 1;
+ done:
+    if (vec)
+	free(vec);
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
+/*! Detect unique constraint for duplicates from parent node
+ * @param[in]  xt    XML parent (may have lists w unique constraints as child)
+ * @param[out] cbret Error buffer (set w netconf error if retval == 0)
+ * @retval     1     Validation OK
+ * @retval     0     Validation failed (cbret set)
+ * @retval    -1     Error
+ * Assume xt:s children are sorted and yang populated.
+ * The routine finds the lists: ie xt may have several lists in the children, 
+ * example
+ * shows two lists [x1,..] and [x2,..]. 
+ *   xt:  {a, b, [x1, x1, x1], d, e, f, [x2, x2, x2], g}
+ * Then call check_unique_list on each list.
+ * The function does this using a single iteration and uses the fact that the
+ * xml symbols share yang symbols: ie [x1..] has yang y1 and d has yd.
+ * Example, x has an associated yang list node with list of unique constraints
+ *         y-list->y-unique - "a"
+ *      xt->x ->  ab
+ *          x ->  bc
+ *          x ->  ab
+ */
+static int
+check_unique_parent(cxobj *xt,
+		    cbuf  *cbret)
+{
+    int         retval = -1;
+    cxobj      *x = NULL;
+    yang_stmt  *y;
+    yang_stmt  *yp = NULL; /* previous in list */
+    yang_stmt  *yu;
+    int         ret;
+	    
+    /* */
+    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
+	if ((y = xml_spec(x)) == NULL)
+	    continue;
+	if (y == yp) /* If same yang as previous x, then skip (eg same list) */
+	    continue;
+	yp = y;
+	if (yang_keyword_get(y) != Y_LIST)
+	    continue;
+	yu = NULL;
+	while ((yu = yn_each(y, yu)) != NULL) {
+	    if (yang_keyword_get(yu) != Y_UNIQUE)
+		continue;
+	    /* Here is a list w unique constraints identified by:
+	     * its first element x, its yang spec y, its parent xt, and 
+	     * a unique yang spec yu,
+	     */
+	    if ((ret = check_unique_list(x, xt, y, yu, cbret)) < 0)
+		goto done;
+	    if (ret == 0)
+		goto fail;
+	}
+    }
+    retval = 1;
+ done:
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
+
 /*! Validate a single XML node with yang specification for added entry
  * 1. Check if mandatory leafs present as subs.
  * 2. Check leaf values, eg int ranges and string regexps.
@@ -831,6 +992,10 @@ xml_yang_validate_all(cxobj   *xt,
 	default:
 	    break;
 	}
+	if ((ret = check_unique_parent(xt, cbret)) < 0)
+	    goto done;
+	if (ret == 0)
+	    goto fail;
 	/* must sub-node RFC 7950 Sec 7.5.3. Can be several. 
 	* XXX. use yang path instead? */
 	yc = NULL;

lib/src/clixon_yang.c

@@ -1213,7 +1213,6 @@ yang_print_cbuf(cbuf      *cb,
     return 0;
 }
 
-
 /*! Populate yang leafs after parsing. Create cv and fill it in.
  *
  * Populate leaf in 2nd round of yang parsing, now that context is complete:
@@ -1308,6 +1307,7 @@ ys_populate_list(yang_stmt *ys,
     
     if ((ykey = yang_find(ys, Y_KEY, NULL)) == NULL)
 	return 0;
+    if (ys->ys_cvec)
 	cvec_free(ys->ys_cvec);
     if ((ys->ys_cvec = yang_arg2cvec(ykey, " ")) == NULL)
 	return -1;
@@ -1681,6 +1681,18 @@ ys_populate_feature(clicon_handle h,
     return retval;
 }
 
+/*! Populate the unique statement with a cvec
+ */
+static int
+ys_populate_unique(yang_stmt *ys)
+{
+    if (ys->ys_cvec)
+	cvec_free(ys->ys_cvec);
+    if ((ys->ys_cvec = yang_arg2cvec(ys, " ")) == NULL)
+	return -1;
+    return 0;
+}
+
 /*! Populate unknown node with extension
  */
 static int
@@ -1770,6 +1782,10 @@ ys_populate(yang_stmt *ys,
 	if (ys_populate_identity(ys, NULL) < 0)
 	    goto done;
 	break;
+    case Y_UNIQUE:
+	if (ys_populate_unique(ys) < 0)
+	    goto done;
+	break;
     case Y_UNKNOWN:
 	if (ys_populate_unknown(ys) < 0)
 	    goto done;

test/test_unique.sh

@@ -0,0 +1,226 @@
+#!/bin/bash
+# Yang list unique tests
+# Use example in RFC7890 7.8.3.1, modify fields and also test a variant with
+# a single unique identifier (rfc example has two)
+# The test adds the rfc conf that fails, then one that passes, then makes add
+# to fail it and then del to pass it.
+# Then makes a fail / pass test on the single field case
+# Last, a complex unsorted list with several sub-elements.
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyang=$dir/unique.yang
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+</clixon-config>
+EOF
+
+# Example (the list server part) from RFC7950 Sec 7.8.3.1 w changed types
+cat <<EOF > $fyang
+module unique{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
+  prefix un;
+  container c{
+     leaf a{
+       type string;
+     }
+     list server {
+       description "RFC7950 7.8.3.1";
+       key "name";
+       unique "ip port";
+       leaf name {
+         type string;
+       }
+       leaf ip {
+         type string;
+       }
+       leaf port {
+         type uint16;
+       }
+     }
+     list other {
+       description "random inserted data";
+       key a;
+       leaf a{
+         type string;
+       }
+     }
+     list single {
+       description "similar with just a single unique field";
+       key "name";
+       unique "ip";
+       leaf name {
+         type string;
+       }
+       leaf ip {
+         type string;
+       }
+     }
+     leaf b{
+       type string;
+     }
+  }
+}
+EOF
+
+new "test params: -f $cfg"
+
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+    new "start backend -s init -f $cfg"
+    # start new backend
+    start_backend -s init -f $cfg
+
+    new "waiting"
+    sleep $RCWAIT
+fi
+
+# RFC test two-field caes
+new "Add not valid example"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><c xmlns="urn:example:clixon"><server>
+       <name>smtp</name>
+       <ip>192.0.2.1</ip>
+       <port>25</port>
+     </server>
+     <server>
+       <name>http</name>
+       <ip>192.0.2.1</ip>
+       <port>25</port>
+     </server>
+</c></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf validate (should fail)"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" '^<rpc-reply><rpc-error><error-type>protocol</error-type><error-tag>operation-failed</error-tag><error-app-tag>data-not-unique</error-app-tag><error-severity>error</error-severity><error-info><non-unique><ip>192.0.2.1</ip></non-unique><non-unique><port>25</port></non-unique></error-info></rpc-error></rpc-reply>]]>]]>$'
+
+new "netconf discard-changes"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "Add valid example"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><c xmlns="urn:example:clixon"><server>
+       <name>smtp</name>
+       <ip>192.0.2.1</ip>
+       <port>25</port>
+     </server>
+     <server>
+       <name>http</name>
+       <ip>192.0.2.1</ip>
+     </server>
+     <server>
+       <name>ftp</name>
+       <ip>192.0.2.1</ip>
+     </server>
+</c></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf validate ok"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "make it invalid by adding port to ftp entry"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>none</default-operation><config><c xmlns="urn:example:clixon"><server><name>ftp</name><port operation="create">25</port></server>
+</c></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf validate (should fail)"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" '^<rpc-reply><rpc-error><error-type>protocol</error-type><error-tag>operation-failed</error-tag><error-app-tag>data-not-unique</error-app-tag><error-severity>error</error-severity><error-info><non-unique><ip>192.0.2.1</ip></non-unique><non-unique><port>25</port></non-unique></error-info></rpc-error></rpc-reply>]]>]]>$'
+
+new "make it valid by deleting port from smtp entry"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>none</default-operation><config><c xmlns="urn:example:clixon"><server><name>smtp</name><port operation="delete">25</port></server>
+</c></config></edit-config></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
+
+new "netconf validate ok"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf discard-changes"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+# Then test single-field case
+new "Add not valid example"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><c xmlns="urn:example:clixon"><single>
+       <name>smtp</name>
+       <ip>192.0.2.1</ip>
+     </single>
+     <single>
+       <name>http</name>
+       <ip>192.0.2.1</ip>
+     </single>
+</c></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf validate (should fail)"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" '^<rpc-reply><rpc-error><error-type>protocol</error-type><error-tag>operation-failed</error-tag><error-app-tag>data-not-unique</error-app-tag><error-severity>error</error-severity><error-info><non-unique><ip>192.0.2.1</ip></non-unique></error-info></rpc-error></rpc-reply>]]>]]>$'
+
+new "make valid by replacing IP of http entry"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>none</default-operation><config><c xmlns="urn:example:clixon"><single><name>http</name><ip>178.23.34.1</ip></single>
+</c></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf validate ok"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf discard-changes"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+# Then test composite case (detect duplicates among other elements)
+# and also unordered
+
+new "Add not valid example"
+expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><c xmlns="urn:example:clixon">
+     <b>other</b>
+     <single>
+       <name>smtp</name>
+       <ip>192.0.2.1</ip>
+     </single>
+     <a>other</a>
+     <server>
+       <name>smtp</name>
+       <ip>192.0.2.1</ip>
+       <port>25</port>
+     </server>
+     <single>
+       <name>http</name>
+       <ip>192.0.2.1</ip>
+     </single>
+     <other><a>xx</a></other>
+     <server>
+       <name>http</name>
+       <ip>192.0.2.1</ip>
+       <port>25</port>
+     </server>
+</c></config></edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf validate (should fail)"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" '^<rpc-reply><rpc-error><error-type>protocol</error-type><error-tag>operation-failed</error-tag><error-app-tag>data-not-unique</error-app-tag><error-severity>error</error-severity><error-info><non-unique><ip>192.0.2.1</ip></non-unique><non-unique><port>25</port></non-unique></error-info></rpc-error></rpc-reply>]]>]]>$'
+
+new "netconf discard-changes"
+expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+if [ $BE -eq 0 ]; then
+    exit # BE
+fi
+
+new "Kill backend"
+# Check if premature kill
+pid=`pgrep -u root -f clixon_backend`
+if [ -z "$pid" ]; then
+    err "backend already dead"
+fi
+# kill backend
+stop_backend -f $cfg
+
+rm -rf $dir