Augmented XML uses default namespace

* Instead of using prefixes for augmented XML, assign the default namespace
Test: moved datastore format tests from test_cli to new test_datastore_format

CHANGELOG.md

@@ -50,6 +50,12 @@ Planned: July 2022
 
 Users may have to change how they access the system
 
+* Augmented XML uses default namespace
+  * Instead of using prefixes for augmented XML, assign the default namespace
+  * This does not change the semantics, but changes the way XML rpefixes are used
+  * Example augmented ipv4 into interface:
+    * Previously: `<interface><ip:ipv4 xmlns:ip="urn:...:ietf-ip"><ip:enabled>...`
+    * Now: `<interface><ipv4 xmlns="urn:...:ietf-ip"><enabled>...` 
 * TEXT file format changed
   * With new parsing of TEXT format, the output is changed
     * Namespace/modulename added to top-level

lib/src/clixon_text_syntax_parse.y

@@ -124,7 +124,7 @@ static cxobj*
 text_create_node(clixon_text_syntax_yacc *ts,
 		 char                    *name)
 {
-    cxobj     *xn;
+    cxobj     *xn = NULL;
     yang_stmt *ymod;
     char      *ns;
     char      *prefix = NULL;

lib/src/clixon_xml_map.c

@@ -1359,11 +1359,14 @@ xmlns_assign(cxobj *x)
 }
 
 /*! Given a src element node x0 and a target node x1, assign (optional) prefix and namespace
+ * @param[in]  x1       XML tree
+ * @param[in]  x1p      XML tree parent
+ * @retval     0        OK
+ * @retval     -1       OK
  * @see assign_namespace_element  this is a subroutine
  */
 static int
-assign_namespace(cxobj *x0, /* source */
+assign_namespace(cxobj     *x1, /* target */
-		 cxobj *x1, /* target */
 		 cxobj     *x1p,
 		 int        isroot,
 		 char      *ns,
@@ -1374,8 +1377,6 @@ assign_namespace(cxobj *x0, /* source */
     char  *pexist = NULL;
     cvec  *nsc0 = NULL;
     cvec  *nsc = NULL;
-    yang_stmt *y;
-    int        ret;
     
     /* 2a. Detect if namespace is declared in x1 target parent */
     if (xml2prefix(x1p, ns, &pexist) == 1){
@@ -1416,8 +1417,7 @@ assign_namespace(cxobj *x0, /* source */
 	    }
 	    goto ok; /* skip */
 	}
-	else { /* namespace does not exist in target x1, use source prefix 
+	else { /* namespace does not exist in target x1, 
-		* use the prefix defined in the module
 		*/
 	    if (isroot){
 		if (prefix0 && (prefix1 = strdup(prefix0)) == NULL){
@@ -1426,20 +1426,11 @@ assign_namespace(cxobj *x0, /* source */
 		}
 	    }
 	    else{
-		char *ptmp;
+		if (prefix0 == NULL){ /* Use default namespace, may break use of previous default 
-		if ((y = xml_spec(x0)) == NULL){
+				       * somewhere in x1
-		    clicon_err(OE_YANG, ENOENT, "XML %s does not have yang spec",
+				       */
-			       xml_name(x0));
+		    prefix1 = NULL;
-		    goto done;
 		}
-		/* Find local (imported) prefix for that module namespace */
-		if ((ret = yang_find_prefix_by_namespace(y, ns, &ptmp)) < 0)
-		    goto done;
-		if (ret == 1 && (prefix1 = strdup(ptmp)) == NULL){
-		    clicon_err(OE_UNIX, errno, "strdup");
-		    goto done;
-		}   
-
 	    }
 	}
 	if (add_namespace(x1, x1, prefix1, ns) < 0)
@@ -1465,7 +1456,8 @@ assign_namespace(cxobj *x0, /* source */
  * 1. Find N=namespace(x0)
  * 2. Detect if N is declared in x1 parent
  * 3. If yes, assign prefix to x1
- * 4. If no, create new prefix/namespace binding and assign that to x1p (x1 if x1p is root)
+ * 4. If no, if default namespace use that, otherwise create new prefix/namespace binding and assign 
+ *    that to x1p
  * 5. Add prefix to x1, if any
  * 6. Ensure x1 cache is updated
  * @note switch use of x0 and x1 compared to datastore text_modify
@@ -1493,7 +1485,7 @@ assign_namespace_element(cxobj *x0, /* source */
 		   prefix0?prefix0:"NULL");
 	goto done;
     }
-    if (assign_namespace(x0, x1, x1p, isroot, namespace, prefix0) < 0)
+    if (assign_namespace(x1, x1p, isroot, namespace, prefix0) < 0)
 	goto done;
     /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
     retval = 0;

test/test_augment.sh

@@ -307,7 +307,7 @@ new "delete interfaces"
 expectpart "$(curl $CURLOPTS -X DELETE $RCPROTO://localhost/restconf/data/ietf-interfaces:interfaces)" 0 "HTTP/$HVER 204"
 
 # augmented lists
-XML="<interfaces xmlns=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\"><mymod:ports xmlns:mymod=\"urn:example:augment\"><mymod:id>22</mymod:id><mymod:str>nisse</mymod:str></mymod:ports><mymod:ports xmlns:mymod=\"urn:example:augment\"><mymod:id>44</mymod:id><mymod:str>kalle</mymod:str></mymod:ports></interfaces>"
+XML="<interfaces xmlns=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\"><ports xmlns=\"urn:example:augment\"><id>22</id><str>nisse</str></ports><ports xmlns=\"urn:example:augment\"><id>44</id><str>kalle</str></ports></interfaces>"
 new "netconf PUT augmented list"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><default-operation>merge</default-operation><target><candidate/></target><config>$XML</config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>" 
 

test/test_cli.sh

@@ -16,7 +16,6 @@ APPNAME=example
 
 cfg=$dir/conf_yang.xml
 clidir=$dir/cli
-
 fyang=$dir/clixon-example.yang
 
 test -d ${clidir} || rm -rf ${clidir}
@@ -71,12 +70,6 @@ module clixon-example {
 	    leaf value{
 		type string;
 	    }
-	    leaf-list array1{
-	      type string;
-            }
-	    leaf-list array2{
-	      type string;
-            }	    
 	}
     }
     rpc example {
@@ -241,35 +234,17 @@ expectpart "$($clixon_cli -1 -f $cfg -l o show compare text)" 0 "+            ad
 new "cli start shell"
 expectpart "$($clixon_cli -1 -f $cfg -l o shell echo foo)" 0 "foo" 
 
-# For formats, create three leaf-lists
+new "cli save"
-new "cli create leaflist array1 a"
+expectpart "$($clixon_cli -1 -f $cfg -l o save $dir/foo cli)" 0 "^$"
-expectpart "$($clixon_cli -1 -f $cfg -l o set table parameter a array1 a)" 0 "^$"
 
-new "cli create leaflist array1 b1 b2"
+new "cli delete all"
-expectpart "$($clixon_cli -1 -f $cfg -l o set table parameter a array1 \"b1 b2\")" 0 "^$"
+expectpart "$($clixon_cli -1 -f $cfg -l o delete all)" 0 "^$"
 
-new "cli create leaflist array2 c1 c2"
+new "cli load"
-expectpart "$($clixon_cli -1 -f $cfg -l o set table parameter a array2 \"c1 c2\")" 0 "^$"
+expectpart "$($clixon_cli -1 -f $cfg -l o load $dir/foo cli)" 0 "^$"
 
-new "cli commit"
+new "cli check load"
-expectpart "$($clixon_cli -1 -f $cfg -l o commit)" 0 "^$"
+expectpart "$($clixon_cli -1 -f $cfg -l o show conf cli)" 0 "interfaces interface eth/0/0 ipv4 enabled true"
-
-for format in cli text xml json; do
-    new "cli save $format"
-    expectpart "$($clixon_cli -1 -f $cfg -l o save $dir/config.$format $format)" 0 "^$"
-
-    new "cli delete all"
-    expectpart "$($clixon_cli -1 -f $cfg -l o delete all)" 0 "^$"
-
-    new "cli load $format"
-    expectpart "$($clixon_cli -1 -f $cfg -l o load $dir/config.$format $format)" 0 "^$"
-
-    if [ $format != json ]; then # XXX JSON identity problem
-	new "cli check compare $format"
-	expectpart "$($clixon_cli -1 -f $cfg -l o show compare xml)" 0 "^$" --not-- "i" # interface?
-    fi
-
-done
 
 new "cli debug set"
 expectpart "$($clixon_cli -1 -f $cfg -l o debug cli 1)" 0 "^$"

test/test_datastore_format.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+# Datastore format tests
+# Go through all formats and save and load a simple config via the CLI
+# Add as appropriate
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+# include err() and new() functions and creates $dir
+
+cfg=$dir/conf_yang.xml
+clidir=$dir/cli
+
+fyang=$dir/clixon-example.yang
+fyang1=$dir/clixon-augment.yang
+
+formatdir=$dir/format
+test -d ${formatdir} || rm -rf ${formatdir}
+mkdir $formatdir
+
+test -d ${clidir} || rm -rf ${clidir}
+mkdir $clidir
+
+# Use yang in example
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+</clixon-config>
+EOF
+
+cat <<EOF > $fyang
+module clixon-example {
+    yang-version 1.1;
+    namespace "urn:example:clixon";
+    prefix ex;
+    /* Generic config data */
+    container table{
+	list parameter{
+	    key name;
+	    leaf name{
+		type string;
+	    }
+	    leaf value{
+		type string;
+	    }
+	    leaf-list array1{
+	      type string;
+            }
+/*	    leaf-list array2{
+	      type string;
+            }	    
+*/
+	}
+    }
+}
+EOF
+
+cat <<EOF > $fyang1
+module clixon-augment {
+    yang-version 1.1;
+    namespace "urn:example:augment";
+    prefix aug;
+    import clixon-example {
+       prefix ex;
+    }
+    augment "/ex:table/ex:parameter" {
+        leaf-list array2{
+	      type string;
+        }	        
+    }
+}
+EOF
+
+
+cat <<EOF > $clidir/ex.cli
+# Clixon example specification
+CLICON_MODE="example";
+CLICON_PROMPT="%U@%H %W> ";
+CLICON_PLUGIN="example_cli";
+
+set @datamodel, cli_auto_set();
+delete("Delete a configuration item") {
+      @datamodel, cli_auto_del(); 
+      all("Delete whole candidate configuration"), delete_all("candidate");
+}
+validate("Validate changes"), cli_validate();
+commit("Commit the changes"), cli_commit();
+quit("Quit"), cli_quit();
+discard("Discard edits (rollback 0)"), discard_changes();
+show("Show a particular state of the system"){
+    xpath("Show configuration") <xpath:string>("XPATH expression") <ns:string>("Namespace"), show_conf_xpath("candidate");
+    compare("Compare candidate and running databases"), compare_dbs((int32)0);{
+    		     xml("Show comparison in xml"), compare_dbs((int32)0);
+		     text("Show comparison in text"), compare_dbs((int32)1);
+    }
+    configuration("Show configuration"), cli_auto_show("datamodel", "candidate", "text", true, false);{
+	    cli("Show configuration as CLI commands"), cli_auto_show("datamodel", "candidate", "cli", true, false, "set ");
+	    xml("Show configuration as XML"), cli_auto_show("datamodel", "candidate", "xml", true, false, "set ");
+	    text("Show configuration as TEXT"), cli_auto_show("datamodel", "candidate", "text", true, false, "set ");
+  }
+}
+save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename", "xml"){
+    cli("Save configuration as CLI commands"), save_config_file("candidate","filename", "cli");
+    xml("Save configuration as XML"), save_config_file("candidate","filename", "xml");
+    json("Save configuration as JSON"), save_config_file("candidate","filename", "json");
+    text("Save configuration as TEXT"), save_config_file("candidate","filename", "text");
+}
+load("Load configuration from XML file") <filename:string>("Filename (local filename)"),load_config_file("filename", "replace");{
+	cli("Replace candidate with file containing CLI commands"), load_config_file("filename", "replace", "cli");
+	xml("Replace candidate with file containing XML"), load_config_file("filename", "replace", "xml");
+	json("Replace candidate with file containing JSON"), load_config_file("filename", "replace", "json");
+	text("Replace candidate with file containing TEXT"), load_config_file("filename", "replace", "text");
+}
+
+EOF
+
+new "test params: -f $cfg"
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -z -f $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+    new "start backend -s init -f $cfg"
+    start_backend -s startup -f $cfg
+fi
+
+new "wait backend"
+wait_backend
+
+# For formats, create three leaf-lists
+new "cli create leaflist array1 a"
+expectpart "$($clixon_cli -1 -f $cfg -l o set table parameter a array1 a)" 0 "^$"
+
+new "cli create leaflist array1 b1 b2"
+expectpart "$($clixon_cli -1 -f $cfg -l o set table parameter a array1 \"b1 b2\")" 0 "^$"
+
+new "cli create leaflist array2 c1 c2"
+expectpart "$($clixon_cli -1 -f $cfg -l o set table parameter a array2 \"c1 c2\")" 0 "^$"
+
+new "cli commit"
+expectpart "$($clixon_cli -1 -f $cfg -l o commit)" 0 "^$"
+
+for format in cli text xml json; do
+    new "cli save $format"
+    expectpart "$($clixon_cli -1 -f $cfg -l o save $formatdir/config.$format $format)" 0 "^$"
+
+    new "cli delete all"
+    expectpart "$($clixon_cli -1 -f $cfg -l o delete all)" 0 "^$"
+
+    new "cli load $format"
+    expectpart "$($clixon_cli -1 -f $cfg -l o load $formatdir/config.$format $format)" 0 "^$"
+
+    if [ $format != json ]; then # XXX JSON identity problem
+	new "cli check compare $format"
+	expectpart "$($clixon_cli -1 -f $cfg -l o show compare xml)" 0 "^$" --not-- "i" # interface?
+    fi
+
+done
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u root -f clixon_backend)
+    if [ -z "$pid" ]; then
+	err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+rm -rf $dir
+
+new "endtest"
+endtest

test/test_datastore_repair.sh

@@ -77,7 +77,7 @@ sudo chmod 666 $dir/startup_db
 
 # This is how it should look after repair, using prefixes
 AFTER=$(cat <<EOF
-<x xmlns="urn:example:a"><y><b:z xmlns:b="urn:example:b"><b:w>foo</b:w></b:z></y></x>
+<x xmlns="urn:example:a"><y><z xmlns="urn:example:b"><w>foo</w></z></y></x>
 EOF
 )
 

test/test_netconf.sh

@@ -143,7 +143,7 @@ expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "$rpc" "" "<rpc-r
 rpc="<rpc $DEFAULTNS><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/if:interfaces/if:interface[if:name='eth1']/if:enabled/../..\" xmlns:if=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\"/></get-config></rpc>"
 
 new "netconf get config xpath parent"
-expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "$rpc" "" "<rpc-reply $DEFAULTNS><data><interfaces xmlns=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\"><interface><name>eth/0/0</name><enabled>true</enabled></interface><interface><name>eth1</name><enabled>true</enabled><ip:ipv4 xmlns:ip=\"urn:ietf:params:xml:ns:yang:ietf-ip\"><ip:enabled>true</ip:enabled><ip:forwarding>false</ip:forwarding><ip:address><ip:ip>9.2.3.4</ip:ip><ip:prefix-length>24</ip:prefix-length></ip:address></ip:ipv4></interface></interfaces></data></rpc-reply>"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "$rpc" "" "<rpc-reply $DEFAULTNS><data><interfaces xmlns=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\"><interface><name>eth/0/0</name><enabled>true</enabled></interface><interface><name>eth1</name><enabled>true</enabled><ipv4 xmlns=\"urn:ietf:params:xml:ns:yang:ietf-ip\"><enabled>true</enabled><forwarding>false</forwarding><address><ip>9.2.3.4</ip><prefix-length>24</prefix-length></address></ipv4></interface></interfaces></data></rpc-reply>"
 
 new "netconf validate missing type"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><validate><source><candidate/></source></validate></rpc>" "<rpc-reply $DEFAULTNS><rpc-error>" ""