Merge branch 'develop' for 3.4.0

.gitignore

@@ -0,0 +1,49 @@
+*.o
+*.so.*
+*_parse.tab.c
+*_parse.tab.h
+lex.*_parse.c
+
+Makefile
+apps/Makefile
+apps/*/Makefile
+docker/Makefile
+docker/*/Makefile
+etc/Makefile
+example/Makefile
+lib/Makefile
+lib/*/Makefile
+autom4te.cache/
+
+clixon.conf.cpp
+clixon.mk
+config.log
+config.status
+
+apps/backend/clixon_backend
+apps/backend/test
+apps/backend/test.c
+apps/cli/clixon_cli
+apps/cli/test
+apps/cli/test.c
+apps/dbctrl/clixon_dbctrl
+apps/netconf/clixon_netconf
+apps/restconf/clixon_restconf
+apps/xmldb/clixon_xmldb
+
+docker/backend/Dockerfile
+docker/cli/Dockerfile
+docker/netconf/Dockerfile
+
+etc/clixonrc
+
+example/*.conf
+
+include/clixon_config.h
+
+lib/src/build.c
+lib/clixon/clixon.h
+
+build-root/*.tar.xz
+build-root/*.rpm
+build-root/rpmbuild

CHANGELOG.md

@@ -1,10 +1,55 @@
 # Clixon Changelog
 
+## 3.4.0 (Upcoming)
+### Known issues
+* Please use text datastore, key-value datastore no up-to-date
+
+### Major changes:
+* Optimized search performance for large lists by sorting and binary search.
+  * New CLICON_XML_SORT configuration option. Default is true. Disable by setting to false.
+  * Added yang ordered-by user. The default (ordered-by system) will now sort lists and leaf-lists alphabetically to increase search performance. Note that this may change outputs.
+  * If you need legacy order, either set CLICON_XML_SORT to false, or set that list to "ordered-by user".
+  * This replaces XML hash experimental code, ie xml_child_hash variables and all xml_hash_ functions have been removed.
+  * Implementation detail: Cached keys are stored in in yang Y_LIST nodes as cligen vector, see ys_populate_list()  
+
+* Datastore cache introduced: cache XML tree in memory for faster get access.
+  * Reads are cached. Writes are written to disk.
+  * New CLICON_XMLDB_CACHE configuration option. Default is true. To disable set to false.
+  * With cache, you cannot have multiple backends (with single datastore). You need to have a single backend.
+  * Thanks netgate for proposing this.
+
+* Changed C functional API for XML creation and parsing for better coherency and closer YANG/XML integration. This may require your action.
+  * New yang spec parameter has been added to most functions (default NULL) and functions have been removed and renamed. You may need to change the XML calls as follows.
+  * xml_new(name, parent) --> xml_new(name, xn_parent, yspec)
+  * xml_new_spec(name, parent, spec) --> xml_new(name, parent, spec)
+  * clicon_xml_parse(&xt, format, ...) --> xml_parse_va(&xt, yspec, format, ...)
+  * clicon_xml_parse_file(fd, &xt, endtag) --> xml_parse_file(fd, endtag, yspec, &xt)
+  * clicon_xml_parse_string(&str, &xt) --> xml_parse_string(str, yspec, &xt)
+  * clicon_xml_parse_str(str, &xt) --> xml_parse_string(str, yspec, &xt)
+  * xml_parse(str, xt) --> xml_parse_string(str, yspec, &xt)
+  * Backward compatibility is enabled by (will be removed in 3.5.0:
+  ```
+      configure --with-xml-compat
+  ```
+  
+### Minor changes:
+* Better semantic versioning, eg MAJOR/MINOR/PATCH, where increment in PATCH does not change API.
+* Added CLICON_XMLDB_PRETTY option. If set to false, XML database files will be more compact.
+* Added CLICON_XMLDB_FORMAT option. Default is "xml". If set to "json", XML database files uses JSON format.
+* Clixon_backend now returns -1/255 on error instead of 0. Useful for systemd restarts, for example.
+* Experimental: netconf yang rpc. That is, using ietf-netconf@2011-06-01.yang
+  formal specification instead of hardcoded C-code.
+
+### Corrected Bugs
+
 * Fixed bug that deletes running on startup if backup started with -m running.
   When clixon starts again, running is lost.
   The error was that the running (or startup) configuration may fail when
   clixon backend starts. 
-  The fix now makes a copy of running and copies it back on failure
+  The fix now makes a copy of running and copies it back on failure.
+* datastore/keyvalue/Makefile was left behind on make distclean. Fixed by conditional configure. Thanks renato@netgate.com.
+* Escape " in JSON names and strings and values
+  
 
 ## 3.3.3 (25 November 2017)
 

Makefile.in

@@ -106,11 +106,51 @@ clean:
 
 distclean:
 	rm -f Makefile TAGS config.status config.log *~ .depend
-	rm -rf Makefile autom4te.cache 
+	rm -rf autom4te.cache
-	rm -rf clixon.conf.cpp clixon.mk
+	rm -rf clixon.conf.cpp clixon.mk build-root/rpmbuild
+	rm -f build-root/*.tar.xz build-root/*.rpm extras/rpm/Makefile
 	for i in $(SUBDIRS) doc example docker; \
 		do (cd $$i && $(MAKE) $(MFLAGS) $@); done
 
+export BR=$(CURDIR)/build-root
+
+$(BR)/scripts/.version:
+ifneq ("$(wildcard /etc/redhat-release)","")
+	$(shell $(BR)/scripts/version rpm-string > $(BR)/scripts/.version)
+else
+	$(shell $(BR)/scripts/version > $(BR)/scripts/.version)
+endif
+
+DIST_FILE = $(BR)/clixon-$(shell extras/scripts/version).tar
+DIST_SUBDIR = clixon-$(shell extras/scripts/version | cut -f1 -d-)
+
+dist:
+	@if git rev-parse 2> /dev/null ; then \
+	    git archive \
+	      --prefix=$(DIST_SUBDIR)/ \
+	      --format=tar \
+	      -o $(DIST_FILE) \
+	    HEAD ; \
+	    git describe > $(BR)/.version ; \
+	else \
+	    (cd .. ; tar -cf $(DIST_FILE) $(DIST_SUBDIR) --exclude=*.tar) ; \
+	    extras/scripts/version > $(BR)/.version ; \
+	fi
+	@tar --append \
+	  --file $(DIST_FILE) \
+	  --transform='s,.*/.version,$(DIST_SUBDIR)/extras/scripts/.version,' \
+	  $(BR)/.version
+	@$(RM) $(BR)/.version $(DIST_FILE).xz
+	@xz -v --threads=0 $(DIST_FILE)
+	@$(RM) $(BR)/clixon-latest.tar.xz
+	@ln -rs $(DIST_FILE).xz $(BR)/clixon-latest.tar.xz
+
+pkg-rpm: dist
+	make -C extras/rpm
+
+pkg-srpm: dist
+	make -C extras/rpm srpm
+
 docker:
 	for i in docker; \
 		do (cd $$i && $(MAKE) $(MFLAGS) $@); done

apps/backend/backend_client.c

@@ -403,7 +403,11 @@ from_client_edit_config(clicon_handle h,
 		    "</rpc-error></rpc-reply>");
 	    goto ok;
 	}
-
+	/* Cant do this earlier since we dont have a yang spec to
+	 * the upper part of the tree, until we get the "config" tree.
+	 */
+	if (xml_child_sort && xml_apply0(xc, CX_ELMNT, xml_sort, NULL) < 0)
+	    goto done;
 	if (xmldb_put(h, target, operation, xc) < 0){
 	    cprintf(cbret, "<rpc-reply><rpc-error>"
 		    "<error-tag>operation-failed</error-tag>"

apps/backend/backend_commit.c

@@ -130,7 +130,8 @@ generic_validate(yang_spec          *yspec,
  * @param[in] h         Clicon handle
  * @param[in] candidate The candidate database. The wanted backend state
  * @retval    0         OK  
- * @retval   -1         Fatal error or netconf error XXX Differentiate
+ * @retval   -1         Fatal error or validation fail
+ * @note Need to differentiate between error and validation fail
  */
 static int
 validate_common(clicon_handle       h, 
@@ -215,7 +216,10 @@ validate_common(clicon_handle       h,
  * do something more drastic?
  * @param[in]  h         Clicon handle
  * @param[in]  candidate A candidate database, not necessarily "candidate"
-*/
+ * @retval    0         OK  
+ * @retval   -1         Fatal error or validation fail
+ * @note Need to differentiate between error and validation fail
+ */
 int
 candidate_commit(clicon_handle h, 
 		 char         *candidate)
@@ -285,7 +289,7 @@ from_client_commit(clicon_handle h,
 		piddb);
 	goto ok;
     }
-    if (candidate_commit(h, "candidate") < 0){
+    if (candidate_commit(h, "candidate") < 0){ /* Assume validation fail, nofatal */
 	clicon_debug(1, "Commit candidate failed");
 	cprintf(cbret, "<rpc-reply><rpc-error>"
 		"<error-tag>invalid-value</error-tag>"
@@ -295,8 +299,6 @@ from_client_commit(clicon_handle h,
 		"</rpc-error></rpc-reply>",
 		clicon_err_reason);
         goto ok;
-
-	goto ok;
     }
     cprintf(cbret, "<rpc-reply><ok/></rpc-reply>");
  ok:

apps/backend/backend_main.c

@@ -74,9 +74,9 @@
 
 /* Command line options to be passed to getopt(3) */
 #ifdef BACKEND_STARTUP_COMPAT
-#define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:IRCrg:py:x:" /* substitute s: for IRCc:r */
+#define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:IRCrg:y:x:" /* substitute s: for IRCc:r */
 #else
-#define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:g:py:x:" /* substitute s: for IRCc:r */
+#define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:g:y:x:" /* substitute s: for IRCc:r */
 #endif
 
 /*! Terminate. Cannot use h after this */
@@ -148,7 +148,6 @@ usage(char *argv0, clicon_handle h)
 	    "    -C\t\tCall plugin_reset() in plugins to reset system state in candidate db (use with -I)\n"
 	    "    -r\t\tReload running database\n"
 #endif /* BACKEND_STARTUP_COMPAT */
-	    "    -p \t\tPrint database yang specification\n"
 	    "    -g <group>\tClient membership required to this group (default: %s)\n"
 	    "    -y <file>\tOverride yang spec file (dont include .yang suffix)\n"
 	    "    -x <plugin>\tXMLDB plugin\n",
@@ -165,7 +164,7 @@ static int
 db_reset(clicon_handle h, 
 	 char         *db)
 {
-    if (xmldb_delete(h, db) != 0 && errno != ENOENT) 
+    if (xmldb_exists(h, db) == 1 && xmldb_delete(h, db) != 0 && errno != ENOENT) 
 	return -1;
     if (xmldb_create(h, db) < 0)
 	return -1;
@@ -297,7 +296,7 @@ rundb_main(clicon_handle h,
 	clicon_err(OE_UNIX, errno, "open(%s)", extraxml_file);
 	goto done;
     }
-    if (clicon_xml_parse_file(fd, &xt, "</clicon>") < 0)
+    if (xml_parse_file(fd, &xt, "</clicon>") < 0)
 	goto done;
     if ((xn = xml_child_i(xt, 0)) != NULL)
 	if (xmldb_put(h, "tmp", OP_MERGE, xn) < 0)
@@ -443,7 +442,7 @@ load_extraxml(clicon_handle h,
 	clicon_err(OE_UNIX, errno, "open(%s)", filename);
 	goto done;
     }
-    if (clicon_xml_parse_file(fd, &xt, "</config>") < 0)
+    if (xml_parse_file(fd, "</config>", NULL, &xt) < 0)
 	goto done;
     /* Replace parent w first child */
     if (xml_rootchild(xt, 0, &xt) < 0)
@@ -503,13 +502,22 @@ startup_mode_init(clicon_handle h)
 
 /*! Clixon running startup mode: Commit running db configuration into running 
  *
-        copy   reset              commit merge
+OK:
-running----+   |--------------------+-----+------>
+        copy   reset              commit   merge
-            \                      /     /
+running----+   |--------------------+--------+------>
-candidate    +--------------------+     /
+            \                      /        /
-                                       /
+candidate    +--------------------+        /
-tmp           |-------+-----+---------+
+                                          /
+tmp           |-------+-----+------------+---|
              reset   extra  file
+
+COMMIT ERROR:
+        copy   reset              copy 
+running----+   |--------------------+------> EXIT
+            \                      /       
+candidate    +--------------------+        
+
+ * @note: if commit fails, copy candidate to running and exit
  */
 static int
 startup_mode_running(clicon_handle h,
@@ -523,9 +531,6 @@ startup_mode_running(clicon_handle h,
     /* Load plugins and call plugin_init() */
     if (plugin_initiate(h) != 0) 
 	goto done;
-    /* Clear running db */
-    if (db_reset(h, "running") < 0)
-	goto done;
     /* Clear tmp db */
     if (db_reset(h, "tmp") < 0)
 	goto done;
@@ -535,12 +540,21 @@ startup_mode_running(clicon_handle h,
     /* Get application extra xml from file */
     if (load_extraxml(h, extraxml_file, "tmp") < 0)   
 	goto done;	    
-    /* Commit original running */
+    /* Clear running db */
+    if (db_reset(h, "running") < 0)
+	goto done;
+    /* Commit original running. Assume -1 is validate fail */
     if (candidate_commit(h, "candidate") < 0){
-        clicon_log(LOG_NOTICE, "%s: Commit of saved running failed, exiting.", __FUNCTION__);
+	/*  (1) We cannot differentiate between fatal errors and validation
-        /* Reinstate original */
+	 *      failures
-        if (xmldb_copy(h, "candidate", "running") < 0)
+	 *  (2) If fatal error, we should exit
-            goto done;
+	 *  (3) If validation fails we cannot continue. How could we?
+	 *  (4) Need to restore the running db since we destroyed it above
+	 */
+	clicon_log(LOG_NOTICE, "%s: Commit of saved running failed, exiting.", __FUNCTION__);
+	/* Reinstate original */
+	if (xmldb_copy(h, "candidate", "running") < 0)
+	    goto done;
 	goto done;
     }
     /* Merge user reset state and extra xml file (no commit) */
@@ -548,17 +562,31 @@ startup_mode_running(clicon_handle h,
 	goto done;
     retval = 0;
  done:
+    if (xmldb_delete(h, "tmp") < 0)
+	goto done;
     return retval;
 }
 
 /*! Clixon startup startup mode: Commit startup configuration into running state
-               reset              commit merge
+
-running        |--------------------+-----+------>
+
-                                   /     /
+backup         +--------------------|
-startup       --------------------+     /
+         copy / reset              commit merge
-                                       /
+running   |-+----|--------------------+-----+------>
-tmp           |-------+-----+---------+
+                                     /     /
+startup    -------------------------+-->  /
+                                         /
+tmp        -----|-------+-----+---------+--|
              reset   extra  file
+
+COMMIT ERROR:
+backup         +------------------------+--|
+         copy / reset               copy \
+running   |-+----|--------------------+---+------->EXIT
+                               error / 
+startup    -------------------------+--|    
+
+ * @note: if commit fails, copy backup to commit and exit
  */
 static int
 startup_mode_startup(clicon_handle h,
@@ -566,6 +594,9 @@ startup_mode_startup(clicon_handle h,
 {
     int     retval = -1;
 
+    /* Stash original running to backup */
+    if (xmldb_copy(h, "running", "backup") < 0)
+	goto done;
     /* If startup does not exist, clear it */
     if (xmldb_exists(h, "startup") != 1) /* diff */
 	if (xmldb_create(h, "startup") < 0) /* diff */
@@ -573,9 +604,6 @@ startup_mode_startup(clicon_handle h,
     /* Load plugins and call plugin_init() */
     if (plugin_initiate(h) != 0) 
 	goto done;
-    /* Clear running db */
-    if (db_reset(h, "running") < 0)
-	goto done;
     /* Clear tmp db */
     if (db_reset(h, "tmp") < 0)
 	goto done;
@@ -585,20 +613,36 @@ startup_mode_startup(clicon_handle h,
     /* Get application extra xml from file */
     if (load_extraxml(h, extraxml_file, "tmp") < 0)   
 	goto done;	    
-    /* Commit startup */
+    /* Clear running db */
-    if (candidate_commit(h, "startup") < 0) /* diff */
+    if (db_reset(h, "running") < 0)
 	goto done;
+    /* Commit startup */
+    if (candidate_commit(h, "startup") < 0){ /* diff */
+	/*  We cannot differentiate between fatal errors and validation
+	 *  failures
+	 *  In both cases we copy back the original running and quit
+	 */
+	clicon_log(LOG_NOTICE, "%s: Commit of startup failed, exiting.", __FUNCTION__);
+	if (xmldb_copy(h, "backup", "running") < 0)
+	    goto done;
+	goto done;
+    }
     /* Merge user reset state and extra xml file (no commit) */
     if (db_merge(h, "tmp", "running") < 0)
 	goto done;
     retval = 0;
  done:
+    if (xmldb_delete(h, "backup") < 0)
+	goto done;
+    if (xmldb_delete(h, "tmp") < 0)
+	goto done;
     return retval;
 }
 
 int
 main(int argc, char **argv)
 {
+    int           retval = -1;
     char          c;
     int           zap;
     int           foreground;
@@ -616,12 +660,14 @@ main(int argc, char **argv)
     struct stat   st;
     clicon_handle h;
     int           help = 0;
-    int           printspec = 0;
     int           pid;
     char         *pidfile;
     char         *sock;
     int           sockfamily;
     char         *xmldb_plugin;
+    int           xml_cache;
+    int           xml_pretty;
+    char         *xml_format;
 
     /* In the startup, logs to stderr & syslog and debug flag set later */
     clicon_log_init(__PROGRAM__, LOG_INFO, CLICON_LOG_STDERR|CLICON_LOG_SYSLOG);
@@ -740,9 +786,6 @@ main(int argc, char **argv)
 	case 'g': /* config socket group */
 	    clicon_option_str_set(h, "CLICON_SOCK_GROUP", optarg);
 	    break;
-	case 'p' : /* Print spec */
-	    printspec++;
-	    break;
 	case 'y' :{ /* Override yang module or absolute filename */
 	    clicon_option_str_set(h, "CLICON_YANG_MODULE_MAIN", optarg);
 	    break;
@@ -782,6 +825,7 @@ main(int argc, char **argv)
 	    unlink(pidfile);   
 	if (sockfamily==AF_UNIX && lstat(sock, &st) == 0)
 	    unlink(sock);   
+	backend_terminate(h);
 	exit(0); /* OK */
     }
     else
@@ -812,7 +856,8 @@ main(int argc, char **argv)
 		"or create the group and add the user to it. On linux for example:"
 		"  sudo groupadd %s\n" 
 		"  sudo usermod -a -G %s user\n", 
-		   config_group, clicon_configfile(h), config_group, config_group);
+		   config_group, clicon_configfile(h),
+		   config_group, config_group);
 	return -1;
     }
 
@@ -826,7 +871,7 @@ main(int argc, char **argv)
     if (xmldb_connect(h) < 0)
 	goto done;
     /* Parse db spec file */
-    if (yang_spec_main(h, stdout, printspec) < 0)
+    if (yang_spec_main(h) == NULL)
 	goto done;
 
     /* Set options: database dir and yangspec (could be hidden in connect?)*/
@@ -834,6 +879,15 @@ main(int argc, char **argv)
 	goto done;
     if (xmldb_setopt(h, "yangspec", clicon_dbspec_yang(h)) < 0)
 	goto done;
+    if ((xml_cache = clicon_option_bool(h, "CLICON_XMLDB_CACHE")) >= 0)
+	if (xmldb_setopt(h, "xml_cache", (void*)(intptr_t)xml_cache) < 0)
+	    goto done;
+    if ((xml_format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) >= 0)
+	if (xmldb_setopt(h, "format", (void*)xml_format) < 0)
+	    goto done;
+    if ((xml_pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) >= 0)
+	if (xmldb_setopt(h, "pretty", (void*)(intptr_t)xml_pretty) < 0)
+	    goto done;
     /* If startup mode is not defined, eg via OPTION or -s, assume old method */
     startup_mode = clicon_startup_mode(h);
     if (startup_mode == -1){ 	/* Old style, fragmented mode, phase out */
@@ -921,9 +975,10 @@ main(int argc, char **argv)
 
     if (event_loop() < 0)
 	goto done;
+    retval = 0;
   done:
-    clicon_log(LOG_NOTICE, "%s: %u Terminated", __PROGRAM__, getpid());
+    clicon_log(LOG_NOTICE, "%s: %u Terminated retval:%d", __PROGRAM__, getpid(), retval);
     backend_terminate(h); /* Cannot use h after this */
 
-    return 0;
+    return retval;
 }

apps/backend/backend_plugin.c

@@ -752,7 +752,7 @@ backend_statedata_call(clicon_handle        h,
     for (i = 0; i < _nplugins; i++)  {
 	p = &_plugins[i];
 	if (p->p_statedata) {
-	    if ((x = xml_new("config", NULL)) == NULL)
+	    if ((x = xml_new("config", NULL, NULL)) == NULL)
 		goto done;
 	    if ((p->p_statedata)(h, xpath, x) < 0){
 		retval = 1;

apps/backend/backend_plugin.h

@@ -40,10 +40,15 @@
  * Types
  */
 
-/*! Transaction data
+/*! Transaction data describing a system transition from a src to target state
  * Clicon internal, presented as void* to app's callback in the 'transaction_data'
  * type in clicon_backend_api.h
- * XXX: move to .c file?
+ * The struct contains source and target XML tree (e.g. candidate/running)
+ * But primarily a set of XML tree vectors (dvec, avec, cvec) and associated lengths
+ * These contain the difference between src and target XML, ie "what has changed".
+ * It is up to the validate callbacks to ensure that these changes are OK
+ * It is up to the commit callbacks to enforce these changes in the "state" of 
+ *the system.
  */
 typedef struct {
     uint64_t   td_id;       /* Transaction id */

apps/cli/cli_common.c

@@ -233,12 +233,12 @@ cli_dbxml(clicon_handle       h,
     if (api_path_fmt2api_path(api_path_fmt, cvv, &api_path) < 0)
 	goto done;
     /* Create config top-of-tree */
-    if ((xtop = xml_new("config", NULL)) == NULL)
+    if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
     if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0)
 	goto done;
-    if ((xa = xml_new("operation", xbot)) == NULL)
+    if ((xa = xml_new("operation", xbot, NULL)) == NULL)
 	goto done;
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa,  xml_operation2str(op)) < 0)
@@ -251,7 +251,7 @@ cli_dbxml(clicon_handle       h,
 		clicon_err(OE_UNIX, errno, "cv2str_dup");
 		goto done;
 	    }
-	    if ((xb = xml_new("body", xbot)) == NULL)
+	    if ((xb = xml_new("body", xbot, NULL)) == NULL)
 		goto done; 
 	    xml_type_set(xb, CX_BODY);
 	    if (xml_value_set(xb,  str) < 0)
@@ -724,8 +724,7 @@ load_config_file(clicon_handle h,
     opstr  = cv_string_get(cvec_i(argv, 1));
     if (strcmp(opstr, "merge") == 0) 
 	replace = 0;
-    else
+    else if (strcmp(opstr, "replace") == 0) 
-    if (strcmp(opstr, "replace") == 0) 
 	replace = 1;
     else{
 	clicon_err(OE_PLUGIN, 0, "No such op: %s, expected merge or replace", opstr);	
@@ -738,7 +737,7 @@ load_config_file(clicon_handle h,
     filename = cv_string_get(cv);
     if (stat(filename, &st) < 0){
  	clicon_err(OE_UNIX, 0, "load_config: stat(%s): %s", 
- 		filename, strerror(errno));
+		   filename, strerror(errno));
 	goto done;
     }
     /* Open and parse local file into xml */
@@ -746,30 +745,27 @@ load_config_file(clicon_handle h,
 	clicon_err(OE_UNIX, errno, "%s: open(%s)", __FUNCTION__, filename);
 	goto done;
     }
-    if (clicon_xml_parse_file(fd, &xt, "</clicon>") < 0)
+    if (xml_parse_file(fd, "</clicon>", NULL, &xt) < 0)
 	goto done;
     if (xt == NULL)
 	goto done;
-
+    if ((cbxml = cbuf_new()) == NULL)
-    //    if ((xn = xml_child_i(xt, 0)) != NULL){
+	goto done;
-    
+    x = NULL;
-	if ((cbxml = cbuf_new()) == NULL)
+    while ((x = xml_child_each(xt, x, -1)) != NULL) {
+	/* Ensure top-level is "config", maybe this is too rough? */
+	xml_name_set(x, "config");
+	if (clicon_xml2cbuf(cbxml, x, 0, 0) < 0)
 	    goto done;
-	x = NULL;
+    }
-	while ((x = xml_child_each(xt, x, -1)) != NULL) {
+    if (clicon_rpc_edit_config(h, "candidate",
-	    /* Ensure top-level is "config", maybe this is too rough? */
+			       replace?OP_REPLACE:OP_MERGE, 
-	    xml_name_set(x, "config");
+			       cbuf_get(cbxml)) < 0)
-	    if (clicon_xml2cbuf(cbxml, x, 0, 0) < 0)
+	goto done;
-		goto done;
+    cbuf_free(cbxml);
-	}
+    //    }
-	if (clicon_rpc_edit_config(h, "candidate",
-				   replace?OP_REPLACE:OP_MERGE, 
-				   cbuf_get(cbxml)) < 0)
-	    goto done;
-	cbuf_free(cbxml);
-	//    }
     ret = 0;
-  done:
+ done:
     if (xt)
 	xml_free(xt);
     if (fd != -1)
@@ -837,7 +833,7 @@ save_config_file(clicon_handle h,
 	clicon_rpc_generate_error("Get configuration", xerr);
 	goto done;
     }
-    if ((f = fopen(filename, "wb")) == NULL){
+    if ((f = fopen(filename, "w")) == NULL){
 	clicon_err(OE_CFG, errno, "Creating file %s", filename);
 	goto done;
     } 
@@ -1189,7 +1185,7 @@ cli_copy_config(clicon_handle h,
     }
     toname = cv_string_get(tocv);
     /* Create copy xml tree x2 */
-    if ((x2 = xml_new("new", NULL)) == NULL)
+    if ((x2 = xml_new("new", NULL, NULL)) == NULL)
 	goto done;
     if (xml_copy(x1, x2) < 0)
 	goto done;

apps/cli/cli_generate.c

@@ -547,7 +547,6 @@ yang2cli_list(clicon_handle h,
 {
     yang_stmt    *yc;
     yang_stmt    *yd;
-    yang_stmt    *ykey;
     yang_stmt    *yleaf;
     int           i;
     cg_var       *cvi;
@@ -568,13 +567,7 @@ yang2cli_list(clicon_handle h,
 	cprintf(cbuf, "(\"%s\")", helptext);
     }
     /* Loop over all key variables */
-    if ((ykey = yang_find((yang_node*)ys, Y_KEY, NULL)) == NULL){
+    cvk = ys->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
-	clicon_err(OE_XML, 0, "List statement \"%s\" has no key", ys->ys_argument);
-	goto done;
-    }
-    /* The value is a list of keys: <key>[ <key>]*  */
-    if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
-	goto done;
     cvi = NULL;
     /* Iterate over individual keys  */
     while ((cvi = cvec_each(cvk, cvi)) != NULL) {
@@ -614,8 +607,6 @@ yang2cli_list(clicon_handle h,
   done:
     if (helptext)
 	free(helptext);
-    if (cvk)
-	cvec_free(cvk);
     return retval;
 }
 

apps/cli/cli_main.c

@@ -242,9 +242,9 @@ main(int argc, char **argv)
     int          logdst = CLICON_LOG_STDERR;
     char        *restarg = NULL; /* what remains after options */
     int          dump_configfile_xml = 0;
+    yang_spec   *yspec;
     
     /* Defaults */
-
     /* In the startup, logs to stderr & debug flag set later */
     clicon_log_init(__PROGRAM__, LOG_INFO, logdst); 
     /* Initiate CLICON handle */
@@ -356,7 +356,7 @@ main(int argc, char **argv)
 	    clicon_option_str_set(h, "CLICON_CLI_MODE", optarg);
 	    break;
 	case 'q' : /* Quiet mode */
-	    clicon_option_str_set(h, "CLICON_QUIET", "on"); 
+	    clicon_quiet_mode_set(h, 1);
 	    break;
 	case 'p' : /* Print spec */
 	    printspec++;
@@ -397,20 +397,17 @@ main(int argc, char **argv)
     cv_exclude_keys(clicon_cli_varonly(h)); 
 
     /* Parse db specification as cli*/
-    if (yang_spec_main(h, stdout, printspec) < 0)
+    if ((yspec = yang_spec_main(h)) == NULL)
 	goto done;
+    if (printspec)
+	yang_print(stdout, (yang_node*)yspec);
 
     /* Create tree generated from dataspec. If no other trees exists, this is
      * the only one.
      */
     if (clicon_cli_genmodel(h)){
-	yang_spec         *yspec;      /* yang spec */
 	parse_tree         pt = {0,};  /* cli parse tree */
 
-	if ((yspec = clicon_dbspec_yang(h)) == NULL){
-	    clicon_err(OE_FATAL, 0, "No YANG DB_SPEC");
-	    goto done;
-	}
 	/* Create cli command tree from dbspec */
 	if (yang2cli(h, yspec, &pt, clicon_cli_genmodel_type(h)) < 0)
 	    goto done;

apps/cli/cli_show.c

@@ -166,7 +166,7 @@ expand_dbvar(void   *h,
     xcur = xt; /* default top-of-tree */
     xpathcur = xpath;
     /* Create config top-of-tree */
-    if ((xtop = xml_new("config", NULL)) == NULL)
+    if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
     /* This is primarily to get "y", 
@@ -209,8 +209,7 @@ expand_dbvar(void   *h,
 	else
 	    bodystr = xml_body(x);
 	if (bodystr == NULL){
-	    clicon_err(OE_CFG, 0, "No xml body");
+	    continue; /* no body, cornercase */
-	    goto done;
 	}
 	/* detect duplicates */
 	for (k=0; k<j; k++){

apps/netconf/netconf_lib.c

@@ -190,7 +190,7 @@ netconf_output(int   s,
     clicon_debug(1, "SEND %s", msg);
     if (debug > 1){ /* XXX: below only works to stderr, clicon_debug may log to syslog */
 	cxobj *xt = NULL;
-	if (clicon_xml_parse_string(&buf, &xt) == 0){
+	if (xml_parse_string(buf, NULL, &xt) == 0){
 	    clicon_xml2file(stderr, xml_child_i(xt, 0), 0, 0);
 	    fprintf(stderr, "\n");
 	    xml_free(xt);

apps/netconf/netconf_main.c

@@ -98,7 +98,7 @@ process_incoming_packet(clicon_handle h,
     }
     str = str0;
     /* Parse incoming XML message */
-    if (clicon_xml_parse_string(&str, &xreq) < 0){
+    if (xml_parse_string(str, NULL, &xreq) < 0){ 
 	if ((cbret = cbuf_new()) == NULL){
 	    cprintf(cbret, "<rpc-reply><rpc-error>"
 		"<error-tag>operation-failed</error-tag>"
@@ -114,9 +114,8 @@ process_incoming_packet(clicon_handle h,
 	goto done;
     }
     free(str0);
-    if ((xrpc=xpath_first(xreq, "//rpc")) != NULL){
+    if ((xrpc=xpath_first(xreq, "//rpc")) != NULL)
         isrpc++;
-    }
     else
         if (xpath_first(xreq, "//hello") != NULL)
 	    ;
@@ -215,9 +214,10 @@ netconf_input_cb(int   s,
 			      buf[i],
 			      &xml_state)) {
 		/* OK, we have an xml string from a client */
-		if (process_incoming_packet(h, cb) < 0){
+		/* Remove trailer */
+		*(((char*)cbuf_get(cb)) + cbuf_len(cb) - strlen("]]>]]>")) = '\0';
+		if (process_incoming_packet(h, cb) < 0)
 		    goto done;
-		}
 		if (cc_closed)
 		    break;
 		cbuf_reset(cb);
@@ -268,10 +268,11 @@ netconf_terminate(clicon_handle h)
 {
     yang_spec      *yspec;
 
-
     clicon_rpc_close_session(h);
     if ((yspec = clicon_dbspec_yang(h)) != NULL)
 	yspec_free(yspec);
+    if ((yspec = clicon_netconf_yang(h)) != NULL)
+	yspec_free(yspec);
     event_exit();
     clicon_handle_exit(h);
     return 0;
@@ -302,7 +303,8 @@ usage(clicon_handle h,
 }
 
 int
-main(int argc, char **argv)
+main(int    argc,
+     char **argv)
 {
     char             c;
     char            *tmp;
@@ -337,7 +339,6 @@ main(int argc, char **argv)
 	     use_syslog = 1;
 	     break;
 	}
-
     /* 
      * Logs, error and debug to stderr or syslog, set debug level
      */
@@ -379,12 +380,16 @@ main(int argc, char **argv)
     argv += optind;
 
     /* Parse yang database spec file */
-    if (yang_spec_main(h, stdout, 0) < 0)
+    if (yang_spec_main(h) == NULL)
+	goto done;
+
+    /* Parse netconf yang spec file  */
+    if (yang_spec_netconf(h) == NULL)
 	goto done;
 
     /* Initialize plugins group */
     if (netconf_plugin_load(h) < 0)
-	return -1;
+	goto done;
 
     /* Call start function is all plugins before we go interactive */
     tmp = *(argv-1);

apps/netconf/netconf_plugin.c

@@ -197,8 +197,10 @@ catch:
     return -1;
 }
 
-
 /*! See if there is any callback registered for this tag
+ *
+ * Look for local (client-side) netconf plugins. This feature may no 
+ * longer be necessary as generic RPC:s should be handled by backend.
  *
  * @param[in]  h       clicon handle
  * @param[in]  xn      Sub-tree (under xorig) at child of rpc: <rpc><xn></rpc>.
@@ -215,12 +217,6 @@ netconf_plugin_callbacks(clicon_handle h,
 {
     int            retval = -1;
     netconf_reg_t *nreg;
-    yang_spec     *yspec;
-    yang_stmt     *yrpc;
-    yang_stmt     *yinput;
-    yang_stmt     *youtput;
-    cxobj         *xoutput;
-    cbuf          *cb = NULL;
   
     if (deps != NULL){
 	nreg = deps;
@@ -234,57 +230,8 @@ netconf_plugin_callbacks(clicon_handle h,
 	    nreg = NEXTQ(netconf_reg_t *, nreg);
 	} while (nreg != deps);
     }
-    if ((yspec =  clicon_dbspec_yang(h)) == NULL){
-	clicon_err(OE_YANG, ENOENT, "No yang spec");
-	goto done;
-    }
-    if ((cb = cbuf_new()) == NULL){
-	clicon_err(OE_UNIX, 0, "cbuf_new");
-	goto done;
-    }
-    /* create absolute path */
-    cprintf(cb, "/%s:%s", xml_namespace(xn), xml_name(xn)); 
-    /* Find yang rpc statement, return yang rpc statement if found */
-    if (yang_abs_schema_nodeid(yspec, cbuf_get(cb), &yrpc) < 0)
-	goto done;
-    /* Check if found */
-    if (yrpc != NULL){
-	if ((yinput = yang_find((yang_node*)yrpc, Y_INPUT, NULL)) != NULL){
-	    xml_spec_set(xn, yinput); /* needed for xml_spec_populate */
-	    if (xml_apply(xn, CX_ELMNT, xml_spec_populate, yinput) < 0)
-		goto done;
-	    if (xml_apply(xn, CX_ELMNT, 
-			  (xml_applyfn_t*)xml_yang_validate_all, NULL) < 0)
-		goto done;
-	    if (xml_yang_validate_add(xn, NULL) < 0)
-		goto done;
-	}
-	/* 
-	 * 1. Check xn arguments with input statement.
-	 * 2. Send to backend as clicon_msg-encode()
-	 * 3. In backend to similar but there call actual backend
-	 */
-	if (clicon_rpc_netconf_xml(h, xml_parent(xn), xret, NULL) < 0)
-	    goto done;
-	/* Sanity check of outgoing XML */
-	if ((youtput = yang_find((yang_node*)yrpc, Y_OUTPUT, NULL)) != NULL){
-	    xoutput=xpath_first(*xret, "/");
-	    xml_spec_set(xoutput, youtput); /* needed for xml_spec_populate */
-	    if (xml_apply(xoutput, CX_ELMNT, xml_spec_populate, youtput) < 0)
-		goto done;
-	    if (xml_apply(xoutput, CX_ELMNT, 
-			  (xml_applyfn_t*)xml_yang_validate_all, NULL) < 0)
-		goto done;
-	    if (xml_yang_validate_add(xoutput, NULL) < 0)
-		goto done;
-	}
-	retval = 1; /* handled by callback */
-	goto done;
-    }
     retval = 0;
  done:
-    if (cb)
-	cbuf_free(cb);
     return retval;
 }
     

apps/netconf/netconf_rpc.c

@@ -135,7 +135,7 @@ netconf_get_config(clicon_handle h,
      cxobj      *xconf;
 
      if ((source = netconf_get_target(xn, "source")) == NULL){
-	 clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	 xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			  "<error-tag>missing-element</error-tag>"
 			  "<error-type>protocol</error-type>"
 			  "<error-severity>error</error-severity>"
@@ -163,7 +163,7 @@ netconf_get_config(clicon_handle h,
 	     /* xml_filter removes parts of xml tree not matching */
 	     if ((strcmp(xml_name(xfilterconf), xml_name(xconf))!=0) ||
 		 xml_filter(xfilterconf, xconf) < 0){
-		     clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+		     xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 				      "<error-tag>operation-failed</error-tag>"
 				      "<error-type>applicatio</error-type>"
 				      "<error-severity>error</error-severity>"
@@ -173,7 +173,7 @@ netconf_get_config(clicon_handle h,
 	 }
      }
      else{
-	 clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	 xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			  "<error-tag>operation-failed</error-tag>"
 			  "<error-type>applicatio</error-type>"
 			  "<error-severity>error</error-severity>"
@@ -241,7 +241,7 @@ get_edit_opts(cxobj               *xn,
     retval = 1; /* hunky dory */
     return retval;
  parerr: /* parameter error, xret set */
-    clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+    xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 		     "<error-tag>invalid-value</error-tag>"
 		     "<error-type>protocol</error-type>"
 		     "<error-severity>error</error-severity>"
@@ -317,7 +317,7 @@ netconf_edit_config(clicon_handle h,
     /* must have target, and it should be candidate */
     if ((target = netconf_get_target(xn, "target")) == NULL ||
 	strcmp(target, "candidate")){
-	clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			 "<error-tag>missing-element</error-tag>"
 			 "<error-type>protocol</error-type>"
 			 "<error-severity>error</error-severity>"
@@ -329,7 +329,7 @@ netconf_edit_config(clicon_handle h,
     if ((xfilter = xpath_first(xn, "filter")) != NULL) {
 	if ((ftype = xml_find_value(xfilter, "type")) != NULL)
 	    if (strcmp(ftype,"restconf")){
-		clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+		xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 				 "<error-tag>invalid-value</error-tag>"
 				 "<error-type>protocol</error-type>"
 				 "<error-severity>error</error-severity>"
@@ -339,7 +339,7 @@ netconf_edit_config(clicon_handle h,
     }
     if ((x = xpath_first(xn, "default-operation")) != NULL){
 	if (xml_operation(xml_body(x), &operation) < 0){
-	    clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	    xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			     "<error-tag>invalid-value</error-tag>"
 			     "<error-type>protocol</error-type>"
 			     "<error-severity>error</error-severity>"
@@ -353,7 +353,7 @@ netconf_edit_config(clicon_handle h,
 	goto ok;
     /* not supported opts */
     if (testopt!=TEST_THEN_SET || erropt!=STOP_ON_ERROR){
-	clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			 "<error-tag>operation-not-supported</error-tag>"
 			 "<error-type>protocol</error-type>"
 			 "<error-severity>error</error-severity>"
@@ -403,7 +403,7 @@ netconf_copy_config(clicon_handle h,
     char     *target; /* filenames */
 
     if ((source = netconf_get_target(xn, "source")) == NULL){
-	clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			 "<error-tag>missing-element</error-tag>"
 			 "<error-type>protocol</error-type>"
 			 "<error-severity>error</error-severity>"
@@ -412,7 +412,7 @@ netconf_copy_config(clicon_handle h,
 	goto ok;
     }
     if ((target = netconf_get_target(xn, "target")) == NULL){
-	clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			 "<error-tag>missing-element</error-tag>"
 			 "<error-type>protocol</error-type>"
 			 "<error-severity>error</error-severity>"
@@ -450,7 +450,7 @@ netconf_delete_config(clicon_handle h,
 
     if ((target = netconf_get_target(xn, "target")) == NULL ||
 	strcmp(target, "running")==0){
-	clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			 "<error-tag>missing-element</error-tag>"
 			 "<error-type>protocol</error-type>"
 			 "<error-severity>error</error-severity>"
@@ -486,7 +486,7 @@ netconf_lock(clicon_handle h,
     char    *target;
 
     if ((target = netconf_get_target(xn, "target")) == NULL){
-	 clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			  "<error-tag>missing-element</error-tag>"
 			  "<error-type>protocol</error-type>"
 			  "<error-severity>error</error-severity>"
@@ -566,7 +566,7 @@ netconf_get(clicon_handle h,
 	     /* xml_filter removes parts of xml tree not matching */
 	     if ((strcmp(xml_name(xfilterconf), xml_name(xconf))!=0) ||
 		 xml_filter(xfilterconf, xconf) < 0){
-		     clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+		 xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 				      "<error-tag>operation-failed</error-tag>"
 				      "<error-type>applicatio</error-type>"
 				      "<error-severity>error</error-severity>"
@@ -576,7 +576,7 @@ netconf_get(clicon_handle h,
 	 }
      }
      else{
-	 clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	 xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			  "<error-tag>operation-failed</error-tag>"
 			  "<error-type>applicatio</error-type>"
 			  "<error-severity>error</error-severity>"
@@ -627,7 +627,7 @@ netconf_kill_session(clicon_handle h,
     cxobj *xs;
 
     if ((xs = xpath_first(xn, "//session-id")) == NULL){
-	 clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			  "<error-tag>missing-element</error-tag>"
 			  "<error-type>protocol</error-type>"
 			  "<error-severity>error</error-severity>"
@@ -658,7 +658,7 @@ netconf_validate(clicon_handle h,
     char  *target;
 
     if ((target = netconf_get_target(xn, "source")) == NULL){
-	 clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 			  "<error-tag>missing-element</error-tag>"
 			  "<error-type>protocol</error-type>"
 			  "<error-severity>error</error-severity>"
@@ -826,7 +826,7 @@ netconf_create_subscription(clicon_handle h,
     if ((xfilter = xpath_first(xn, "//filter")) != NULL){
 	if ((ftype = xml_find_value(xfilter, "type")) != NULL){
 	    if (strcmp(ftype, "xpath") != 0){
-		clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+		xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 				 "<error-tag>operation-failed</error-tag>"
 				 "<error-type>application</error-type>"
 				 "<error-severity>error</error-severity>"
@@ -850,68 +850,195 @@ netconf_create_subscription(clicon_handle h,
     return retval;
 }
 
+/*! See if there is any callback registered for this tag
+ *
+ * @param[in]  h       clicon handle
+ * @param[in]  xn      Sub-tree (under xorig) at child of rpc: <rpc><xn></rpc>.
+ * @param[out] xret    Return XML, error or OK
+ *
+ * @retval -1   Error
+ * @retval  0   OK, not found handler.
+ * @retval  1   OK, handler called
+ */
+static int
+netconf_application_rpc(clicon_handle h,
+			cxobj        *xn, 
+			cxobj       **xret)
+{
+    int            retval = -1;
+    yang_spec     *yspec = NULL; /* application yspec */
+    yang_stmt     *yrpc = NULL;
+    yang_stmt     *yinput;
+    yang_stmt     *youtput;
+    cxobj         *xoutput;
+    cbuf          *cb = NULL;
+
+    /* First check system / netconf RPC:s */
+    if ((cb = cbuf_new()) == NULL){
+	clicon_err(OE_UNIX, 0, "cbuf_new");
+	goto done;
+    }
+    /* Find yang rpc statement, return yang rpc statement if found 
+       Check application RPC */
+    if ((yspec =  clicon_dbspec_yang(h)) == NULL){
+	clicon_err(OE_YANG, ENOENT, "No yang spec");
+	goto done;
+    }
+    cbuf_reset(cb);
+    //    if (xml_namespace(xn))
+	cprintf(cb, "/%s:%s", xml_namespace(xn), xml_name(xn));
+	//    else
+	//	cprintf(cb, "/%s", xml_name(xn)); /* XXX not accepdted by below */
+    /* Find yang rpc statement, return yang rpc statement if found */
+    if (yang_abs_schema_nodeid(yspec, cbuf_get(cb), &yrpc) < 0)
+	goto done;
+
+    /* Check if found */
+    if (yrpc != NULL){
+	if ((yinput = yang_find((yang_node*)yrpc, Y_INPUT, NULL)) != NULL){
+	    xml_spec_set(xn, yinput); /* needed for xml_spec_populate */
+	    if (xml_apply(xn, CX_ELMNT, xml_spec_populate, yinput) < 0)
+		goto done;
+	    if (xml_apply(xn, CX_ELMNT, 
+			  (xml_applyfn_t*)xml_yang_validate_all, NULL) < 0)
+		goto done;
+	    if (xml_yang_validate_add(xn, NULL) < 0)
+		goto done;
+	}
+	/* 
+	 * 1. Check xn arguments with input statement.
+	 * 2. Send to backend as clicon_msg-encode()
+	 * 3. In backend to similar but there call actual backend
+	 */
+	if (clicon_rpc_netconf_xml(h, xml_parent(xn), xret, NULL) < 0)
+	    goto done;
+	/* Sanity check of outgoing XML */
+	if ((youtput = yang_find((yang_node*)yrpc, Y_OUTPUT, NULL)) != NULL){
+	    xoutput=xpath_first(*xret, "/");
+	    xml_spec_set(xoutput, youtput); /* needed for xml_spec_populate */
+	    if (xml_apply(xoutput, CX_ELMNT, xml_spec_populate, youtput) < 0)
+		goto done;
+	    if (xml_apply(xoutput, CX_ELMNT, 
+			  (xml_applyfn_t*)xml_yang_validate_all, NULL) < 0)
+		goto done;
+	    if (xml_yang_validate_add(xoutput, NULL) < 0)
+		goto done;
+	}
+	retval = 1; /* handled by callback */
+	goto done;
+    }
+    retval = 0;
+ done:
+    if (cb)
+	cbuf_free(cb);
+    return retval;
+}
+    
+
+
 /*! The central netconf rpc dispatcher. Look at first tag and dispach to sub-functions.
  * Call plugin handler if tag not found. If not handled by any handler, return
  * error.
  * @param[in]  h       clicon handle
  * @param[in]  xn      Sub-tree (under xorig) at <rpc>...</rpc> level.
  * @param[out] xret    Return XML, error or OK
+ * @retval     0       OK, can also be netconf error 
+ * @retval    -1       Error, fatal
  */
 int
 netconf_rpc_dispatch(clicon_handle h,
 		     cxobj        *xn, 
 		     cxobj       **xret)
 {
+    int         retval = -1;
     cxobj      *xe;
-    int         ret = 0;
+    yang_spec  *yspec = NULL; 
     
+    /* Check incoming RPC against system / netconf RPC:s */
+    if ((yspec = clicon_netconf_yang(h)) == NULL){
+	clicon_err(OE_YANG, ENOENT, "No netconf yang spec");
+	goto done;
+    }
     xe = NULL;
     while ((xe = xml_child_each(xn, xe, CX_ELMNT)) != NULL) {
-	if (strcmp(xml_name(xe), "get-config") == 0)
+	if (strcmp(xml_name(xe), "get-config") == 0){
-	    return netconf_get_config(h, xe, xret);
+	    if (netconf_get_config(h, xe, xret) < 0)
-	else if (strcmp(xml_name(xe), "edit-config") == 0)
+		goto done;
-	    return netconf_edit_config(h, xe, xret);
+	}
-        else if (strcmp(xml_name(xe), "copy-config") == 0)
+	else if (strcmp(xml_name(xe), "edit-config") == 0){
-	    return netconf_copy_config(h, xe, xret);
+	    if (netconf_edit_config(h, xe, xret) < 0)
-	else if (strcmp(xml_name(xe), "delete-config") == 0)
+		goto done;
-	    return netconf_delete_config(h, xe, xret);
+	}
-	else if (strcmp(xml_name(xe), "lock") == 0) 
+        else if (strcmp(xml_name(xe), "copy-config") == 0){
-	    return netconf_lock(h, xe, xret);
+	    if (netconf_copy_config(h, xe, xret) < 0)
-	else if (strcmp(xml_name(xe), "unlock") == 0)
+		goto done;
-	    return netconf_unlock(h, xe, xret);
+	}
-	else if (strcmp(xml_name(xe), "get") == 0)
+	else if (strcmp(xml_name(xe), "delete-config") == 0){
-	    return netconf_get(h, xe, xret);
+	    if (netconf_delete_config(h, xe, xret) < 0)
-	else if (strcmp(xml_name(xe), "close-session") == 0)
+		goto done;
-	    return netconf_close_session(h, xe, xret);
+	}
-	else if (strcmp(xml_name(xe), "kill-session") == 0) 
+	else if (strcmp(xml_name(xe), "lock") == 0) {
-	    return netconf_kill_session(h, xe, xret);
+	    if (netconf_lock(h, xe, xret) < 0)
+		goto done;
+	}
+	else if (strcmp(xml_name(xe), "unlock") == 0){
+	    if (netconf_unlock(h, xe, xret) < 0)
+		goto done;
+	}
+	else if (strcmp(xml_name(xe), "get") == 0){
+	    if (netconf_get(h, xe, xret) < 0)
+		goto done;
+	}
+	else if (strcmp(xml_name(xe), "close-session") == 0){
+	    if (netconf_close_session(h, xe, xret) < 0)
+		goto done;
+	}
+	else if (strcmp(xml_name(xe), "kill-session") == 0) {
+	    if (netconf_kill_session(h, xe, xret) < 0)
+		goto done;
+	}
 	/* Validate capability :validate */
-	else if (strcmp(xml_name(xe), "validate") == 0)
+	else if (strcmp(xml_name(xe), "validate") == 0){
-	   return netconf_validate(h, xe, xret);
+	    if (netconf_validate(h, xe, xret) < 0)
+		goto done;
+	}
 	/* Candidate configuration capability :candidate */
-	else if (strcmp(xml_name(xe), "commit") == 0)
+	else if (strcmp(xml_name(xe), "commit") == 0){
-	    return netconf_commit(h, xe, xret);
+	    if (netconf_commit(h, xe, xret) < 0)
-	else if (strcmp(xml_name(xe), "discard-changes") == 0)
+		goto done;
-	    return netconf_discard_changes(h, xe, xret);
+	}
+	else if (strcmp(xml_name(xe), "discard-changes") == 0){
+	    if (netconf_discard_changes(h, xe, xret) < 0)
+		goto done;
+	}
 	/* RFC 5277 :notification */
-	else if (strcmp(xml_name(xe), "create-subscription") == 0)
+	else if (strcmp(xml_name(xe), "create-subscription") == 0){
-	    return netconf_create_subscription(h, xe, xret);
+	    if (netconf_create_subscription(h, xe, xret) < 0)
+		goto done;
+	}
 	/* Others */
-	else{
+	else {
-	    if ((ret = netconf_plugin_callbacks(h, xe, xret)) < 0)
+	    /* Look for local (client-side) netconf plugins. This feature may no 
-	       return -1;
+	     * longer be necessary as generic RPC:s should be handled by backend.
-	   if (ret == 0){ /* not handled by callback */
+	     */
-		clicon_xml_parse(xret, "<rpc-reply><rpc-error>"
+	    if ((retval = netconf_plugin_callbacks(h, xe, xret)) < 0)
+		goto done;
+	    if (retval == 0)
+		if ((retval = netconf_application_rpc(h, xe, xret)) < 0)
+		    goto done;
+	    if (retval == 0){ /* not handled by callback */
+		xml_parse_va(xret, NULL, "<rpc-reply><rpc-error>"
 				 "<error-tag>operation-failed</error-tag>"
 				 "<error-type>rpc</error-type>"
 				 "<error-severity>error</error-severity>"
 				 "<error-message>%s</error-message>"
 				 "<error-info>Not recognized</error-info>"
 				 "</rpc-error></rpc-reply>", xml_name(xe));
-		return 0;
+		goto done;
-	   }
+	    }
-	   
+	}
-       }
+    }
-   }
+    retval = 0;
-    return ret;
+ done:
+    return retval;
 }

apps/restconf/restconf_main.c

@@ -357,7 +357,7 @@ main(int    argc,
 	return -1;
 
     /* Parse yang database spec file */
-    if (yang_spec_main(h, NULL, 0) < 0)
+    if (yang_spec_main(h) == NULL)
 	goto done;
 
     if ((sockpath = clicon_option_str(h, "CLICON_RESTCONF_PATH")) == NULL){

apps/restconf/restconf_methods.c

@@ -369,14 +369,14 @@ api_data_post(clicon_handle h,
     for (i=0; i<pi; i++)
 	api_path = index(api_path+1, '/');
     /* Create config top-of-tree */
-    if ((xtop = xml_new("config", NULL)) == NULL)
+    if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
     if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0)
 	goto done;
     /* Parse input data as json or xml into xml */
     if (parse_xml){
-	if (clicon_xml_parse_str(data, &xdata) < 0){
+	if (xml_parse_string(data, NULL, &xdata) < 0){
 	    clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
 	    goto done;
 	}
@@ -388,7 +388,7 @@ api_data_post(clicon_handle h,
     /* Add xdata to xbot */
     x = NULL;
     while ((x = xml_child_each(xdata, x, CX_ELMNT)) != NULL) {
-	if ((xa = xml_new("operation", x)) == NULL)
+	if ((xa = xml_new("operation", x, NULL)) == NULL)
 	    goto done;
 	xml_type_set(xa, CX_ATTR);
 	if (xml_value_set(xa,  xml_operation2str(op)) < 0)
@@ -488,14 +488,14 @@ api_data_put(clicon_handle h,
     for (i=0; i<pi; i++)
 	api_path = index(api_path+1, '/');
     /* Create config top-of-tree */
-    if ((xtop = xml_new("config", NULL)) == NULL)
+    if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
     if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0)
 	goto done;
     /* Parse input data as json or xml into xml */
     if (parse_xml){
-	if (clicon_xml_parse_str(data, &xdata) < 0){
+	if (xml_parse_string(data, NULL, &xdata) < 0){
 	    clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
 	    goto done;
 	}
@@ -510,7 +510,7 @@ api_data_put(clicon_handle h,
 	goto done;
     }
     x = xml_child_i(xdata,0);
-    if ((xa = xml_new("operation", x)) == NULL)
+    if ((xa = xml_new("operation", x, NULL)) == NULL)
 	goto done;
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa,  xml_operation2str(op)) < 0)
@@ -613,12 +613,12 @@ api_data_delete(clicon_handle h,
     for (i=0; i<pi; i++)
 	api_path = index(api_path+1, '/');
     /* Create config top-of-tree */
-    if ((xtop = xml_new("config", NULL)) == NULL)
+    if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
     if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0)
 	goto done;
-    if ((xa = xml_new("operation", xbot)) == NULL)
+    if ((xa = xml_new("operation", xbot, NULL)) == NULL)
 	goto done;
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa,  xml_operation2str(op)) < 0)
@@ -721,7 +721,7 @@ api_operation_post(clicon_handle h,
      * eg <rpc><fib-route><name>
      */
     /* Create config top-of-tree */
-    if ((xtop = xml_new("rpc", NULL)) == NULL)
+    if ((xtop = xml_new("rpc", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
     if (api_path2xml(oppath, yspec, xtop, 1, &xbot, &y) < 0)
@@ -729,7 +729,7 @@ api_operation_post(clicon_handle h,
     if (data && strlen(data)){
 	/* Parse input data as json or xml into xml */
 	if (parse_xml){
-	    if (clicon_xml_parse_str(data, &xdata) < 0){
+	    if (xml_parse_string(data, NULL, &xdata) < 0){
 		clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
 		goto done;
 	    }
@@ -768,7 +768,7 @@ api_operation_post(clicon_handle h,
 	
 	if ((cookie = FCGX_GetParam("HTTP_COOKIE", r->envp)) != NULL &&
 	    get_user_cookie(cookie, "c-user", &cookieval) ==0){
-	    if ((xa = xml_new("id", xtop)) == NULL)
+	    if ((xa = xml_new("id", xtop, NULL)) == NULL)
 		goto done;
 	    xml_type_set(xa, CX_ATTR);
 	    if (xml_value_set(xa,  cookieval) < 0)

build-root/scripts/version

@@ -0,0 +1 @@
+../../extras/scripts/version

configure

@@ -632,6 +632,7 @@ CPP
 OBJEXT
 EXEEXT
 ac_ct_CC
+with_xml_compat
 with_config_compat
 with_startup_compat
 with_keyvalue
@@ -659,6 +660,7 @@ build_os
 build_vendor
 build_cpu
 build
+CLIGEN_PREFIX
 CLIGEN_VERSION
 CLIXON_VERSION_MINOR
 CLIXON_VERSION_MAJOR
@@ -712,6 +714,7 @@ with_keyvalue
 with_qdbm
 with_startup_compat
 with_config_compat
+with_xml_compat
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1351,7 +1354,8 @@ Optional Packages:
   --with-keyvalue         enable support for key-value xmldb datastore
   --with-qdbm=dir         Use QDBM here, if keyvalue
   --with-startup-compat   Backward compatibility of backend startup commands
-  --with-config-compat    Backward compatibility of ocnfiguration file
+  --with-config-compat    Backward compatibility of configuration file
+  --with-xml-compat       Backward compatibility of XML API
 
 Some influential environment variables:
   CC          C compiler command
@@ -2153,11 +2157,16 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 : ${CFLAGS="-O2"}
 
 CLIXON_VERSION_MAJOR="3"
-CLIXON_VERSION_MINOR="3"
+CLIXON_VERSION_MINOR="4"
-CLIXON_VERSION_PATCH="3"
+CLIXON_VERSION_PATCH="0"
-CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\""
+CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\""
 # Fix to specific version (eg 3.5) or head (3)
 CLIGEN_VERSION="3"
+if test "$prefix" = "NONE"; then
+	CLIGEN_PREFIX="$ac_default_prefix"
+else
+	CLIGEN_PREFIX="$prefix"
+fi
 
 ac_config_headers="$ac_config_headers include/clixon_config.h lib/clixon/clixon.h"
 
@@ -2347,8 +2356,12 @@ test -n "$target_alias" &&
 
  # If yes, compile apps/restconf
  # If yes, compile datastore/keyvalue
- # If yes, backward compatible backend startup
+# If yes, backward compatible with 3.3.2 backend startup
- # If yes, backward compatible .conf configuration
+
+# If yes, backward compatible with 3.3.2 .conf configuration
+
+# If yes, backward compatible with 3.3.3 XML api new and parse functions
+
 
 #
 ac_ext=c
@@ -3560,6 +3573,7 @@ if test "${with_cligen}"; then
   echo "Using CLIGEN here: ${with_cligen}"
   CPPFLAGS="-I${with_cligen}/include ${CPPFLAGS}"
   LDFLAGS="-L${with_cligen}/lib ${LDFLAGS}"
+  test -d "$with_cligen" && CLIGEN_PREFIX="$with_cligen"
 fi
 
 
@@ -4042,9 +4056,12 @@ else
   as_fn_error $? "libqdbm-dev required" "$LINENO" 5
 fi
 
+   ac_config_files="$ac_config_files datastore/keyvalue/Makefile"
+
 fi
 
-# This is for backward compatibility of backend startup commands
+# This is for backward compatibility of backend startup commands in 3.3.3
+# Will be removed in 3.4.0
 
 # Check whether --with-startup_compat was given.
 if test "${with_startup_compat+set}" = set; then :
@@ -4061,7 +4078,8 @@ _ACEOF
 
 fi
 
-# This is for backward compatibility of .conf configuration file
+# This is for backward compatibility of .conf configuration file in 3.3.3
+# Will be removed in 3.4.0
 
 # Check whether --with-config_compat was given.
 if test "${with_config_compat+set}" = set; then :
@@ -4078,6 +4096,24 @@ _ACEOF
 
 fi
 
+# This is for backward compatibility of XML create and parse API in 3.4.0
+# Will be removed in 3.5.0
+
+# Check whether --with-xml_compat was given.
+if test "${with_xml_compat+set}" = set; then :
+  withval=$with_xml_compat;
+else
+  with_xml_compat=no
+fi
+
+if test "x${with_xml_compat}" == xyes; then
+
+cat >>confdefs.h <<_ACEOF
+#define XML_COMPAT $with_xml_compat
+_ACEOF
+
+fi
+
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for crypt in -lcrypt" >&5
 $as_echo_n "checking for crypt in -lcrypt... " >&6; }
 if ${ac_cv_lib_crypt_crypt+:} false; then :
@@ -4325,7 +4361,8 @@ _ACEOF
 
 
 
-ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/docker/Makefile docker/Makefile docker/cli/Makefile docker/cli/Dockerfile docker/backend/Makefile docker/backend/Dockerfile docker/netconf/Makefile docker/netconf/Dockerfile datastore/Makefile datastore/keyvalue/Makefile datastore/text/Makefile yang/Makefile doc/Makefile"
+# See also datastore/keyvalue/Makefile in with_keyvalue clause above
+ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/docker/Makefile extras/rpm/Makefile docker/Makefile docker/cli/Makefile docker/cli/Dockerfile docker/backend/Makefile docker/backend/Dockerfile docker/netconf/Makefile docker/netconf/Dockerfile datastore/Makefile datastore/text/Makefile yang/Makefile doc/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -5018,6 +5055,7 @@ do
   case $ac_config_target in
     "include/clixon_config.h") CONFIG_HEADERS="$CONFIG_HEADERS include/clixon_config.h" ;;
     "lib/clixon/clixon.h") CONFIG_HEADERS="$CONFIG_HEADERS lib/clixon/clixon.h" ;;
+    "datastore/keyvalue/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/keyvalue/Makefile" ;;
     "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;;
     "lib/Makefile") CONFIG_FILES="$CONFIG_FILES lib/Makefile" ;;
     "lib/src/Makefile") CONFIG_FILES="$CONFIG_FILES lib/src/Makefile" ;;
@@ -5032,6 +5070,7 @@ do
     "etc/clixonrc") CONFIG_FILES="$CONFIG_FILES etc/clixonrc" ;;
     "example/Makefile") CONFIG_FILES="$CONFIG_FILES example/Makefile" ;;
     "example/docker/Makefile") CONFIG_FILES="$CONFIG_FILES example/docker/Makefile" ;;
+    "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;;
     "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;;
     "docker/cli/Makefile") CONFIG_FILES="$CONFIG_FILES docker/cli/Makefile" ;;
     "docker/cli/Dockerfile") CONFIG_FILES="$CONFIG_FILES docker/cli/Dockerfile" ;;
@@ -5040,7 +5079,6 @@ do
     "docker/netconf/Makefile") CONFIG_FILES="$CONFIG_FILES docker/netconf/Makefile" ;;
     "docker/netconf/Dockerfile") CONFIG_FILES="$CONFIG_FILES docker/netconf/Dockerfile" ;;
     "datastore/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/Makefile" ;;
-    "datastore/keyvalue/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/keyvalue/Makefile" ;;
     "datastore/text/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/text/Makefile" ;;
     "yang/Makefile") CONFIG_FILES="$CONFIG_FILES yang/Makefile" ;;
     "doc/Makefile") CONFIG_FILES="$CONFIG_FILES doc/Makefile" ;;

configure.ac

@@ -42,11 +42,16 @@ AC_INIT(lib/clixon/clixon.h.in)
 : ${CFLAGS="-O2"}
 
 CLIXON_VERSION_MAJOR="3"
-CLIXON_VERSION_MINOR="3"
+CLIXON_VERSION_MINOR="4"
-CLIXON_VERSION_PATCH="3"
+CLIXON_VERSION_PATCH="0"
-CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\""
+CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\""
 # Fix to specific version (eg 3.5) or head (3) 
 CLIGEN_VERSION="3" 
+if test "$prefix" = "NONE"; then
+	CLIGEN_PREFIX="$ac_default_prefix"
+else
+	CLIGEN_PREFIX="$prefix"
+fi
 
 AC_CONFIG_HEADERS([include/clixon_config.h lib/clixon/clixon.h])
 
@@ -62,6 +67,7 @@ AC_SUBST(CLIXON_VERSION_STRING)
 AC_SUBST(CLIXON_VERSION_MAJOR)
 AC_SUBST(CLIXON_VERSION_MINOR)
 AC_SUBST(CLIGEN_VERSION) # Bind to specific CLIgen version
+AC_SUBST(CLIGEN_PREFIX)
 
 AC_MSG_RESULT(CLIXON version is ${CLIXON_VERSION})
 
@@ -80,8 +86,12 @@ AC_SUBST(AR)
 AC_SUBST(RANLIB)
 AC_SUBST(with_restconf) # If yes, compile apps/restconf
 AC_SUBST(with_keyvalue) # If yes, compile datastore/keyvalue
-AC_SUBST(with_startup_compat) # If yes, backward compatible backend startup
+# If yes, backward compatible with 3.3.2 backend startup
-AC_SUBST(with_config_compat) # If yes, backward compatible .conf configuration
+AC_SUBST(with_startup_compat) 
+# If yes, backward compatible with 3.3.2 .conf configuration
+AC_SUBST(with_config_compat) 
+# If yes, backward compatible with 3.3.3 XML api new and parse functions
+AC_SUBST(with_xml_compat) 
 
 #
 AC_PROG_CC()
@@ -122,6 +132,7 @@ if test "${with_cligen}"; then
   echo "Using CLIGEN here: ${with_cligen}"
   CPPFLAGS="-I${with_cligen}/include ${CPPFLAGS}"
   LDFLAGS="-L${with_cligen}/lib ${LDFLAGS}"
+  test -d "$with_cligen" && CLIGEN_PREFIX="$with_cligen"
 fi
 
 AC_CHECK_HEADERS(cligen/cligen.h,, AC_MSG_ERROR(cligen missing. Try: git clone https://github.com/olofhagsand/cligen.git))
@@ -154,9 +165,11 @@ if test "x${with_keyvalue}" == xyes; then
    # Problem: depot.h may be in qdbm/depot.h. 
    AC_CHECK_HEADERS(depot.h,,[AC_CHECK_HEADERS(qdbm/depot.h,,AC_MSG_ERROR(libqdbm-dev required))])
    AC_CHECK_LIB(qdbm, dpopen,, AC_MSG_ERROR(libqdbm-dev required))
+   AC_CONFIG_FILES(datastore/keyvalue/Makefile)
 fi
 
-# This is for backward compatibility of backend startup commands
+# This is for backward compatibility of backend startup commands in 3.3.3
+# Will be removed in 3.4.0
 AC_ARG_WITH([startup_compat],
             [AS_HELP_STRING([--with-startup-compat],[Backward compatibility of backend startup commands])],
 	    [],
@@ -165,15 +178,27 @@ if test "x${with_startup_compat}" == xyes; then
    AC_DEFINE_UNQUOTED(BACKEND_STARTUP_COMPAT, $with_startup_compat, [Backward compatible backend startup command-line options])
 fi
 
-# This is for backward compatibility of .conf configuration file
+# This is for backward compatibility of .conf configuration file in 3.3.3
+# Will be removed in 3.4.0
 AC_ARG_WITH([config_compat],
-            [AS_HELP_STRING([--with-config-compat],[Backward compatibility of ocnfiguration file])],
+            [AS_HELP_STRING([--with-config-compat],[Backward compatibility of configuration file])],
 	    [],
             [with_config_compat=no])
 if test "x${with_config_compat}" == xyes; then
    AC_DEFINE_UNQUOTED(CONFIG_COMPAT, $with_config_compat, [Backward compatible of .conf configuration files])
 fi
 
+# Clixon 3.4.0 changes XML creation and parse API
+# Set this for backward compat and migration. 
+# Will be removed in 3.5.0
+AC_ARG_WITH([xml_compat],
+            [AS_HELP_STRING([--with-xml-compat],[Backward compatibility of XML API])],
+	    [],
+            [with_xml_compat=no])
+if test "x${with_xml_compat}" == xyes; then
+   AC_DEFINE_UNQUOTED(XML_COMPAT, $with_xml_compat, [Backward compatible of XML API])
+fi
+
 AC_CHECK_LIB(crypt, crypt)
 AC_CHECK_HEADERS(crypt.h)
 
@@ -197,6 +222,7 @@ AC_DEFINE_UNQUOTED(CLIXON_DATADIR, "${prefix}/share/clixon", [Clixon data dir fo
 
 AH_BOTTOM([#include <clixon_custom.h>])
 
+# See also datastore/keyvalue/Makefile in with_keyvalue clause above
 AC_OUTPUT(Makefile
 	  lib/Makefile
 	  lib/src/Makefile 
@@ -211,6 +237,7 @@ AC_OUTPUT(Makefile
 	  etc/clixonrc
 	  example/Makefile
 	  example/docker/Makefile
+	  extras/rpm/Makefile
 	  docker/Makefile 
 	  docker/cli/Makefile 
 	  docker/cli/Dockerfile
@@ -219,7 +246,6 @@ AC_OUTPUT(Makefile
 	  docker/netconf/Makefile 
 	  docker/netconf/Dockerfile
 	  datastore/Makefile
-	  datastore/keyvalue/Makefile
 	  datastore/text/Makefile
 	  yang/Makefile
 	  doc/Makefile 

datastore/datastore_client.c

@@ -88,7 +88,8 @@ usage(char *argv0)
 		"\t-y <dir>\tYang directory (where modules are stored). Mandatory\n"
 		"\t-m <module>\tYang module. Mandatory\n"
 		"and command is either:\n"
-		"\tget <xpath>\n"
+		"\tget [<xpath>]\n"
+ 	        "\tmget <nr> [<xpath>]\n"
 		"\tput (merge|replace|create|delete|remove) <xml>\n"
 		"\tcopy <todb>\n"
 		"\tlock <pid>\n"
@@ -121,6 +122,8 @@ main(int argc, char **argv)
     int                 pid;
     enum operation_type op;
     cxobj              *xt = NULL;
+    int                 i;
+    char               *xpath;
 
     /* In the startup, logs to stderr & debug flag set later */
     clicon_log_init(__PROGRAM__, LOG_INFO, CLICON_LOG_STDERR); 
@@ -213,12 +216,38 @@ main(int argc, char **argv)
     if (strcmp(cmd, "get")==0){
 	if (argc != 1 && argc != 2)
 	    usage(argv0);
-	if (xmldb_get(h, db, argc==2?argv[1]:"/", 0, &xt) < 0)
+	if (argc==2)
+	    xpath = argv[1];
+	else
+	    xpath = "/";
+	if (xmldb_get(h, db, xpath, 0, &xt) < 0)
 	    goto done;
 	clicon_xml2file(stdout, xt, 0, 0);	
 	
 	fprintf(stdout, "\n");
     }
+    else if (strcmp(cmd, "mget")==0){
+	int nr;
+	if (argc != 2 && argc != 3)
+	    usage(argv0);
+	nr = atoi(argv[1]);
+	if (argc==3)
+	    xpath = argv[2];
+	else
+	    xpath = "/";
+	for (i=0;i<nr;i++){
+	    if (xmldb_get(h, db, xpath, 0, &xt) < 0)
+		goto done;
+	    if (xt == NULL){
+		clicon_err(OE_DB, 0, "xt is NULL");
+		goto done;
+	    }
+	    clicon_xml2file(stdout, xt, 0, 0);
+	    xml_free(xt);
+	    xt = NULL;
+	}
+	fprintf(stdout, "\n");
+    }
     else if (strcmp(cmd, "put")==0){
 	if (argc != 3){
 	    clicon_err(OE_DB, 0, "Unexpected nr of args: %d", argc);
@@ -228,7 +257,7 @@ main(int argc, char **argv)
 	    clicon_err(OE_DB, 0, "Unrecognized operation: %s", argv[1]);
 	    usage(argv0);
 	}
-	if (clicon_xml_parse_str(argv[2], &xt) < 0)
+	if (xml_parse_string(argv[2], NULL, &xt) < 0)
 	    goto done;
 	if (xml_rootchild(xt, 0, &xt) < 0)
 	    goto done;

datastore/keyvalue/clixon_keyvalue.c

@@ -201,14 +201,7 @@ append_listkeys(cbuf      *ckey,
     char      *bodyenc;
     int        i=0;
 
-    if ((ykey = yang_find((yang_node*)ys, Y_KEY, NULL)) == NULL){
+    cvk = ys->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
-	clicon_err(OE_XML, errno, "%s: List statement \"%s\" has no key", 
-		   __FUNCTION__, ys->ys_argument);
-	goto done;
-    }
-    /* The value is a list of keys: <key>[ <key>]*  */
-    if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
-	goto done;
     cvi = NULL;
     /* Iterate over individual keys  */
     while ((cvi = cvec_each(cvk, cvi)) != NULL) {
@@ -230,8 +223,6 @@ append_listkeys(cbuf      *ckey,
     }
     retval = 0;
  done:
-    if (cvk)
-	cvec_free(cvk);
     return retval;
 }
 

datastore/text/clixon_xmldb_text.c

@@ -30,6 +30,16 @@
   the terms of any one of the Apache License version 2 or the GPL.
 
   ***** END LICENSE BLOCK *****
+1000 entries
+valgrind --tool=callgrind datastore_client -d candidate -b /tmp/text -p ../datastore/text/text.so -y /tmp -m ietf-ip mget 300 /x/y[a=574][b=574] > /dev/null
+  xml_copy_marked 87% 200x 
+    yang_key_match 81% 600K
+      yang_arg2cvec 52% 400K
+      cvecfree      23% 400K
+
+10000 entries
+valgrind --tool=callgrind datastore_client -d candidate -b /tmp/text -p ../datastore/text/text.so -y /tmp -m ietf-ip mget 10 /x/y[a=574][b=574] > /dev/null
+
  */
 
 #ifdef HAVE_CONFIG_H
@@ -71,9 +81,28 @@ struct text_handle {
     int            th_magic;    /* magic */
     char          *th_dbdir;    /* Directory of database files */
     yang_spec     *th_yangspec; /* Yang spec if this datastore */
-    clicon_hash_t *th_dbs;      /* Hash of databases */
+    clicon_hash_t *th_dbs;      /* Hash of db_elements. key is dbname */
 };
 
+/* Struct per database in hash */
+struct db_element{
+    int    de_pid;
+    cxobj *de_xml;
+};
+
+/* Keep datastore text in memory so that get operation need only read memory.
+ * Write to file on modification or file change.
+ * Assumes single backend
+ * XXX MOVE TO HANDLE all three below
+ */
+static int xmltree_cache = 1;
+
+/* Format */
+static char *xml_format = "xml";
+
+/* Store xml/json pretty-printed. Or not. */
+static int xml_pretty = 1;
+
 /*! Check struct magic number for sanity checks
  * return 0 if OK, -1 if fail.
  */
@@ -161,12 +190,28 @@ text_disconnect(xmldb_handle xh)
 {
     int                 retval = -1;
     struct text_handle *th = handle(xh);
+    struct db_element  *de;
+    char              **keys = NULL;
+    size_t              klen;
+    int                 i;
 	
     if (th){
 	if (th->th_dbdir)
 	    free(th->th_dbdir);
-	if (th->th_dbs)
+	if (th->th_dbs){
+	    if (xmltree_cache){
+		if ((keys = hash_keys(th->th_dbs, &klen)) == NULL)
+		    return 0;
+		for(i = 0; i < klen; i++) 
+		    if ((de = hash_value(th->th_dbs, keys[i], NULL)) != NULL){
+			if (de->de_xml)
+			    xml_free(de->de_xml);
+		    }
+		if (keys)
+		    free(keys);
+	    }
 	    hash_free(th->th_dbs);
+	}
 	free(th);
     }
     retval = 0;
@@ -193,6 +238,12 @@ text_getopt(xmldb_handle xh,
 	*value = th->th_yangspec;
     else if (strcmp(optname, "dbdir") == 0)
 	*value = th->th_dbdir;
+    else if (strcmp(optname, "xml_cache") == 0)
+	*value = &xmltree_cache;
+    else if (strcmp(optname, "format") == 0)
+	*value = xml_format;
+    else if (strcmp(optname, "pretty") == 0)
+	*value = &xml_pretty;
     else{
 	clicon_err(OE_PLUGIN, 0, "Option %s not implemented by plugin", optname);
 	goto done;
@@ -202,9 +253,9 @@ text_getopt(xmldb_handle xh,
     return retval;
 }
 
-/*! Set value of generic plugin option. Type of value is givenby context
+/*! Set value of generic plugin option. Type of value is given by context
  * @param[in]  xh      XMLDB handle
- * @param[in]  optname Option name
+ * @param[in]  optname Option name: yangspec, xml_cache, format, prettyprint
  * @param[in]  value   Value of option
  * @retval     0       OK
  * @retval    -1       Error
@@ -225,6 +276,22 @@ text_setopt(xmldb_handle xh,
 	    goto done;
 	}
     }
+    else if (strcmp(optname, "xml_cache") == 0){
+	xmltree_cache = (intptr_t)value;
+    }
+    else if (strcmp(optname, "format") == 0){
+	if (strcmp(value,"xml")==0)
+	    xml_format = "xml";
+	else if (strcmp(value,"json")==0)
+	    xml_format = "json";
+	else{
+	    clicon_err(OE_PLUGIN, 0, "Option %s unrecognized format: %s", optname, value);
+	    goto done;
+	}
+    }
+    else if (strcmp(optname, "pretty") == 0){
+	xml_pretty = (intptr_t)value;
+    }
     else{
 	clicon_err(OE_PLUGIN, 0, "Option %s not implemented by plugin", optname);
 	goto done;
@@ -234,7 +301,6 @@ text_setopt(xmldb_handle xh,
     return retval;
 }
 
-
 /*! Ensure that xt only has a single sub-element and that is "config" 
  */
 static int
@@ -273,6 +339,78 @@ singleconfigroot(cxobj  *xt,
     return retval;
 }
 
+/*! Given XML tree x0 with marked nodes, copy marked nodes to new tree x1
+ * Two marks are used: XML_FLAG_MARK and XML_FLAG_CHANGE
+ *
+ * The algorithm works as following:
+ * (1) Copy individual nodes marked with XML_FLAG_CHANGE 
+ * until nodes marked with XML_FLAG_MARK are reached, where 
+ * (2) the complete subtree of that node is copied. 
+ * (3) Special case: key nodes in lists are copied if any node in list is marked
+ */
+static int
+xml_copy_marked(cxobj *x0, 
+		cxobj *x1)
+{
+    int        retval = -1;
+    int        mark;
+    cxobj     *x;
+    cxobj     *xcopy;
+    int        iskey;
+    yang_stmt *yt;
+    char      *name;
+
+    assert(x0 && x1);
+    yt = xml_spec(x0); /* can be null */
+    /* Go through children to detect any marked nodes:
+     * (3) Special case: key nodes in lists are copied if any 
+     * node in list is marked
+     */
+    mark = 0;
+    x = NULL;
+    while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) {
+	if (xml_flag(x, XML_FLAG_MARK|XML_FLAG_CHANGE)){
+	    mark++;
+	    break;
+	}
+    }
+    x = NULL;
+    while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) {
+	name = xml_name(x);
+	if (xml_flag(x, XML_FLAG_MARK)){
+	    /* (2) the complete subtree of that node is copied. */
+	    if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL)
+		goto done;
+	    if (xml_copy(x, xcopy) < 0) 
+		goto done;
+	    continue; 
+	}
+	if (xml_flag(x, XML_FLAG_CHANGE)){
+	    /*  Copy individual nodes marked with XML_FLAG_CHANGE */
+	    if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL)
+		goto done;
+	    if (xml_copy_marked(x, xcopy) < 0) /*  */
+		goto done;
+	}
+	/* (3) Special case: key nodes in lists are copied if any 
+	 * node in list is marked */
+	if (mark && yt && yt->ys_keyword == Y_LIST){
+	    /* XXX: I think yang_key_match is suboptimal here */
+	    if ((iskey = yang_key_match((yang_node*)yt, name)) < 0)
+		goto done;
+	    if (iskey){
+		if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL)
+		    goto done;
+		if (xml_copy(x, xcopy) < 0) 
+		    goto done;
+	    }
+	}
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Get content of database using xpath. return a set of matching sub-trees
  * The function returns a minimal tree that includes all sub-trees that match
  * xpath.
@@ -295,57 +433,94 @@ text_get(xmldb_handle xh,
     size_t          xlen;
     int             i;
     struct text_handle *th = handle(xh);
+    struct db_element *de = NULL;
 
-    if (text_db2file(th, db, &dbfile) < 0)
-	goto done;
-    if (dbfile==NULL){
-	clicon_err(OE_XML, 0, "dbfile NULL");
-	goto done;
-    }
     if ((yspec = th->th_yangspec) == NULL){
 	clicon_err(OE_YANG, ENOENT, "No yang spec");
 	goto done;
     }
-    if ((fd = open(dbfile, O_RDONLY)) < 0){
+    if (xmltree_cache){
-	clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
+	if ((de = hash_value(th->th_dbs, db, NULL)) != NULL)
-	goto done;
+	    xt = de->de_xml; 
     }
-    /* Parse file into XML tree */
+    if (xt == NULL){
-    if ((clicon_xml_parse_file(fd, &xt, "</config>")) < 0)
+	if (text_db2file(th, db, &dbfile) < 0)
-	goto done;
-    /* Always assert a top-level called "config". 
-       To ensure that, deal with two cases:
-       1. File is empty <top/> -> rename top-level to "config" */
-    if (xml_child_nr(xt) == 0){ 
-	if (xml_name_set(xt, "config") < 0)
 	    goto done;
-    }
+	if (dbfile==NULL){
-    /* 2. File is not empty <top><config>...</config></top> -> replace root */
+	    clicon_err(OE_XML, 0, "dbfile NULL");
-    else{ 
-	/* There should only be one element and called config */
-	if (singleconfigroot(xt, &xt) < 0)
 	    goto done;
-    }
+	}
+	if ((fd = open(dbfile, O_RDONLY)) < 0){
+	    clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
+	    goto done;
+	}    
+	/* Parse file into XML tree */
+	if (strcmp(xml_format,"json")==0){
+	    if ((json_parse_file(fd, yspec, &xt)) < 0)
+		goto done;
+	}
+	else if ((xml_parse_file(fd, "</config>", yspec, &xt)) < 0)
+	    goto done;
+	/* Always assert a top-level called "config". 
+	   To ensure that, deal with two cases:
+	   1. File is empty <top/> -> rename top-level to "config" */
+	if (xml_child_nr(xt) == 0){ 
+	    if (xml_name_set(xt, "config") < 0)
+		goto done;     
+	}
+	/* 2. File is not empty <top><config>...</config></top> -> replace root */
+	else{ 
+	    /* There should only be one element and called config */
+	    if (singleconfigroot(xt, &xt) < 0)
+		goto done;
+	}
+    } /* xt == NULL */
     /* Here xt looks like: <config>...</config> */
-    /* Add yang specification backpointer to all XML nodes */
-    if (xml_apply(xt, CX_ELMNT, xml_spec_populate, yspec) < 0)
-	goto done;
 
     if (xpath_vec(xt, xpath?xpath:"/", &xvec, &xlen) < 0)
 	goto done;
 
-    /* If vectors are specified then mark the nodes found and
+    /* If vectors are specified then mark the nodes found with all ancestors
-     * then filter out everything else,
+     * and filter out everything else,
      * otherwise return complete tree.
      */
-    if (xvec != NULL){
+    if (xvec != NULL)
-	for (i=0; i<xlen; i++)
+	for (i=0; i<xlen; i++){
 	    xml_flag_set(xvec[i], XML_FLAG_MARK);
-    }
+	    if (xmltree_cache)
-    /* Remove everything that is not marked */
+		xml_apply_ancestor(xvec[i], (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
-    if (!xml_flag(xt, XML_FLAG_MARK))
+	}
-	if (xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL) < 0)
+
+    if (xmltree_cache){
+	/* Copy the matching parts of the (relevant) XML tree.
+	 * If cache was NULL, also write to datastore cache
+	 */
+	cxobj *x1;
+	struct db_element de0 = {0,};
+
+	if (de != NULL)
+	    de0 = *de;
+
+	x1 = xml_new(xml_name(xt), NULL, xml_spec(xt));
+	/* Copy everything that is marked */
+	if (xml_copy_marked(xt, x1) < 0)
 	    goto done;
+	if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
+	    goto done;
+	if (xml_apply(x1, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
+	    goto done;
+	if (de0.de_xml == NULL){
+	    de0.de_xml = xt;
+	    hash_add(th->th_dbs, db, &de0, sizeof(de0));
+	}
+	xt = x1;
+    }
+    else{
+	/* Remove everything that is not marked */
+	if (!xml_flag(xt, XML_FLAG_MARK))
+	    if (xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL) < 0)
+		goto done;
+    }
     /* reset flag */
     if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)XML_FLAG_MARK) < 0)
 	goto done;
@@ -360,13 +535,11 @@ text_get(xmldb_handle xh,
     if (xml_apply(xt, CX_ELMNT, xml_default, NULL) < 0)
 	goto done;
     /* Order XML children according to YANG */
-    if (xml_apply(xt, CX_ELMNT, xml_order, NULL) < 0)
+    if (!xml_child_sort && xml_apply(xt, CX_ELMNT, xml_order, NULL) < 0)
-	goto done;
-
-#if (XML_CHILD_HASH==1)
-    /* Add hash */
-    if (xml_apply0(xt, CX_ELMNT, xml_hash_op, (void*)1) < 0)
 	goto done;
+#if 0 /* debug */
+    if (xml_child_sort && xml_apply0(xt, -1, xml_sort_verify, NULL) < 0)
+	clicon_log(LOG_NOTICE, "%s: verify failed #2", __FUNCTION__);
 #endif
     if (debug>1)
     	clicon_xml2file(stderr, xt, 0, 1);
@@ -433,7 +606,7 @@ text_modify(cxobj              *x0,
 	case OP_REPLACE:
 	    if (x0==NULL){
 		//		int iamkey=0;
-		if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL)
+		if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		    goto done;
 #if 0
 		/* If it is key I dont want to mark it */
@@ -445,24 +618,20 @@ text_modify(cxobj              *x0,
 #endif
 		    xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */
 		if (x1bstr){ /* empty type does not have body */
-		    if ((x0b = xml_new("body", x0)) == NULL)
+		    if ((x0b = xml_new("body", x0, NULL)) == NULL)
 			goto done; 
 		    xml_type_set(x0b, CX_BODY);
 		}
 	    }
 	    if (x1bstr){
 		if ((x0b = xml_body_get(x0)) == NULL){
-		    if ((x0b = xml_new("body", x0)) == NULL)
+		    if ((x0b = xml_new("body", x0, NULL)) == NULL)
 			goto done; 
 		    xml_type_set(x0b, CX_BODY);
 		}
 		if (xml_value_set(x0b, x1bstr) < 0)
 		    goto done;
 	    }
-#if (XML_CHILD_HASH==1)
-	    if (xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0)
-		goto done;
-#endif
 	    break;
 	case OP_DELETE:
 	    if (x0==NULL){
@@ -500,22 +669,18 @@ text_modify(cxobj              *x0,
 		if (x0){
 		    xml_purge(x0);
 		}
-		if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL)
+		if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		    goto done;
 		if (xml_copy(x1, x0) < 0)
 		    goto done;
 		break;
 	    }
 	    if (x0==NULL){
-		if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL)
+		if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		    goto done;
 		if (op==OP_NONE)
 		    xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */
 	    }
-#if (XML_CHILD_HASH==1)
-	    if (xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0)
-		goto done;
-#endif
 	    /* First pass: mark existing children in base */
 	    /* Loop through children of the modification tree */
 	    if ((x0vec = calloc(xml_child_nr(x1), sizeof(x1))) == NULL){
@@ -561,6 +726,7 @@ text_modify(cxobj              *x0,
 	} /* CONTAINER switch op */
     } /* else Y_CONTAINER  */
     // ok:
+    xml_sort(x0p, NULL);
     retval = 0;
  done:
     if (x0vec)
@@ -636,10 +802,6 @@ text_modify_top(cxobj              *x0,
 
 /*! For containers without presence and no children, remove
  * @param[in]   x       XML tree node
- * @note This should really be unnecessary since yspec should be set on creation
- * @code
- * xml_apply(xc, CX_ELMNT, xml_spec_populate, yspec)
- * @endcode
  * See section 7.5.1 in rfc6020bis-02.txt:
  * No presence:
  * those that exist only for organizing the hierarchy of data nodes:
@@ -689,14 +851,14 @@ text_put(xmldb_handle        xh,
     struct text_handle *th = handle(xh);
     char               *dbfile = NULL;
     int                 fd = -1;
+    FILE               *f = NULL;
     cbuf               *cb = NULL;
     yang_spec          *yspec;
     cxobj              *x0 = NULL;
+    struct db_element  *de = NULL;
     
-    if (text_db2file(th, db, &dbfile) < 0)
+    if ((yspec =  th->th_yangspec) == NULL){
-	goto done;
+	clicon_err(OE_YANG, ENOENT, "No yang spec");
-    if (dbfile==NULL){
-	clicon_err(OE_XML, 0, "dbfile NULL");
 	goto done;
     }
     if (x1 && strcmp(xml_name(x1),"config")!=0){
@@ -704,29 +866,41 @@ text_put(xmldb_handle        xh,
 		   xml_name(x1));
 	goto done;
     }
-    if ((yspec =  th->th_yangspec) == NULL){
+    if (xmltree_cache){
-	clicon_err(OE_YANG, ENOENT, "No yang spec");
+	if ((de = hash_value(th->th_dbs, db, NULL)) != NULL)
-	goto done;
+	    x0 = de->de_xml; 
     }
-    if ((fd = open(dbfile, O_RDONLY)) < 0) {
+    if (x0 == NULL){
-	clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
+	if (text_db2file(th, db, &dbfile) < 0)
-	goto done;
-    }    
-    /* Parse file into XML tree */
-    if ((clicon_xml_parse_file(fd, &x0, "</config>")) < 0)
-	goto done;
-    /* Always assert a top-level called "config". 
-       To ensure that, deal with two cases:
-       1. File is empty <top/> -> rename top-level to "config" */
-    if (xml_child_nr(x0) == 0){ 
-	if (xml_name_set(x0, "config") < 0)
 	    goto done;
-    }
+	if (dbfile==NULL){
-    /* 2. File is not empty <top><config>...</config></top> -> replace root */
+	    clicon_err(OE_XML, 0, "dbfile NULL");
-    else{ 
-	/* There should only be one element and called config */
-	if (singleconfigroot(x0, &x0) < 0)
 	    goto done;
+	}
+	if ((fd = open(dbfile, O_RDONLY)) < 0) {
+	    clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
+	    goto done;
+	}    
+	/* Parse file into XML tree */
+	if (strcmp(xml_format,"json")==0){
+	    if ((json_parse_file(fd, yspec, &x0)) < 0)
+		goto done;
+	}
+	else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
+	    goto done;
+	/* Always assert a top-level called "config". 
+	   To ensure that, deal with two cases:
+	   1. File is empty <top/> -> rename top-level to "config" */
+	if (xml_child_nr(x0) == 0){ 
+	    if (xml_name_set(x0, "config") < 0)
+		goto done;     
+	}
+	/* 2. File is not empty <top><config>...</config></top> -> replace root */
+	else{ 
+	    /* There should only be one element and called config */
+	    if (singleconfigroot(x0, &x0) < 0)
+		goto done;
+	}
     }
     /* Here x0 looks like: <config>...</config> */
     if (strcmp(xml_name(x0),"config")!=0){
@@ -736,21 +910,16 @@ text_put(xmldb_handle        xh,
     }
 
     /* Add yang specification backpointer to all XML nodes */
-    if (xml_apply(x0, CX_ELMNT, xml_spec_populate, yspec) < 0)
+    /* XXX: where is this created? Add yspec */
-       goto done;
-
-    /* Add yang specification backpointer to all XML nodes */
     if (xml_apply(x1, CX_ELMNT, xml_spec_populate, yspec) < 0)
        goto done;
-
+#if 0 /* debug */
-#if (XML_CHILD_HASH==1)
+    if (xml_child_sort && xml_apply0(x1, -1, xml_sort_verify, NULL) < 0)
-	/* Add hash */
+	clicon_log(LOG_NOTICE, "%s: verify failed #1", __FUNCTION__);
-    if (xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0)
-	goto done;
 #endif
-
     /* 
-     * Modify base tree x with modification x1
+     * Modify base tree x with modification x1. This is where the
+     * new tree is made.
      */
     if (text_modify_top(x0, x1, yspec, op) < 0)
 	goto done;
@@ -767,40 +936,60 @@ text_put(xmldb_handle        xh,
     /* Remove (prune) nodes that are marked (non-presence containers w/o children) */
     if (xml_tree_prune_flagged(x0, XML_FLAG_MARK, 1) < 0)
 	goto done;
-    // output:
+#if 0 /* debug */
-    /* Print out top-level xml tree after modification to file */
+    if (xml_child_sort && xml_apply0(x0, -1, xml_sort_verify, NULL) < 0)
-    if ((cb = cbuf_new()) == NULL){
+	clicon_log(LOG_NOTICE, "%s: verify failed #3", __FUNCTION__);
-	clicon_err(OE_XML, errno, "cbuf_new");
+#endif
+    /* Write back to datastore cache if first time */
+    if (xmltree_cache){
+	struct db_element de0 = {0,};
+	if (de != NULL)
+	    de0 = *de;
+	if (de0.de_xml == NULL){
+	    de0.de_xml = x0;
+	    hash_add(th->th_dbs, db, &de0, sizeof(de0));
+	}
+    }
+    if (dbfile == NULL){
+	if (text_db2file(th, db, &dbfile) < 0)
+	    goto done;
+	if (dbfile==NULL){
+	    clicon_err(OE_XML, 0, "dbfile NULL");
+	    goto done;
+	}
+    }
+    if (fd != -1){
+	close(fd);
+	fd = -1;
+    }
+    if ((f = fopen(dbfile, "w")) == NULL){
+	clicon_err(OE_CFG, errno, "Creating file %s", dbfile);
 	goto done;
     } 
-    if (clicon_xml2cbuf(cb, x0, 0, 1) < 0)
+    if (strcmp(xml_format,"json")==0){
-	goto done;
+	if (xml2json(f, x0, xml_pretty) < 0)
-    /* Reopen file in write mode */
+	    goto done;
-    close(fd);
-    if ((fd = open(dbfile, O_WRONLY | O_TRUNC, S_IRWXU)) < 0) {
-	clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
-	goto done;
     }
-    if (write(fd, cbuf_get(cb), cbuf_len(cb)) < 0){
+    else if (clicon_xml2file(f, x0, 0, xml_pretty) < 0)
-	clicon_err(OE_UNIX, errno, "write(%s)", dbfile);
 	goto done;
-    }
     retval = 0;
  done:
+    if (f != NULL)
+	fclose(f);
     if (dbfile)
 	free(dbfile);
     if (fd != -1)
 	close(fd);
     if (cb)
 	cbuf_free(cb);
-    if (x0)
+    if (!xmltree_cache && x0)
 	xml_free(x0);
     return retval;
 }
 
 /*! Copy database from db1 to db2
  * @param[in]  xh  XMLDB handle
- * @param[in]  from  Source database copy
+ * @param[in]  from  Source database
  * @param[in]  to    Destination database
  * @retval -1  Error
  * @retval  0  OK
@@ -808,14 +997,44 @@ text_put(xmldb_handle        xh,
 int 
 text_copy(xmldb_handle xh, 
 	  const char  *from,
-	  const char   *to)
+	  const char  *to)
 {
     int                 retval = -1;
     struct text_handle *th = handle(xh);
     char               *fromfile = NULL;
     char               *tofile = NULL;
+    struct db_element  *de = NULL;
+    struct db_element  *de2 = NULL;
 
     /* XXX lock */
+    if (xmltree_cache){
+	/* 1. Free xml tree in "to"
+	 */
+	if ((de = hash_value(th->th_dbs, to, NULL)) != NULL){
+	    if (de->de_xml != NULL){
+		xml_free(de->de_xml);
+		de->de_xml = NULL;
+	    }
+	}
+	/* 2. Copy xml tree from "from" to "to" 
+	 * 2a) create "to" if it does not exist
+	 */
+	if ((de2 = hash_value(th->th_dbs, from, NULL)) != NULL){
+	    if (de2->de_xml != NULL){
+		struct db_element de0 = {0,};
+		cxobj *x, *xcopy;
+		x = de2->de_xml;
+		if (de != NULL)
+		    de0 = *de;
+		if ((xcopy = xml_new(xml_name(x), NULL, xml_spec(x))) == NULL)
+		    goto done;
+		if (xml_copy(x, xcopy) < 0) 
+		    goto done;
+		de0.de_xml = xcopy;
+		hash_add(th->th_dbs, to, &de0, sizeof(de0));
+	    }
+	}
+    }
     if (text_db2file(th, from, &fromfile) < 0)
 	goto done;
     if (text_db2file(th, to, &tofile) < 0)
@@ -844,8 +1063,13 @@ text_lock(xmldb_handle xh,
 	  int          pid)
 {
     struct text_handle *th = handle(xh);
+    struct db_element  *de = NULL;
+    struct db_element   de0 = {0,};
 
-    hash_add(th->th_dbs, db, &pid, sizeof(pid));
+    if ((de = hash_value(th->th_dbs, db, NULL)) != NULL)
+	de0 = *de;
+    de0.de_pid = pid;
+    hash_add(th->th_dbs, db, &de0, sizeof(de0));
     clicon_debug(1, "%s: locked by %u",  db, pid);
     return 0;
 }
@@ -863,10 +1087,12 @@ text_unlock(xmldb_handle xh,
 	    const char  *db)
 {
     struct text_handle *th = handle(xh);
-    int zero = 0;
+    struct db_element  *de = NULL;
 
-    hash_add(th->th_dbs, db, &zero, sizeof(zero));
+    if ((de = hash_value(th->th_dbs, db, NULL)) != NULL){
-    //	    hash_del(th->th_dbs, db);
+	de->de_pid = 0;
+	hash_add(th->th_dbs, db, de, sizeof(*de));
+    }
     return 0;
 }
 
@@ -884,15 +1110,16 @@ text_unlock_all(xmldb_handle xh,
     char              **keys = NULL;
     size_t              klen;
     int                 i;
-    int                *val;
+    struct db_element  *de;
-    size_t              vlen;
 
     if ((keys = hash_keys(th->th_dbs, &klen)) == NULL)
 	return 0;
     for(i = 0; i < klen; i++) 
-	if ((val = hash_value(th->th_dbs, keys[i], &vlen)) != NULL &&
+	if ((de = hash_value(th->th_dbs, keys[i], NULL)) != NULL &&
-	    *val == pid)
+	    de->de_pid == pid){
-	    hash_del(th->th_dbs, keys[i]);
+	    de->de_pid = 0;
+	    hash_add(th->th_dbs, keys[i], de, sizeof(*de));
+	}
     if (keys)
 	free(keys);
     return 0;
@@ -910,13 +1137,11 @@ text_islocked(xmldb_handle xh,
 	      const char  *db)
 {
     struct text_handle *th = handle(xh);
-    size_t              vlen;
+    struct db_element  *de;
-    int                *val;
 
-    if ((val = hash_value(th->th_dbs, db, &vlen)) == NULL)
+    if ((de = hash_value(th->th_dbs, db, NULL)) == NULL)
 	return 0;
-    return *val;
+    return de->de_pid;
-    return 0;
 }
 
 /*! Check if db exists 
@@ -961,13 +1186,25 @@ text_delete(xmldb_handle xh,
     int                 retval = -1;
     char               *filename = NULL;
     struct text_handle *th = handle(xh);
+    struct db_element  *de = NULL;
+    cxobj              *xt = NULL;
+    struct stat         sb;
     
+    if (xmltree_cache){
+	if ((de = hash_value(th->th_dbs, db, NULL)) != NULL){
+	    if ((xt = de->de_xml) != NULL){
+		xml_free(xt);
+		de->de_xml = NULL;
+	    }
+	}
+    }
     if (text_db2file(th, db, &filename) < 0)
 	goto done;
-    if (unlink(filename) < 0){
+    if (lstat(filename, &sb) == 0)
-	clicon_err(OE_DB, errno, "unlink %s", filename);
+	if (unlink(filename) < 0){
-	goto done;
+	    clicon_err(OE_DB, errno, "unlink %s", filename);
-    }
+	    goto done;
+	}
     retval = 0;
  done:
     if (filename)
@@ -990,7 +1227,18 @@ text_create(xmldb_handle xh,
     struct text_handle *th = handle(xh);
     char               *filename = NULL;
     int                 fd = -1;
+    struct db_element  *de = NULL;
+    cxobj              *xt = NULL;
 
+    if (xmltree_cache){ /* XXX This should nt really happen? */
+	if ((de = hash_value(th->th_dbs, db, NULL)) != NULL){
+	    if ((xt = de->de_xml) != NULL){
+		assert(xt==NULL); /* XXX */
+		xml_free(xt);
+		de->de_xml = NULL;
+	    }
+	}
+    }
     if (text_db2file(th, db, &filename) < 0)
 	goto done;
     if ((fd = open(filename, O_CREAT|O_WRONLY, S_IRWXU)) == -1) {
@@ -1070,7 +1318,8 @@ usage(char *argv0)
 }
 
 int
-main(int argc, char **argv)
+main(int    argc,
+     char **argv)
 {
     cxobj      *xt;
     cxobj      *xn;
@@ -1105,13 +1354,20 @@ main(int argc, char **argv)
 	xpath = argc>5?argv[5]:NULL;
 	if (xmldb_get(h, db, xpath, &xt, NULL, 1, NULL) < 0)
 	    goto done;
-	clicon_xml2file(stdout, xt, 0, 1);	
+	if (strcmp(xml_format,"json")==0){
+	    if (xml2json(stdout, xt, xml_pretty) < 0)
+		goto done;
+	}
+	else{
+	    if (clicon_xml2file(stdout, xt, 0, xml_pretty) < 0)
+		goto done;
+	}
     }
     else
     if (strcmp(cmd, "put")==0){
 	if (argc != 6)
 	    usage(argv[0]);
-	if (clicon_xml_parse_file(0, &xt, "</clicon>") < 0)
+	if (xml_parse_file(0, "</clicon>", NULL, &xt) < 0)
 	    goto done;
 	if (xml_rootchild(xt, 0, &xn) < 0)
 	    goto done;

datastore/text/clixon_xmldb_text.h

@@ -49,6 +49,5 @@ int text_unlock_all(xmldb_handle h, int pid);
 int text_islocked(xmldb_handle h, const char *db);
 int text_exists(xmldb_handle h, const char *db);
 int text_delete(xmldb_handle h, const char *db);
-int text_init(xmldb_handle h, const char *db);
 
 #endif /* _CLIXON_XMLDB_TEXT_H */

develop.md

@@ -3,6 +3,7 @@
 1. How to document the code
 2. How to work in git (branching)
 3. How the meta-configure stuff works
+4. How to debug
 
 ## How to document the code
 
@@ -47,3 +48,41 @@ configure.ac --.
                     +--> config.status* -+               +--> make*
      Makefile.in ---'                    `-> Makefile ---'
 ```
+
+## How to debug
+
+### Make your own simplified yang and configuration file.
+```
+
+cat <<EOF > /tmp/my.yang
+module mymodule{
+   container x {
+    list y {
+      key "a";
+      leaf a {
+        type string;
+      }
+    }
+  }
+}
+EOF
+cat <<EOF > /tmp/myconf.xml
+<config>
+  <CLICON_CONFIGFILE>/tmp/myconf.xml</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
+ sudo clixon_backend -F -s init -f /tmp/myconf.xml -y /tmp/my.yang
+ ```
+
+### Run valgrind and callgrind
+ ```
+  valgrind --leak-check=full --show-leak-kinds=all clixon_netconf -qf /tmp/myconf.xml -y /tmp/my.yang
+  valgrind --tool=callgrind clixon_netconf -qf /tmp/myconf.xml -y /tmp/my.yang
+  sudo kcachegrind
+ ```

example/routing_backend.c

@@ -165,11 +165,11 @@ plugin_statedata(clicon_handle h,
     cxobj **xvec = NULL;
 
     /* Example of (static) statedata, real code would poll state */
-    if (xml_parse("<interfaces-state><interface>"
+    if (xml_parse_string("<interfaces-state><interface>"
-		   "<name>eth0</name>"
+			 "<name>eth0</name>"
-		   "<type>eth</type>"
+			 "<type>eth</type>"
-		   "<if-index>42</if-index>"
+			 "<if-index>42</if-index>"
-		   "</interface></interfaces-state>", xstate) < 0)
+			 "</interface></interfaces-state>", NULL, &xstate) < 0)
 	goto done;
     retval = 0;
  done:
@@ -225,9 +225,9 @@ plugin_reset(clicon_handle h,
     int    retval = -1;
     cxobj *xt = NULL;
 
-    if (clicon_xml_parse_str("<config><interfaces><interface>"
+    if (xml_parse_string("<config><interfaces><interface>"
 			     "<name>lo</name><type>local</type>"
-			     "</interface></interfaces></config>", &xt) < 0)
+			     "</interface></interfaces></config>", NULL, &xt) < 0)
 	goto done;
     /* Replace parent w fiorst child */
     if (xml_rootchild(xt, 0, &xt) < 0)

example/routing_cli.c

@@ -107,7 +107,7 @@ fib_route_rpc(clicon_handle h,
     /* User supplied variable in CLI command */
     instance = cvec_find(cvv, "instance"); /* get a cligen variable from vector */
     /* Create XML for fib-route netconf RPC */
-    if (clicon_xml_parse(&xtop, "<rpc><fib-route><routing-instance-name>%s</routing-instance-name></fib-route></rpc>", instance) < 0)
+    if (xml_parse_va(&xtop, NULL, "<rpc><fib-route><routing-instance-name>%s</routing-instance-name></fib-route></rpc>", instance) < 0)
 	goto done;
     /* Skip top-level */
     xrpc = xml_child_i(xtop, 0);

extras/rpm/Makefile.in

@@ -0,0 +1,35 @@
+TARBALL=$(shell realpath ../../build-root/clixon-latest.tar.xz)
+BASENAME=$(shell basename $(TARBALL) | sed -e s/.tar.\*//)
+VERSION=$(shell echo $(BASENAME) | cut -f2 -d-)
+RELEASE=$(shell echo $(BASENAME) | cut -f3- -d- | sed -e s/-/_/g)
+BR=$(shell realpath $(CURDIR)/../../build-root)
+RPMBUILD=$(BR)/rpmbuild
+
+all: RPM
+
+spec:
+	@echo $(TARBALL)
+	mkdir -p $(RPMBUILD)/{RPMS,SRPMS,BUILD,SOURCES,SPECS}
+	cp $(TARBALL) $(RPMBUILD)/SOURCES/clixon-$(VERSION)-$(RELEASE).tar.xz
+	cp clixon.spec $(RPMBUILD)/SPECS
+
+srpm: spec
+	rpmbuild -bs \
+	  --define "cligen_prefix @CLIGEN_PREFIX@" \
+	  --define "_topdir $(RPMBUILD)" \
+	  --define "_version $(VERSION)" \
+	  --define "_release $(RELEASE)" \
+	  $(RPMBUILD)/SPECS/clixon.spec
+	mv $$(find $(RPMBUILD)/SRPMS -name \*.src.rpm -type f) $(BR)
+
+# Define DEVELOPER environmrnt variable to prevent .spec to add cligent to the
+# list of build requirements
+RPM: spec
+	rpmbuild -bb \
+	  --define "cligen_prefix @CLIGEN_PREFIX@" \
+	  $${DEVELOPER:+--define "developer yes"} \
+	  --define "_topdir $(RPMBUILD)" \
+	  --define "_version $(VERSION)" \
+	  --define "_release $(RELEASE)" \
+	  $(RPMBUILD)/SPECS/clixon.spec
+	mv $$(find $(RPMBUILD)/RPMS -name \*.rpm -type f) $(BR)

extras/rpm/clixon.spec

@@ -0,0 +1,72 @@
+%{!?_topdir: %define _topdir %(pwd)}
+%{!?cligen_prefix: %define cligen_prefix %{_prefix}}
+
+Name: clixon
+Version: %{_version}
+Release: %{_release}
+Summary: The XML-based command line processing tool CLIXON
+Group: System Environment/Libraries
+License: ASL 2.0 or GPLv2
+URL: http://www.clicon.org
+AutoReq: no
+BuildRequires: flex, bison
+Requires: cligen, fcgi
+
+# Sometimes developers want to build it without installing cligen but passing
+# path using --with-cligen and pointing it to cligen buildroot. Use %{developer}
+# macro for these cases
+%if 0%{!?developer:1}
+BuildRequires: cligen
+%endif
+
+Source: %{name}-%{version}-%{release}.tar.xz
+
+%description
+The XML-based command line processing tool CLIXON.
+
+%package devel
+Summary: CLIXON header files
+Group: Development/Libraries
+Requires: clixon
+
+%description devel
+This package contains header files for CLIXON.
+
+%prep
+%setup
+
+%build
+%configure --with-cligen=%{cligen_prefix} --without-keyvalue
+make
+
+%install
+make DESTDIR=${RPM_BUILD_ROOT} install install-include
+
+%files
+%{_libdir}/*
+%{_bindir}/*
+%{_sbindir}/*
+#%{_sysconfdir}/*
+%{_datadir}/%{name}/*
+/www-data/clixon_restconf
+
+%files devel
+%{_includedir}/%{name}/*
+
+%clean
+
+%post
+/sbin/ldconfig
+
+caps="cap_setuid,cap_fowner,cap_chown,cap_dac_override"
+caps="${caps},cap_kill,cap_net_admin,cap_net_bind_service"
+caps="${caps},cap_net_broadcast,cap_net_raw"
+
+if [ -x /usr/sbin/setcap ]; then
+	/usr/sbin/setcap ${caps}=ep %{_bindir}/clixon_cli
+	/usr/sbin/setcap ${caps}=ep %{_bindir}/clixon_netconf
+	/usr/sbin/setcap ${caps}=ep %{_sbindir}/clixon_backend
+fi
+
+%postun
+/sbin/ldconfig

extras/scripts/version

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Obtained from VPP - https://wiki.fd.io/view/VPP
+#
+
+path=$( cd "$(dirname "${BASH_SOURCE}")" ; pwd -P )
+
+cd "$path"
+
+if [ -f .version ]; then
+    vstring=$(cat .version)
+else
+    vstring=$(git describe)
+    if [ $? != 0 ]; then
+      exit 1
+    fi
+fi
+
+TAG=$(echo ${vstring} | cut -d- -f1 | sed -e 's/^[vR]//')
+ADD=$(echo ${vstring} | cut -s -d- -f2)
+
+git rev-parse 2> /dev/null
+if [ $? == 0 ]; then
+    CMT=$(git describe --dirty | cut -s -d- -f3,4)
+else
+    CMT=$(echo ${vstring} | cut -s -d- -f3,4)
+fi
+CMTR=$(echo $CMT | sed 's/-/_/')
+
+if [ -n "${BUILD_NUMBER}" ]; then
+       BLD="~b${BUILD_NUMBER}"
+fi
+
+if [ "$1" = "rpm-version" ]; then
+  echo ${TAG}
+  exit
+fi
+
+if [ "$1" = "rpm-release" ]; then
+  [ -z "${ADD}" ] && echo release && exit
+  echo ${ADD}${CMTR:+~${CMTR}}${BLD}
+  exit
+fi
+
+  if [ -n "${ADD}" ]; then
+    if [ "$1" = "rpm-string" ]; then
+      echo ${TAG}-${ADD}${CMTR:+~${CMTR}}${BLD}
+    else
+      echo ${TAG}-${ADD}${CMT:+~${CMT}}${BLD}
+    fi
+  else
+    echo ${TAG}-release
+fi

include/clixon_config.h.in

@@ -138,6 +138,9 @@
 /* Define to 1 if you have the ANSI C header files. */
 #undef STDC_HEADERS
 
+/* Backward compatible of XML API */
+#undef XML_COMPAT
+
 /* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
    `char[]'. */
 #undef YYTEXT_POINTER

include/clixon_custom.h

@@ -43,8 +43,6 @@
 int strverscmp (__const char *__s1, __const char *__s2);
 #endif
 
-/* Hash for XML trees list entries
+
- * Experimental
+
- */
-#define XML_CHILD_HASH 1
 

lib/clixon/clixon.h.in

@@ -56,6 +56,8 @@
 #undef CLIXON_VERSION_MINOR
 #undef CLIXON_VERSION_PATCH
 
+#undef XML_COMPAT
+
 /*
  * Use this constant to disable some prototypes that should not be visible outside the lib.
  * This is an alternative to use separate internal include files.
@@ -74,6 +76,7 @@
 #include <clixon/clixon_string.h>
 #include <clixon/clixon_file.h>
 #include <clixon/clixon_xml.h>
+#include <clixon/clixon_xml_sort.h>
 #include <clixon/clixon_proto.h>
 #include <clixon/clixon_proto_client.h>
 #include <clixon/clixon_plugin.h>

lib/clixon/clixon_json.h

@@ -39,10 +39,11 @@
 /*
  * Prototypes
  */
-int json_parse_str(char *str, cxobj **xt);
 int xml2json_cbuf(cbuf *cb, cxobj *x, int pretty);
 int xml2json_cbuf_vec(cbuf *cb, cxobj **vec, size_t veclen, int pretty);
 int xml2json(FILE *f, cxobj *x, int pretty);
 int xml2json_vec(FILE *f, cxobj **vec, size_t veclen, int pretty);
+int json_parse_str(char *str, cxobj **xt);
+int json_parse_file(int fd, yang_spec *yspec, cxobj **xt);
 
 #endif /* _CLIXON_JSON_H */

lib/clixon/clixon_options.h

@@ -74,53 +74,92 @@ enum startup_mode_t{
 /*
  * Prototypes
  */
+
+/* Print registry on file. For debugging. */
+void clicon_option_dump(clicon_handle h, int dblevel);
 /* Initialize options: set defaults, read config-file, etc */
 int clicon_options_main(clicon_handle h);
-
+/*! Check if a clicon option has a value */
-void clicon_option_dump(clicon_handle h, int dblevel);
-
 int clicon_option_exists(clicon_handle h, const char *name);
 
-/* Get a single option via handle */
+/* String options, default NULL */
 char *clicon_option_str(clicon_handle h, const char *name);
-int clicon_option_int(clicon_handle h, const char *name);
-/* Set a single option via handle */
 int clicon_option_str_set(clicon_handle h, const char *name, char *val);
+
+/* Option values gixen as int, default -1 */
+int clicon_option_int(clicon_handle h, const char *name);
 int clicon_option_int_set(clicon_handle h, const char *name, int val);
+
+/* Option values gixen as bool, default false */
+int clicon_option_bool(clicon_handle h, const char   *name);
+int clicon_option_bool_set(clicon_handle h, const char *name, int val);
+
 /* Delete a single option via handle */
 int clicon_option_del(clicon_handle h, const char *name);
 
-char *clicon_configfile(clicon_handle h);
+/*-- Standard option access functions for YANG options --*/
-char *clicon_yang_dir(clicon_handle h);
+static inline char *clicon_configfile(clicon_handle h){
-char *clicon_yang_module_main(clicon_handle h);
+    return clicon_option_str(h, "CLICON_CONFIGFILE");
-char *clicon_yang_module_revision(clicon_handle h);
+}
-char *clicon_backend_dir(clicon_handle h);
+static inline char *clicon_yang_dir(clicon_handle h){
-char *clicon_cli_dir(clicon_handle h);
+    return clicon_option_str(h, "CLICON_YANG_DIR");
-char *clicon_clispec_dir(clicon_handle h);
+}
-char *clicon_netconf_dir(clicon_handle h);
+static inline char *clicon_yang_module_main(clicon_handle h){
-char *clicon_restconf_dir(clicon_handle h);
+    return clicon_option_str(h, "CLICON_YANG_MODULE_MAIN");
-char *clicon_xmldb_plugin(clicon_handle h);
+}
-int   clicon_startup_mode(clicon_handle h);
+static inline char *clicon_yang_module_revision(clicon_handle h){
-int   clicon_sock_family(clicon_handle h);
+    return clicon_option_str(h, "CLICON_YANG_MODULE_REVISION");
-char *clicon_sock(clicon_handle h);
+}
-int   clicon_sock_port(clicon_handle h);
+static inline char *clicon_backend_dir(clicon_handle h){
-char *clicon_backend_pidfile(clicon_handle h);
+    return clicon_option_str(h, "CLICON_BACKEND_DIR");
-char *clicon_sock_group(clicon_handle h);
+}
+static inline char *clicon_netconf_dir(clicon_handle h){
+    return clicon_option_str(h, "CLICON_NETCONF_DIR");
+}
+static inline char *clicon_restconf_dir(clicon_handle h){
+    return clicon_option_str(h, "CLICON_RESTCONF_DIR");
+}
+static inline char *clicon_cli_dir(clicon_handle h){
+    return clicon_option_str(h, "CLICON_CLI_DIR");
+}
+static inline char *clicon_clispec_dir(clicon_handle h){
+    return clicon_option_str(h, "CLICON_CLISPEC_DIR");
+}
+static inline char *clicon_cli_mode(clicon_handle h){
+    return clicon_option_str(h, "CLICON_CLI_MODE");
+}
+static inline char *clicon_sock(clicon_handle h){
+    return clicon_option_str(h, "CLICON_SOCK");
+}
+static inline char *clicon_sock_group(clicon_handle h){
+    return clicon_option_str(h, "CLICON_SOCK_GROUP");
+}
+static inline char *clicon_backend_pidfile(clicon_handle h){
+    return clicon_option_str(h, "CLICON_BACKEND_PIDFILE");
+}
+static inline char *clicon_master_plugin(clicon_handle h){
+    return clicon_option_str(h, "CLICON_MASTER_PLUGIN");
+}
+static inline char *clicon_xmldb_dir(clicon_handle h){
+    return clicon_option_str(h, "CLICON_XMLDB_DIR");
+}
+static inline char *clicon_xmldb_plugin(clicon_handle h){
+    return clicon_option_str(h, "CLICON_XMLDB_PLUGIN");
+}
 
-char *clicon_master_plugin(clicon_handle h);
+/*-- Specific option access functions for YANG options w type conversion--*/
-char *clicon_cli_mode(clicon_handle h);
 int   clicon_cli_genmodel(clicon_handle h);
-int   clicon_cli_varonly(clicon_handle h);
-int   clicon_cli_varonly_set(clicon_handle h, int val);
 int   clicon_cli_genmodel_completion(clicon_handle h);
-
-char *clicon_xmldb_dir(clicon_handle h);
-
-char *clicon_quiet_mode(clicon_handle h);
 enum genmodel_type clicon_cli_genmodel_type(clicon_handle h);
+int   clicon_cli_varonly(clicon_handle h);
+int   clicon_sock_family(clicon_handle h);
+int   clicon_sock_port(clicon_handle h);
+int   clicon_autocommit(clicon_handle h);
+int   clicon_startup_mode(clicon_handle h);
 
-int clicon_autocommit(clicon_handle h);
+/*-- Specific option access functions for non-yang options --*/
-int clicon_autocommit_set(clicon_handle h, int val);
+int clicon_quiet_mode(clicon_handle h);
+int clicon_quiet_mode_set(clicon_handle h, int val);
 
 yang_spec * clicon_dbspec_yang(clicon_handle h);
 int clicon_dbspec_yang_set(clicon_handle h, struct yang_spec *ys);
@@ -128,16 +167,16 @@ int clicon_dbspec_yang_set(clicon_handle h, struct yang_spec *ys);
 char *clicon_dbspec_name(clicon_handle h);
 int clicon_dbspec_name_set(clicon_handle h, char *name);
 
-int clicon_xmldb_plugin_set(clicon_handle h, plghndl_t handle);
+yang_spec *clicon_netconf_yang(clicon_handle h);
+int clicon_netconf_yang_set(clicon_handle h, struct yang_spec *ys);
 
 plghndl_t clicon_xmldb_plugin_get(clicon_handle h);
-
+int clicon_xmldb_plugin_set(clicon_handle h, plghndl_t handle);
-int clicon_xmldb_api_set(clicon_handle h, void *xa_api);
 
 void *clicon_xmldb_api_get(clicon_handle h);
-
+int clicon_xmldb_api_set(clicon_handle h, void *xa_api);
-int clicon_xmldb_handle_set(clicon_handle h, void *xh);
 
 void *clicon_xmldb_handle_get(clicon_handle h);
+int clicon_xmldb_handle_set(clicon_handle h, void *xh);
 
 #endif  /* _CLIXON_OPTIONS_H_ */

lib/clixon/clixon_xml.h

@@ -32,10 +32,16 @@
   ***** END LICENSE BLOCK *****
 
  * XML support functions.
+ * @see https://www.w3.org/TR/2008/REC-xml-20081126
+ *      https://www.w3.org/TR/2009/REC-xml-names-20091208/
  */
 #ifndef _CLIXON_XML_H
 #define _CLIXON_XML_H
 
+/*
+ * Constants
+ */
+
 /*
  * Types
  */
@@ -74,6 +80,11 @@ typedef int (xml_applyfn_t)(cxobj *yn, void *arg);
 #define XML_FLAG_CHANGE 0x08  /* Node is changed (commits) or child changed rec */
 #define XML_FLAG_NONE   0x10  /* Node is added as NONE */
 
+/* Sort and binary search of XML children
+ * Experimental
+ */
+extern int xml_child_sort;
+
 /*
  * Prototypes
  */
@@ -106,10 +117,9 @@ cxobj    *xml_child_each(cxobj *xparent, cxobj *xprev,  enum cxobj_type type);
 
 cxobj   **xml_childvec_get(cxobj *x);
 int       xml_childvec_set(cxobj *x, int len);
-cxobj    *xml_new(char *name, cxobj *xn_parent);
+cxobj    *xml_new(char *name, cxobj *xn_parent, yang_stmt *spec);
-cxobj    *xml_new_spec(char *name, cxobj *xn_parent, void *spec);
+yang_stmt *xml_spec(cxobj *x);
-void     *xml_spec(cxobj *x);
+int       xml_spec_set(cxobj *x, yang_stmt *spec);
-void     *xml_spec_set(cxobj *x, void *spec);
 cxobj    *xml_find(cxobj *xn_parent, char *name);
 
 int       xml_addsub(cxobj *xp, cxobj *xc);
@@ -130,14 +140,12 @@ int       xml_free(cxobj *xn);
 int       xml_print(FILE  *f, cxobj *xn);
 int       clicon_xml2file(FILE *f, cxobj *xn, int level, int prettyprint);
 int       clicon_xml2cbuf(cbuf *xf, cxobj *xn, int level, int prettyprint);
-int       clicon_xml_parse_file(int fd, cxobj **xml_top, char *endtag);
+int       xml_parse_file(int fd, char *endtag, yang_spec *yspec, cxobj **xt);
-/* XXX obsolete */
+int       xml_parse_string(const char *str, yang_spec *yspec, cxobj **xml_top);
-#define clicon_xml_parse_string(str, x) clicon_xml_parse_str((*str), x) 
+int       xml_parse_va(cxobj **xt, yang_spec *yspec, const char *format, ...);
-int       clicon_xml_parse_str(char *str, cxobj **xml_top);
-int       clicon_xml_parse(cxobj **cxtop, char *format, ...);
-int       xml_parse(char *str, cxobj *x_up);
 
 int       xmltree2cbuf(cbuf *cb, cxobj *x, int level);
+int       xml_copy_one(cxobj *xn0, cxobj *xn1);
 int       xml_copy(cxobj *x0, cxobj *x1);
 cxobj    *xml_dup(cxobj *x0);
 
@@ -152,12 +160,17 @@ int       xml_body_int32(cxobj *xb, int32_t *val);
 int       xml_body_uint32(cxobj *xb, uint32_t *val);
 int       xml_operation(char *opstr, enum operation_type *op);
 char     *xml_operation2str(enum operation_type op);
-#if (XML_CHILD_HASH==1)
+
-clicon_hash_t *xml_hash(cxobj *x);
+#ifdef XML_COMPAT /* See CHANGELOG */
-int xml_hash_init(cxobj *x);
+/* MANUAL CHANGE: xml_new(name, parent) --> xml_new(name, parent, NULL) */
-int xml_hash_rm(cxobj *x);
+
-int xml_hash_key(cxobj *x, yang_stmt *y, cbuf *key);
+#define xml_new_spec(name, parent) xml_new(name, parent, NULL)
-int xml_hash_op(cxobj *x, void *arg);
+#define clicon_xml_parse(xt, fmt, ...) xml_parse_va(xt, NULL, fmt, __VA_ARGS__)
+#define clicon_xml_parse_file(fd, xt, etag) xml_parse_file(fd, etag, NULL, xt)
+#define clicon_xml_parse_string(strp, xt) xml_parse_string(*strp, NULL, xt)
+#define clicon_xml_parse_str(str, xt) xml_parse_string(str, NULL, xt)
+#define xml_parse(str, xt) xml_parse_string(str, NULL, &xt)
 #endif
 
+
 #endif /* _CLIXON_XML_H */

lib/clixon/clixon_xml_map.h

@@ -65,7 +65,6 @@ int api_path2xpath_cvv(yang_spec *yspec, cvec *cvv, int offset, cbuf *xpath);
 int api_path2xpath(yang_spec *yspec, char *api_path, cbuf *xpath);
 int api_path2xml(char *api_path, yang_spec *yspec, cxobj *xtop, 
 		 int schemanode, cxobj **xpathp, yang_node **ypathp);
-int match_base_child(cxobj *x0, cxobj *x1c, cxobj **x0cp, yang_stmt *yc);
 int xml_merge(cxobj *x0, cxobj *x1, yang_spec *yspec);
 int yang_enum_int_value(cxobj *node, int32_t *val);
 

lib/clixon/clixon_xml_sort.h

@@ -0,0 +1,58 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2017 Olof Hagsand and Benny Holmgren
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+ * XML sort and earch functions when used with YANG
+ */
+#ifndef _CLIXON_XML_SORT_H
+#define _CLIXON_XML_SORT_H
+
+/* Sort and binary search of XML children
+ * Experimental
+ */
+extern int xml_child_sort;
+
+/*
+ * Prototypes
+ */
+int    xml_child_spec(char  *name, cxobj  *xp, yang_spec *yspec, yang_stmt **yp);
+int    xml_cmp(const void* arg1, const void* arg2);
+int    xml_sort(cxobj *x0, void *arg);
+cxobj *xml_search(cxobj *x, char *name, int yangi, enum rfc_6020 keyword, int keynr, char **keyvec, char **keyval);
+int    xml_insert_pos(cxobj *x0, char *name, int yangi, enum rfc_6020 keyword,
+		      int keynr, char **keyvec, char **keyval, int low,
+		      int upper);
+cxobj *xml_match(cxobj *x0, char *name, enum rfc_6020 keyword, int keynr, char **keyvec, char **keyval);
+int    xml_sort_verify(cxobj *x, void *arg);
+int    match_base_child(cxobj *x0, cxobj *x1c, cxobj **x0cp, yang_stmt *yc);
+
+#endif /* _CLIXON_XML_SORT_H */

lib/clixon/clixon_yang.h

@@ -155,10 +155,17 @@ struct yang_stmt{
 
     char              *ys_argument;  /* String / argument depending on keyword */   
     int                ys_flags;     /* Flags according to YANG_FLAG_* above */
-    cg_var            *ys_cv;        /* cligen variable. The following stmts have cvs::
+    cg_var            *ys_cv;        /* cligen variable. See ys_populate()
-				        leaf, leaf-list, mandatory, fraction-digits */
+					Following stmts have cv:s:
+				        leaf: for default value
+					leaf-list, 
+					config: boolean true or false
+					mandatory: boolean true or false
+					fraction-digits for fraction-digits */
     cvec              *ys_cvec;      /* List of stmt-specific variables 
-					Y_RANGE: range_min, range_max */
+					Y_RANGE: range_min, range_max 
+					Y_LIST: vector of keys
+				     */
     yang_type_cache   *ys_typecache; /* If ys_keyword==Y_TYPE, cache all typedef data except unions */
 };
 
@@ -208,7 +215,7 @@ yang_stmt *yang_find(yang_node *yn, int keyword, char *argument);
 yang_stmt *yang_find_datanode(yang_node *yn, char *argument);
 yang_stmt *yang_find_schemanode(yang_node *yn, char *argument);
 yang_stmt *yang_find_topnode(yang_spec *ysp, char *name, int schemanode);
-
+int        yang_order(yang_stmt *y);
 int        yang_print(FILE *f, yang_node *yn);
 int        yang_print_cbuf(cbuf *cb, yang_node *yn, int marginal);
 int        yang_parse(clicon_handle h, const char *yang_dir, 
@@ -223,7 +230,8 @@ cg_var    *ys_parse(yang_stmt *ys, enum cv_type cvtype);
 int        ys_parse_sub(yang_stmt *ys);
 int        yang_mandatory(yang_stmt *ys);
 int        yang_config(yang_stmt *ys);
-int        yang_spec_main(clicon_handle h, FILE *f, int printspec);
+yang_spec *yang_spec_netconf(clicon_handle h);
+yang_spec *yang_spec_main(clicon_handle h);
 cvec      *yang_arg2cvec(yang_stmt *ys, char *delimi);
 int        yang_key_match(yang_node *yn, char *name);
 

lib/src/Makefile.in

@@ -64,7 +64,7 @@ INCLUDES = -I. @INCLUDES@ -I$(top_srcdir)/lib/clixon -I$(top_srcdir)/include -I$
 
 SRC     = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \
 	  clixon_string.c clixon_handle.c  \
-	  clixon_xml.c clixon_xml_map.c clixon_file.c \
+	  clixon_xml.c clixon_xml_sort.c clixon_xml_map.c clixon_file.c \
 	  clixon_json.c clixon_yang.c clixon_yang_type.c \
 	  clixon_hash.c clixon_options.c clixon_plugin.c \
 	  clixon_proto.c clixon_proto_client.c \

lib/src/clixon_hash.c

@@ -235,8 +235,8 @@ hash_add(clicon_hash_t *hash,
 	h = new;
     }
     
-    /* Make copy of lvalue */
+    /* Make copy of value. aligned */
-    newval = malloc(align4(vlen+3)); /* XXX: qdbm needs aligned mallocs? */
+    newval = malloc(align4(vlen+3)); 
     if (newval == NULL){
 	clicon_err(OE_UNIX, errno, "malloc: %s", strerror(errno));
 	goto catch;

lib/src/clixon_json.c

@@ -54,6 +54,10 @@
 /* clixon */
 #include "clixon_err.h"
 #include "clixon_log.h"
+#include "clixon_queue.h"
+#include "clixon_hash.h"
+#include "clixon_handle.h"
+#include "clixon_yang.h"
 #include "clixon_xml.h"
 #include "clixon_json.h"
 #include "clixon_json_parse.h"
@@ -65,6 +69,12 @@
 */
 #define VEC_ARRAY 1
 
+/* Size of json read buffer when reading from file*/
+#define BUFLEN 1024
+
+/* Name of xml top object created by xml parse functions */
+#define JSON_TOP_SYMBOL "top"
+
 enum array_element_type{
     NO_ARRAY=0,
     FIRST_ARRAY,
@@ -181,32 +191,40 @@ json_escape(char *str)
 
     j = 0;
     for (i=0;i<strlen(str);i++)
-	if (str[i]=='\n')
+	switch (str[i]){
+	case '\n':
+	case '\"':
+	case '\\':
 	    j++;
+	    break;
+	}
     if ((snew = malloc(strlen(str)+1+j))==NULL){
 	clicon_err(OE_XML, errno, "malloc");
 	return NULL;
     }
     j = 0;
     for (i=0;i<strlen(str);i++)
-	if (str[i]=='\n'){
+	switch (str[i]){
+	case '\n':
+	case '\"':
+	case '\\':
 	    snew[j++]='\\';
-	    snew[j++]='n';
+	default: /* fall thru */
-	}
-	else
 	    snew[j++]=str[i];
+	}
     snew[j++]='\0';
     return snew;
 }
 
 /*! Do the actual work of translating XML to JSON 
- * @param[out]   cb       Cligen text buffer containing json on exit
+ * @param[out]   cb        Cligen text buffer containing json on exit
- * @param[in]    x        XML tree structure containing XML to translate
+ * @param[in]    x         XML tree structure containing XML to translate
  * @param[in]    arraytype Does x occur in a array (of its parent) and how?
  * @param[in]    level     Indentation level
  * @param[in]    pretty    Pretty-print output (2 means debug)
  * @param[in]    flat      Dont print NO_ARRAY object name (for _vec call)
  *
+ * @note Does not work with XML attributes
  * The following matrix explains how the mapping is done.
  * You need to understand what arraytype means (no/first/middle/last)
  * and what childtype is (null,body,any)
@@ -242,10 +260,11 @@ xml2json1_cbuf(cbuf                  *cb,
 	       int                    pretty,
 	       int                    flat)
 {
-    int             retval = -1;
+    int              retval = -1;
-    int             i;
+    int              i;
-    cxobj          *xc;
+    cxobj           *xc;
     enum childtype   childt;
+    enum array_element_type xc_arraytype;
 
     childt = childtype(x);
     if (pretty==2)
@@ -322,7 +341,6 @@ xml2json1_cbuf(cbuf                  *cb,
 	break;
     }
     for (i=0; i<xml_child_nr(x); i++){
-	enum array_element_type xc_arraytype;
 	xc = xml_child_i(x, i);
 	xc_arraytype = array_eval(i?xml_child_i(x,i-1):NULL, 
 				xc, 
@@ -465,7 +483,7 @@ xml2json_cbuf_vec(cbuf   *cb,
     cxobj *xp = NULL;
     cxobj *xc;
 
-    if ((xp = xml_new("", NULL)) == NULL)
+    if ((xp = xml_new("", NULL, NULL)) == NULL)
 	goto done;
     for (i=0; i<veclen; i++){
 	xc = xml_dup(vec[i]);
@@ -619,11 +637,87 @@ int
 json_parse_str(char   *str, 
 	       cxobj **xt)
 {
-    if ((*xt = xml_new("top", NULL)) == NULL)
+    if ((*xt = xml_new("top", NULL, NULL)) == NULL)
 	return -1;
     return json_parse(str, "", *xt);
 }
 
+/*! Read a JSON definition from file and parse it into a parse-tree. 
+ *
+ * @param[in]  fd  A file descriptor containing the JSON file (as ASCII characters)
+ * @param[in]  yspec   Yang specification, or NULL
+ * @param[in,out] xt   Pointer to (XML) parse tree. If empty, create.
+ * @retval        0  OK
+ * @retval       -1  Error with clicon_err called
+ *
+ * @code
+ *  cxobj *xt = NULL;
+ *  if (json_parse_file(0, NULL, &xt) < 0)
+ *    err;
+ *  xml_free(xt);
+ * @endcode
+ * @note  you need to free the xml parse tree after use, using xml_free()
+ * @note, If xt empty, a top-level symbol will be added so that <tree../> will be:  <top><tree.../></tree></top>
+ * @note May block on file I/O
+ */
+int 
+json_parse_file(int        fd,
+		yang_spec *yspec,
+		cxobj    **xt)
+{
+    int   retval = -1;
+    int   ret;
+    char *jsonbuf = NULL;
+    int   jsonbuflen = BUFLEN; /* start size */
+    int   oldjsonbuflen;
+    char *ptr;
+    char  ch;
+    int   len = 0;
+    
+    if ((jsonbuf = malloc(jsonbuflen)) == NULL){
+	clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__);
+	goto done;
+    }
+    memset(jsonbuf, 0, jsonbuflen);
+    ptr = jsonbuf;
+    while (1){
+	if ((ret = read(fd, &ch, 1)) < 0){
+	    clicon_err(OE_XML, errno, "%s: read: [pid:%d]\n", 
+		    __FUNCTION__,
+		    (int)getpid());
+	    break;
+	}
+	if (ret != 0)
+	    jsonbuf[len++] = ch;
+	if (ret == 0){
+	    if (*xt == NULL)
+		if ((*xt = xml_new(JSON_TOP_SYMBOL, NULL, NULL)) == NULL)
+		    goto done;
+	    if (len && json_parse(ptr, "", *xt) < 0)
+		goto done;
+	    break;
+	}
+	if (len>=jsonbuflen-1){ /* Space: one for the null character */
+	    oldjsonbuflen = jsonbuflen;
+	    jsonbuflen *= 2;
+	    if ((jsonbuf = realloc(jsonbuf, jsonbuflen)) == NULL){
+		clicon_err(OE_XML, errno, "%s: realloc", __FUNCTION__);
+		goto done;
+	    }
+	    memset(jsonbuf+oldjsonbuflen, 0, jsonbuflen-oldjsonbuflen);
+	    ptr = jsonbuf;
+	}
+    }
+    retval = 0;
+ done:
+    if (retval < 0 && *xt){
+	free(*xt);
+	*xt = NULL;
+    }
+    if (jsonbuf)
+	free(jsonbuf);
+    return retval;    
+}
 
 /*
  * Turn this on to get a json parse and pretty print test program

lib/src/clixon_json_parse.y

@@ -171,7 +171,7 @@ json_current_new(struct clicon_json_yacc_arg *jy,
     cxobj *xn;
 
     clicon_debug(2, "%s", __FUNCTION__);
-    if ((xn = xml_new(name, jy->jy_current)) == NULL)
+    if ((xn = xml_new(name, jy->jy_current, NULL)) == NULL)
 	goto done; 
     jy->jy_current = xn;
     retval = 0;
@@ -207,7 +207,7 @@ json_current_body(struct clicon_json_yacc_arg *jy,
     cxobj *xn;
 
     clicon_debug(2, "%s", __FUNCTION__);
-    if ((xn = xml_new("body", jy->jy_current)) == NULL)
+    if ((xn = xml_new("body", jy->jy_current, NULL)) == NULL)
 	goto done; 
     xml_type_set(xn, CX_BODY);
     if (value && xml_value_append(xn, value)==NULL)

lib/src/clixon_options.c

@@ -150,7 +150,7 @@ clicon_option_readfile_xml(clicon_hash_t *copt,
     }
     clicon_debug(2, "Reading config file %s", __FUNCTION__, filename);
     fd = fileno(f);
-    if (clicon_xml_parse_file(fd, &xt, "</clicon>") < 0)
+    if (xml_parse_file(fd, "</clicon>", yspec, &xt) < 0)
 	goto done;
     if (xml_child_nr(xt)==1 && xml_child_nr_type(xt, CX_BODY)==1){
 	clicon_err(OE_CFG, 0, "Config file %s: Expected XML but is probably old sh style", filename);
@@ -160,8 +160,6 @@ clicon_option_readfile_xml(clicon_hash_t *copt,
 	clicon_err(OE_CFG, 0, "Config file %s: Lacks top-level \"config\" element", filename);
 	goto done;	
     }
-    if (xml_apply0(xc, CX_ELMNT, xml_spec_populate, yspec) < 0)
-	goto done;	
     if (xml_apply0(xc, CX_ELMNT, xml_default, yspec) < 0)
 	goto done;	
     if (xml_apply0(xc, CX_ELMNT, xml_yang_validate_add, NULL) < 0)
@@ -384,8 +382,11 @@ clicon_options_main(clicon_handle h)
 	/* Read configfile */
 	if (clicon_option_readfile_xml(copt, configfile, yspec) < 0)
 	    goto done;
-	if (yspec)
+	/* Specific option handling */
-	    yspec_free(yspec);
+	if (clicon_option_bool(h, "CLICON_XML_SORT") == 1)
+	    xml_child_sort = 1;
+	else
+	    xml_child_sort = 0;
     }
     else {
 #ifdef CONFIG_COMPAT
@@ -404,12 +405,16 @@ clicon_options_main(clicon_handle h)
     }
     retval = 0;
  done:
+    if (yspec) /* The clixon yang-spec is not used after this */
+	yspec_free(yspec);
+
     return retval;
 }
 
 /*! Check if a clicon option has a value
  * @param[in] h       clicon_handle
  * @param[in] name    option name
+ * @retval
  */
 int
 clicon_option_exists(clicon_handle h,
@@ -497,6 +502,49 @@ clicon_option_int_set(clicon_handle h,
     return clicon_option_str_set(h, name, s);
 }
 
+/*! Get options as bool but stored as string
+ *
+ * @param   h    clicon handle
+ * @param   name name of option
+ * @retval  0    false, or does not exist, or does not have a boolean value
+ * @retval  1    true
+ * @code
+ *  if (clicon_option_exists(h, "X")
+ *	return clicon_option_bool(h, "X");
+ *  else
+ *      return 0; # default false? 
+ * @endcode
+ * Note that 0 can be both error and false.
+ * This means that it should be used together with clicon_option_exists() and
+ * supply a default value as shown in the example.
+ */
+int
+clicon_option_bool(clicon_handle h,
+		  const char   *name)
+{
+    char *s;
+
+    if ((s = clicon_option_str(h, name)) == NULL)
+	return 0;
+    if (strcmp(s,"true")==0)
+	return 1;
+    return 0; /* Hopefully false, but anything else than "true" */
+}
+
+/*! Set option given as bool
+ */
+int
+clicon_option_bool_set(clicon_handle h,
+		      const char   *name,
+		      int           val)
+{
+    char s[64];
+    
+    if (snprintf(s, sizeof(s)-1, "%u", val) < 0)
+	return -1;
+    return clicon_option_str_set(h, name, s);
+}
+
 /*! Delete option 
  */
 int
@@ -509,146 +557,18 @@ clicon_option_del(clicon_handle h,
 }
 
 /*-----------------------------------------------------------------
- * Specific option access functions.
+ * Specific option access functions for YANG configuration variables.
- * These should be commonly accessible for all users of clicon lib 
+ * Sometimes overridden by command-line options, 
+ * such as -f for CLICON_CONFIGFILE
+ * @see yang/clixon-config@<date>.yang
+ * You can always use the basic access functions, such as
+ * clicon_option_str[_set]
+ * But sometimes there are type conversions, etc which makes it more
+ * convenient to make wrapper functions. Or not?
  *-----------------------------------------------------------------*/
-
-char *
-clicon_configfile(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_CONFIGFILE");
-}
-
-/*! YANG database specification directory */
-char *
-clicon_yang_dir(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_YANG_DIR");
-}
-
-/*! YANG main module or absolute file name */
-char *
-clicon_yang_module_main(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_YANG_MODULE_MAIN");
-}
-
-/*! YANG revision */
-char *
-clicon_yang_module_revision(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_YANG_MODULE_REVISION");
-}
-
-/*! Directory of backend plugins. If null, no plugins are loaded */
-char *
-clicon_backend_dir(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_BACKEND_DIR");
-}
-
-/* contains .so files CLICON_CLI_DIR */
-char *
-clicon_cli_dir(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_CLI_DIR");
-}
-
-/* contains .cli files - CLICON_CLISPEC_DIR */
-char *
-clicon_clispec_dir(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_CLISPEC_DIR");
-}
-
-char *
-clicon_netconf_dir(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_NETCONF_DIR");
-}
-
-char *
-clicon_restconf_dir(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_RESTCONF_DIR");
-}
-
-char *
-clicon_xmldb_plugin(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_XMLDB_PLUGIN");
-}
-
-int
-clicon_startup_mode(clicon_handle h)
-{
-    char *mode;
-    if ((mode = clicon_option_str(h, "CLICON_STARTUP_MODE")) == NULL)
-	return -1;
-    return clicon_str2int(startup_mode_map, mode);
-}
-
-/*! Get family of backend socket: AF_UNIX, AF_INET or AF_INET6 */
-int
-clicon_sock_family(clicon_handle h)
-{
-    char *s;
-
-    if ((s = clicon_option_str(h, "CLICON_SOCK_FAMILY")) == NULL)
-	return AF_UNIX;
-    else  if (strcmp(s, "IPv4")==0)
-	return AF_INET;
-    else  if (strcmp(s, "IPv6")==0)
-	return AF_INET6;
-    else
-	return AF_UNIX; /* default */
-}
-
-/*! Get information about socket: unix domain filepath, or addr:path */
-char *
-clicon_sock(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_SOCK");
-}
-
-/*! Get port for backend socket in case of AF_INET or AF_INET6 */
-int
-clicon_sock_port(clicon_handle h)
-{
-    char *s;
-
-    if ((s = clicon_option_str(h, "CLICON_SOCK_PORT")) == NULL)
-	return -1;
-    return atoi(s);
-}
-
-char *
-clicon_backend_pidfile(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_BACKEND_PIDFILE");
-}
-
-char *
-clicon_sock_group(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_SOCK_GROUP");
-}
-
-char *
-clicon_master_plugin(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_MASTER_PLUGIN");
-}
-
-/*! Return initial clicon cli mode */
-char *
-clicon_cli_mode(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_CLI_MODE");
-}
-
 /*! Whether to generate CLIgen syntax from datamodel or not (0 or 1)
  * Must be used with a previous clicon_option_exists().
+ * @see clixon-config@<date>.yang CLICON_CLI_GENMODEL
  */
 int
 clicon_cli_genmodel(clicon_handle h)
@@ -661,7 +581,23 @@ clicon_cli_genmodel(clicon_handle h)
 	return 0;
 }
 
-/*! How to generate and show CLI syntax: VARS|ALL */
+/*! Generate code for CLI completion of existing db symbols
+ * @see clixon-config@<date>.yang CLICON_CLI_GENMODEL_COMPLETION
+ */
+int
+clicon_cli_genmodel_completion(clicon_handle h)
+{
+    char const *opt = "CLICON_CLI_GENMODEL_COMPLETION";
+
+    if (clicon_option_exists(h, opt))
+	return clicon_option_int(h, opt);
+    else
+	return 0;
+}
+
+/*! How to generate and show CLI syntax: VARS|ALL 
+ * @see clixon-config@<date>.yang CLICON_CLI_GENMODEL_TYPE
+ */
 enum genmodel_type
 clicon_cli_genmodel_type(clicon_handle h)
 {
@@ -685,32 +621,8 @@ clicon_cli_genmodel_type(clicon_handle h)
     return gt;
 }
 
-
+/*! Get Dont include keys in cvec in cli vars callbacks
-/* eg -q option, dont print notifications on stdout */
+ * @see clixon-config@<date>.yang CLICON_CLI_VARONLY
-char *
-clicon_quiet_mode(clicon_handle h)
-{
-    return clicon_option_str(h, "CLICON_QUIET");
-}
-
-int
-clicon_autocommit(clicon_handle h)
-{
-    char const *opt = "CLICON_AUTOCOMMIT";
-
-    if (clicon_option_exists(h, opt))
-	return clicon_option_int(h, opt);
-    else
-	return 0;
-}
-
-int
-clicon_autocommit_set(clicon_handle h, int val)
-{
-    return clicon_option_int_set(h, "CLICON_AUTOCOMMIT", val);
-}
-
-/*! Dont include keys in cvec in cli vars callbacks
  */
 int
 clicon_cli_varonly(clicon_handle h)
@@ -723,16 +635,43 @@ clicon_cli_varonly(clicon_handle h)
 	return 0;
 }
 
+/*! Get family of backend socket: AF_UNIX, AF_INET or AF_INET6 
+ * @see clixon-config@<date>.yang CLICON_SOCK_FAMILY
+ */
 int
-clicon_cli_varonly_set(clicon_handle h, int val)
+clicon_sock_family(clicon_handle h)
 {
-    return clicon_option_int_set(h, "CLICON_CLI_VARONLY", val);
+    char *s;
+
+    if ((s = clicon_option_str(h, "CLICON_SOCK_FAMILY")) == NULL)
+	return AF_UNIX;
+    else  if (strcmp(s, "IPv4")==0)
+	return AF_INET;
+    else  if (strcmp(s, "IPv6")==0)
+	return AF_INET6;
+    else
+	return AF_UNIX; /* default */
 }
 
+/*! Get port for backend socket in case of AF_INET or AF_INET6 
+ * @see clixon-config@<date>.yang CLICON_SOCK_PORT
+ */
 int
-clicon_cli_genmodel_completion(clicon_handle h)
+clicon_sock_port(clicon_handle h)
 {
-    char const *opt = "CLICON_CLI_GENMODEL_COMPLETION";
+    char *s;
+
+    if ((s = clicon_option_str(h, "CLICON_SOCK_PORT")) == NULL)
+	return -1;
+    return atoi(s);
+}
+
+/*! Set if all configuration changes are committed automatically 
+ */
+int
+clicon_autocommit(clicon_handle h)
+{
+    char const *opt = "CLICON_AUTOCOMMIT";
 
     if (clicon_option_exists(h, opt))
 	return clicon_option_int(h, opt);
@@ -740,14 +679,39 @@ clicon_cli_genmodel_completion(clicon_handle h)
 	return 0;
 }
 
-/*! Where are "running" and "candidate" databases? */
+/*! Which method to boot/start clicon backen
-char *
+ */
-clicon_xmldb_dir(clicon_handle h)
+int
+clicon_startup_mode(clicon_handle h)
 {
-    return clicon_option_str(h, "CLICON_XMLDB_DIR");
+    char *mode;
+    if ((mode = clicon_option_str(h, "CLICON_STARTUP_MODE")) == NULL)
+	return -1;
+    return clicon_str2int(startup_mode_map, mode);
 }
 
-/*! Get YANG specification
+/*---------------------------------------------------------------------
+ * Specific option access functions for non-yang options
+ * Typically dynamic values and more complex datatypes,
+ * Such as handles to plugins, API:s and parsed structures
+ *--------------------------------------------------------------------*/
+
+/* eg -q option, dont print notifications on stdout */
+int
+clicon_quiet_mode(clicon_handle h)
+{
+    char *s;
+    if ((s = clicon_option_str(h, "CLICON_QUIET")) == NULL)
+	return 0; /* default */
+    return atoi(s);
+}
+int
+clicon_quiet_mode_set(clicon_handle h, int val)
+{
+    return clicon_option_int_set(h, "CLICON_QUIET", val);
+}
+
+/*! Get YANG specification for application
  * Must use hash functions directly since they are not strings.
  */
 yang_spec *
@@ -762,7 +726,7 @@ clicon_dbspec_yang(clicon_handle h)
     return NULL;
 }
 
-/*! Set yang database specification
+/*! Set yang specification for application
  * ys must be a malloced pointer
  */
 int
@@ -779,6 +743,39 @@ clicon_dbspec_yang_set(clicon_handle     h,
     return 0;
 }
 
+/*! Get YANG NETCONF specification
+ * Must use hash functions directly since they are not strings.
+ */
+yang_spec *
+clicon_netconf_yang(clicon_handle h)
+{
+    clicon_hash_t  *cdat = clicon_data(h);
+    size_t          len;
+    void           *p;
+
+    if ((p = hash_value(cdat, "netconf_yang", &len)) != NULL)
+	return *(yang_spec **)p;
+    return NULL;
+}
+
+/*! Set yang netconf specification
+ * ys must be a malloced pointer
+ */
+int
+clicon_netconf_yang_set(clicon_handle     h, 
+		       struct yang_spec *ys)
+{
+    clicon_hash_t  *cdat = clicon_data(h);
+
+    /* It is the pointer to ys that should be copied by hash,
+       so we send a ptr to the ptr to indicate what to copy.
+     */
+    if (hash_add(cdat, "netconf_yang", &ys, sizeof(ys)) == NULL)
+	return -1;
+    return 0;
+}
+
+
 /*! Get dbspec name as read from spec. Can be used in CLI '@' syntax.
  * XXX: this we muśt change,...
  */
@@ -798,6 +795,19 @@ clicon_dbspec_name_set(clicon_handle h, char *name)
     return clicon_option_str_set(h, "dbspec_name", name);
 }
 
+/*! Get xmldb datastore plugin handle, as used by dlopen/dlsym/dlclose */
+plghndl_t
+clicon_xmldb_plugin_get(clicon_handle h)
+{
+    clicon_hash_t  *cdat = clicon_data(h);
+    size_t          len;
+    void           *p;
+
+    if ((p = hash_value(cdat, "xmldb_plugin", &len)) != NULL)
+	return *(plghndl_t*)p;
+    return NULL;
+}
+
 /*! Set xmldb datastore plugin handle, as used by dlopen/dlsym/dlclose */
 int
 clicon_xmldb_plugin_set(clicon_handle h, 
@@ -810,16 +820,20 @@ clicon_xmldb_plugin_set(clicon_handle h,
     return 0;
 }
 
-/*! Get xmldb datastore plugin handle, as used by dlopen/dlsym/dlclose */
+/*! Get XMLDB API struct pointer
-plghndl_t
+ * @param[in]  h   Clicon handle
-clicon_xmldb_plugin_get(clicon_handle h)
+ * @retval     xa  XMLDB API struct
+ * @note xa is really of type struct xmldb_api*
+ */
+void *
+clicon_xmldb_api_get(clicon_handle h)
 {
     clicon_hash_t  *cdat = clicon_data(h);
     size_t          len;
-    void           *p;
+    void           *xa;
 
-    if ((p = hash_value(cdat, "xmldb_plugin", &len)) != NULL)
+    if ((xa = hash_value(cdat, "xmldb_api", &len)) != NULL)
-	return *(plghndl_t*)p;
+	return *(void**)xa;
     return NULL;
 }
 
@@ -842,20 +856,19 @@ clicon_xmldb_api_set(clicon_handle     h,
     return 0;
 }
 
-/*! Get XMLDB API struct pointer
+/*! Get XMLDB storage handle
  * @param[in]  h   Clicon handle
- * @retval     xa  XMLDB API struct
+ * @retval     xh  XMLDB storage handle. If not connected return NULL
- * @note xa is really of type struct xmldb_api*
  */
 void *
-clicon_xmldb_api_get(clicon_handle h)
+clicon_xmldb_handle_get(clicon_handle h)
 {
     clicon_hash_t  *cdat = clicon_data(h);
     size_t          len;
-    void           *xa;
+    void           *xh;
 
-    if ((xa = hash_value(cdat, "xmldb_api", &len)) != NULL)
+    if ((xh = hash_value(cdat, "xmldb_handle", &len)) != NULL)
-	return *(void**)xa;
+	return *(void**)xh;
     return NULL;
 }
 
@@ -875,18 +888,4 @@ clicon_xmldb_handle_set(clicon_handle h,
     return 0;
 }
 
-/*! Get XMLDB storage handle
- * @param[in]  h   Clicon handle
- * @retval     xh  XMLDB storage handle. If not connected return NULL
- */
-void *
-clicon_xmldb_handle_get(clicon_handle h)
-{
-    clicon_hash_t  *cdat = clicon_data(h);
-    size_t          len;
-    void           *xh;
 
-    if ((xh = hash_value(cdat, "xmldb_handle", &len)) != NULL)
-	return *(void**)xh;
-    return NULL;
-}

lib/src/clixon_proto.c

@@ -173,7 +173,7 @@ clicon_msg_decode(struct clicon_msg *msg,
     /* body */
     xmlstr = msg->op_body;
     clicon_debug(1, "%s %s", __FUNCTION__, xmlstr);
-    if (clicon_xml_parse_str(xmlstr, xml) < 0)
+    if (xml_parse_string(xmlstr, NULL, xml) < 0)
 	goto done;
     retval = 0;
  done:

lib/src/clixon_proto_client.c

@@ -120,7 +120,7 @@ clicon_rpc_msg(clicon_handle      h,
     }
     clicon_debug(1, "%s retdata:%s", __FUNCTION__, retdata);
     if (retdata &&
-	clicon_xml_parse_str(retdata, &xret) < 0)
+	xml_parse_string(retdata, NULL, &xret) < 0)
 	goto done;
     if (xret0){
 	*xret0 = xret;
@@ -274,7 +274,7 @@ clicon_rpc_get_config(clicon_handle       h,
     if ((xd = xpath_first(xret, "/rpc-reply/rpc-error")) != NULL)
 	xd = xml_parent(xd); /* point to rpc-reply */
     else if ((xd = xpath_first(xret, "/rpc-reply/data")) == NULL)
-	if ((xd = xml_new("data", NULL)) == NULL)
+	if ((xd = xml_new("data", NULL, NULL)) == NULL)
 	    goto done;
     if (xt){
 	if (xml_rm(xd) < 0)
@@ -522,7 +522,7 @@ clicon_rpc_get(clicon_handle       h,
     if ((xd = xpath_first(xret, "/rpc-reply/rpc-error")) != NULL)
 	xd = xml_parent(xd); /* point to rpc-reply */
     else if ((xd = xpath_first(xret, "/rpc-reply/data")) == NULL)
-	if ((xd = xml_new("data", NULL)) == NULL)
+	if ((xd = xml_new("data", NULL, NULL)) == NULL)
 	    goto done;
     if (xt){
 	if (xml_rm(xd) < 0)

lib/src/clixon_sig.c

@@ -182,7 +182,7 @@ pidfile_write(char *pidfile)
     int   retval = -1;
 
     /* Here, there should be no old agent and no pidfile */
-    if ((f = fopen(pidfile, "wb")) == NULL){
+    if ((f = fopen(pidfile, "w")) == NULL){
 	if (errno == EACCES)
 	    clicon_err(OE_DEMON, errno, "Creating pid-file %s (Try run as root?)", pidfile);
 	else

lib/src/clixon_xml_db.c

@@ -56,8 +56,8 @@
 #include "clixon_queue.h"
 #include "clixon_hash.h"
 #include "clixon_handle.h"
-#include "clixon_xml.h"
 #include "clixon_yang.h"
+#include "clixon_xml.h"
 #include "clixon_plugin.h"
 #include "clixon_options.h"
 #include "clixon_xml_db.h"
@@ -382,7 +382,7 @@ xmldb_get(clicon_handle h,
  * The xml may contain the "operation" attribute which defines the operation.
  * @code
  *   cxobj     *xt;
- *   if (clicon_xml_parse_str("<a>17</a>", &xt) < 0)
+ *   if (xml_parse_string("<a>17</a>", yspec, &xt) < 0)
  *     err;
  *   if (xmldb_put(xh, "running", OP_MERGE, xt) < 0)
  *     err;
@@ -432,7 +432,7 @@ xmldb_put(clicon_handle       h,
 
 /*! Copy database from db1 to db2
  * @param[in]  h     Clicon handle
- * @param[in]  from  Source database copy
+ * @param[in]  from  Source database
  * @param[in]  to    Destination database
  * @retval -1  Error
  * @retval  0  OK

lib/src/clixon_xml_map.c

@@ -85,6 +85,7 @@
 #include "clixon_xsl.h"
 #include "clixon_log.h"
 #include "clixon_err.h"
+#include "clixon_xml_sort.h"
 #include "clixon_xml_map.h"
 
 /* Something to do with reverse engineering of junos syntax? */
@@ -228,7 +229,10 @@ xml2cli(FILE              *f,
 	    term = xml_name(x);
 	if (prepend0)
 	    fprintf(f, "%s ", prepend0);
-	fprintf(f, "%s\n", term);
+	if (index(term, ' '))
+	    fprintf(f, "\"%s\"\n", term);
+	else
+	    fprintf(f, "%s\n", term);
 	retval = 0;
 	goto done;
     }
@@ -561,7 +565,7 @@ cvec2xml_1(cvec   *cvv,
     cv = NULL;
     while ((cv = cvec_each(cvv, cv)) != NULL) 
 	len++;
-    if ((xt = xml_new(toptag, xp)) == NULL)
+    if ((xt = xml_new(toptag, xp, NULL)) == NULL)
 	goto err;
     if (xml_childvec_set(xt, len) < 0)
 	goto err;
@@ -570,11 +574,11 @@ cvec2xml_1(cvec   *cvv,
     while ((cv = cvec_each(cvv, cv)) != NULL) {
 	if (cv_type_get(cv)==CGV_ERR || cv_name_get(cv) == NULL)
 	    continue;
-	if ((xn = xml_new(cv_name_get(cv), NULL)) == NULL) /* this leaks */
+	if ((xn = xml_new(cv_name_get(cv), NULL, NULL)) == NULL) /* this leaks */
 	    goto err;
 	xml_parent_set(xn, xt);
 	xml_child_i_set(xt, i++, xn);
-	if ((xb = xml_new("body", xn)) == NULL) /* this leaks */
+	if ((xb = xml_new("body", xn, NULL)) == NULL) /* this leaks */
 	    goto err;
 	xml_type_set(xb, CX_BODY);
 	val = cv2str_dup(cv);
@@ -590,138 +594,6 @@ cvec2xml_1(cvec   *cvv,
     return retval;
 }
 
-/*! Given child tree x1c, find matching child in base tree x0
- * param[in]  x0   Base tree node
- * param[in]  x1c  Modification tree child
- * param[in]  yc   Yang spec of tree child
- * param[out] x0cp Matching base tree child (if any)
- * @note XXX: room for optimization? on 1K calls we have 1M body calls and
-	   500K xml_child_each/cvec_each calls. 
-	   The outer loop is large for large lists
-	   The inner loop is small
-	   Major time in xml_find_body()
-	   Can one do a binary search in the x0 list?
-*/
-int
-match_base_child(cxobj     *x0, 
-		 cxobj     *x1c,
-		 cxobj    **x0cp,
-		 yang_stmt *yc)
-{
-    int        retval = -1;
-    char      *x1cname;
-    cxobj     *x0c = NULL; /* x0 child */
-    cvec      *cvk = NULL; /* vector of index keys */
-    cg_var    *cvi;
-    char      *b0;
-    char      *b1;
-    yang_stmt *ykey;
-    char      *keyname;
-    int        equal;
-    char     **b1vec = NULL;
-    int        i;
-#if (XML_CHILD_HASH==1)
-    cxobj **p;
-    cbuf   *key = NULL; /* cligen buffer hash key */
-    size_t  vlen;
-
-    *x0cp = NULL; /* return value */	    
-    if (xml_hash(x0) == NULL)
-	goto nohash;
-    if ((key = cbuf_new()) == NULL){
-	clicon_err(OE_XML, errno, "cbuf_new");
-	goto done1;
-    }
-    if (xml_hash_key(x1c, yc, key) < 0)
-	goto done;
-    x0c = NULL;
-    if (cbuf_len(key))
-	if ((p = hash_value(xml_hash(x0), cbuf_get(key), &vlen)) != NULL){
-	    assert(vlen == sizeof(x0c));
-	    x0c = *p;
-	}
-    //    fprintf(stderr, "%s get %s = 0x%x\n", __FUNCTION__, cbuf_get(key), (unsigned int)x0c);
-    *x0cp = x0c;
-    retval = 0;
- done1:
-    if (key)
-	cbuf_free(key);
-    return retval;
- nohash:
-#endif /* XML_CHILD_HASH */
-    *x0cp = NULL; /* return value */	    
-    x1cname = xml_name(x1c);
-    switch (yc->ys_keyword){
-    case Y_CONTAINER: 	/* Equal regardless */
-    case Y_LEAF: 	/* Equal regardless */
-	x0c = xml_find(x0, x1cname);
-	break;
-    case Y_LEAF_LIST: /* Match with name and value */
-	if ((b1 = xml_body(x1c)) == NULL)
-	    goto ok;
-	x0c = xml_find_body_obj(x0, x1cname, b1);
-	break;
-    case Y_LIST: /* Match with key values */
-	if ((ykey = yang_find((yang_node*)yc, Y_KEY, NULL)) == NULL){
-	    clicon_err(OE_XML, errno, "%s: List statement \"%s\" has no key", 
-		       __FUNCTION__, yc->ys_argument);
-	    goto done;
-	}
-	/* The value is a list of keys: <key>[ <key>]*  */
-	if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
-	    goto done;
-	cvi = NULL; i = 0;
-	while ((cvi = cvec_each(cvk, cvi)) != NULL) 
-	    i++;
-	if ((b1vec = calloc(i, sizeof(b1))) == NULL){
-	    clicon_err(OE_UNIX, errno, "calloc");
-	    goto done;
-	}
-	cvi = NULL; i = 0;
-	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
-	    keyname = cv_string_get(cvi);
-	    if ((b1 = xml_find_body(x1c, keyname)) == NULL){
-		clicon_err(OE_UNIX, errno, "key %s not found", keyname);
-		goto done;
-	    }
-	    b1vec[i++] = b1;
-	}
-	/* Iterate over x0 tree to (1) find a child that matches name
-	   (2) that have keys that matches */
-	x0c = NULL;
-	while ((x0c = xml_child_each(x0, x0c, CX_ELMNT)) != NULL){
-	    equal = 0;
-	    if (strcmp(xml_name(x0c), x1cname))
-		continue;
-	    /* Must be inner loop */
-	    cvi = NULL; i = 0;
-	    while ((cvi = cvec_each(cvk, cvi)) != NULL) {
-		b1 = b1vec[i++];
-		equal = 0;
-		keyname = cv_string_get(cvi);
-		if ((b0 = xml_find_body(x0c, keyname)) == NULL)
-		    break; /* error case */
-		if (strcmp(b0, b1))
-		    break; /* stop as soon as inequal key found */
-		equal=1; /* reaches here for all keynames, x0c is found. */
-	    }
-	    if (equal) /* x0c and x1c equal, otherwise look for other */
-		break;
-	} /* while x0c */
-	break;
-    default:
-	break;
-    }
- ok:
-    *x0cp = x0c;
-    retval = 0;
- done:
-    if (b1vec)
-	free(b1vec);
-    if (cvk)
-	cvec_free(cvk);
-    return retval;
-}
 
 /*! Find next yang node, either start from yang_spec or some yang-node
  * @param[in]  y    Node spec or sny yang-node
@@ -897,7 +769,6 @@ yang2api_path_fmt_1(yang_stmt *ys,
 		    cbuf      *cb)
 {
     yang_node *yp; /* parent */
-    yang_stmt *ykey;
     int        i;
     cvec      *cvk = NULL; /* vector of index keys */
     int        retval = -1;
@@ -928,14 +799,7 @@ yang2api_path_fmt_1(yang_stmt *ys,
 
     switch (ys->ys_keyword){
     case Y_LIST:
-	if ((ykey = yang_find((yang_node*)ys, Y_KEY, NULL)) == NULL){
+	cvk = ys->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
-	    clicon_err(OE_XML, errno, "%s: List statement \"%s\" has no key", 
-		       __FUNCTION__, ys->ys_argument);
-	    goto done;
-	}
-	/* The value is a list of keys: <key>[ <key>]*  */
-	if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
-	    goto done;
 	if (cvec_len(cvk))
 	    cprintf(cb, "=");
 	/* Iterate over individual keys  */
@@ -953,8 +817,6 @@ yang2api_path_fmt_1(yang_stmt *ys,
     } /* switch */
     retval = 0;
  done:
-    if (cvk)
-	cvec_free(cvk);
     return retval;
 }
 
@@ -1204,7 +1066,7 @@ xml_tree_prune_flagged_sub(cxobj *xt,
     cxobj     *xprev;
     int        iskey;
     int        anykey=0;
-    yang_node *yt;
+    yang_stmt *yt;
 
     mark = 0;
     yt = xml_spec(xt); /* xan be null */
@@ -1219,7 +1081,7 @@ xml_tree_prune_flagged_sub(cxobj *xt,
 	}
 	/* If it is key dont remove it yet (see second round) */
 	if (yt){
-	    if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
+	    if ((iskey = yang_key_match((yang_node*)yt, xml_name(x))) < 0)
 		goto done;
 	    if (iskey){
 		anykey++;
@@ -1247,7 +1109,7 @@ xml_tree_prune_flagged_sub(cxobj *xt,
 	while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
 	    /* If it is key remove it here */
 	    if (yt){
-		if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
+		if ((iskey = yang_key_match((yang_node*)yt, xml_name(x))) < 0)
 		    goto done;
 		if (iskey && xml_purge(x) < 0)
 		    goto done;
@@ -1328,9 +1190,9 @@ xml_default(cxobj *xt,
 	    assert(y->ys_cv);
 	    if (!cv_flag(y->ys_cv, V_UNSET)){  /* Default value exists */
 		if (!xml_find(xt, y->ys_argument)){
-		    if ((xc = xml_new_spec(y->ys_argument, xt, y)) == NULL)
+		    if ((xc = xml_new(y->ys_argument, xt, y)) == NULL)
 			goto done;
-		    if ((xb = xml_new("body", xc)) == NULL)
+		    if ((xb = xml_new("body", xc, NULL)) == NULL)
 			goto done;
 		    xml_type_set(xb, CX_BODY);
 		    if ((str = cv2str_dup(y->ys_cv)) == NULL){
@@ -1344,6 +1206,7 @@ xml_default(cxobj *xt,
 	    }
 	}
     }
+    xml_sort(xt, NULL);
     retval = 0;
  done:
     return retval;
@@ -1372,13 +1235,16 @@ xml_order(cxobj *xt,
 	goto done;
     }
     j0 = 0;
- /* Go through xml children and ensure they are same order as yspec children */
+    /* Go through yang node's children and ensure that the 
+     * xml children follow this order.
+     * Do not order the list or leaf-list children (have same name).
+     */
     for (i=0; i<y->ys_len; i++){
 	yc = y->ys_stmt[i];
 	if (!yang_datanode(yc))
 	    continue;
 	yname = yc->ys_argument;
-	/* First go thru xml children with same name */
+	/* First go thru xml children with same name in rest of list */
 	for (; j0<xml_child_nr(xt); j0++){
 	    xc = xml_child_i(xt, j0);
 	    if (xml_type(xc) != CX_ELMNT)
@@ -1458,6 +1324,7 @@ xml_non_config_data(cxobj *xt,
     return retval;
 }
 
+
 /*! Add yang specification backpoint to XML node
  * @param[in]   xt      XML tree node
  * @param[in]   arg     Yang spec
@@ -1474,21 +1341,21 @@ xml_spec_populate(cxobj  *x,
 {
     int        retval = -1;
     yang_spec *yspec = (yang_spec*)arg;
-    char      *name;
     yang_stmt *y=NULL;  /* yang node */
-    cxobj     *xp;      /* xml parent */ 
-    yang_stmt *yp;      /* parent yang */
 
-    name = xml_name(x);
+    if (xml_child_spec(xml_name(x), xml_parent(x), yspec, &y) < 0)
+	goto done;
+#if 0
     if ((xp = xml_parent(x)) != NULL &&
 	(yp = xml_spec(xp)) != NULL)
 	y = yang_find_datanode((yang_node*)yp, xml_name(x));
     else
 	y = yang_find_topnode(yspec, name, 0); /* still NULL for config */
+#endif
     if (y)
 	xml_spec_set(x, y);
     retval = 0;
-    // done:
+ done:
     return retval;
 }
 
@@ -1529,7 +1396,6 @@ api_path2xpath_cvv(yang_spec *yspec,
     yang_stmt *y = NULL;
     char      *val;
     char      *v;
-    yang_stmt *ykey;
     cg_var    *cvi;
 
     for (i=offset; i<cvec_len(cvv); i++){
@@ -1560,17 +1426,7 @@ api_path2xpath_cvv(yang_spec *yspec,
 		*v = '\0';
 		v++;
 	    }
-	    /* Find keys */
+	    cvk = y->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
-	    if ((ykey = yang_find((yang_node*)y, Y_KEY, NULL)) == NULL){
-		clicon_err(OE_XML, errno, "%s: List statement \"%s\" has no key", 
-			   __FUNCTION__, y->ys_argument);
-		goto done;
-	    }
-	    clicon_debug(1, "ykey:%s", ykey->ys_argument);
-
-	    /* The value is a list of keys: <key>[ <key>]*  */
-	    if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
-		goto done;
 	    cvi = NULL;
 	    /* Iterate over individual yang keys  */
 	    cprintf(xpath, "/%s", name);
@@ -1648,7 +1504,6 @@ api_path2xml_vec(char             **vec,
     char      *name;
     char      *restval = NULL;
     char      *restval_enc;
-    yang_stmt *ykey;
     cxobj     *xn = NULL; /* new */
     cxobj     *xb;        /* body */
     cvec      *cvk = NULL; /* vector of index keys */
@@ -1700,25 +1555,17 @@ api_path2xml_vec(char             **vec,
 	    clicon_err(OE_XML, 0, "malformed key, expected '=<restval>'");
 	    goto done;
 	}
-	if ((x = xml_new_spec(y->ys_argument, x0, y)) == NULL)
+	if ((x = xml_new(y->ys_argument, x0, y)) == NULL)
 	    goto done;
 	xml_type_set(x, CX_ELMNT);
-	if ((xb = xml_new("body", x)) == NULL)
+	if ((xb = xml_new("body", x, NULL)) == NULL)
 	    goto done; 
 	xml_type_set(xb, CX_BODY);
 	if (restval && xml_value_set(xb, restval) < 0)
 	    goto done;
 	break;
     case Y_LIST:
-	/* Get the yang list key */
+	cvk = y->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
-	if ((ykey = yang_find((yang_node*)y, Y_KEY, NULL)) == NULL){
-	    clicon_err(OE_XML, errno, "%s: List statement \"%s\" has no key", 
-		       __FUNCTION__, y->ys_argument);
-	    goto done;
-	}
-	/* The value is a list of keys: <key>[ <key>]*  */
-	if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
-	    goto done;
 	if (valvec){
 	    free(valvec);
 	    valvec = NULL;
@@ -1738,31 +1585,26 @@ api_path2xml_vec(char             **vec,
 	}
 	cvi = NULL;
 	/* create list object */
-	if ((x = xml_new_spec(name, x0, y)) == NULL)
+	if ((x = xml_new(name, x0, y)) == NULL)
 	    goto done; 
 	xml_type_set(x, CX_ELMNT);
 	j = 0;
 	/* Create keys */
 	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
 	    keyname = cv_string_get(cvi);
-
+	    if ((xn = xml_new(keyname, x, NULL)) == NULL)
-	    if ((xn = xml_new(keyname, x)) == NULL)
 		goto done; 
 	    xml_type_set(xn, CX_ELMNT);
-	    if ((xb = xml_new("body", xn)) == NULL)
+	    if ((xb = xml_new("body", xn, NULL)) == NULL)
 		goto done; 
 	    xml_type_set(xb, CX_BODY);
 	    val2 = valvec?valvec[j++]:NULL;
 	    if (xml_value_set(xb, val2) <0)
 		goto done;
 	}
-	if (cvk){
-	    cvec_free(cvk);
-	    cvk = NULL;
-	}
 	break;
     default: /* eg Y_CONTAINER, Y_LEAF */
-	if ((x = xml_new_spec(name, x0, y)) == NULL)
+	if ((x = xml_new(name, x0, y)) == NULL)
 	    goto done; 
 	xml_type_set(x, CX_ELMNT);
 	break;
@@ -1858,17 +1700,17 @@ xml_merge1(cxobj              *x0,
     if (y0->yn_keyword == Y_LEAF_LIST || y0->yn_keyword == Y_LEAF){
 	x1bstr = xml_body(x1);
 	if (x0==NULL){
-	    if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL)
+	    if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		goto done;
 	    if (x1bstr){ /* empty type does not have body */
-		if ((x0b = xml_new("body", x0)) == NULL)
+		if ((x0b = xml_new("body", x0, NULL)) == NULL)
 		    goto done; 
 		xml_type_set(x0b, CX_BODY);
 	    }
 	}
 	if (x1bstr){
 	    if ((x0b = xml_body_get(x0)) == NULL){
-		if ((x0b = xml_new("body", x0)) == NULL)
+		if ((x0b = xml_new("body", x0, NULL)) == NULL)
 		    goto done; 
 		xml_type_set(x0b, CX_BODY);
 	    }
@@ -1879,7 +1721,7 @@ xml_merge1(cxobj              *x0,
     } /* if LEAF|LEAF_LIST */
     else { /* eg Y_CONTAINER, Y_LIST  */
 	if (x0==NULL){
-	    if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL)
+	    if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		goto done;
 	}
 	/* Loop through children of the modification tree */
@@ -1992,6 +1834,7 @@ done:
     return retval;
 }
 
+
 /*
  * Turn this on for uni-test programs
  * Usage: clixon_string join

lib/src/clixon_xml_parse.h

@@ -32,6 +32,8 @@
   ***** END LICENSE BLOCK *****
 
  * XML parser
+ * @see https://www.w3.org/TR/2008/REC-xml-20081126
+ *      https://www.w3.org/TR/2009/REC-xml-names-20091208
  */
 #ifndef _CLIXON_XML_PARSE_H_
 #define _CLIXON_XML_PARSE_H_
@@ -39,6 +41,7 @@
 /*
  * Types
  */
+/*! XML parser yacc handler struct */
 struct xml_parse_yacc_arg{
     char       *ya_parse_string; /* original (copy of) parse string */
     int         ya_linenum;      /* Number of \n in parsed buffer */
@@ -46,7 +49,8 @@ struct xml_parse_yacc_arg{
 
     cxobj      *ya_xelement;     /* xml active element */
     cxobj      *ya_xparent;      /* xml parent element*/
-    int         ya_skipspace;    /* If set, remove all non-terminal bodies (strip prettyr-print) */
+    int         ya_skipspace;    /* If set, remove all non-terminal bodies (strip pretty-print) */
+    yang_spec  *ya_yspec;        /* If set, top-level yang-spec */
 };
 
 extern char *clixon_xml_parsetext;

lib/src/clixon_xml_parse.l

@@ -32,6 +32,8 @@
   ***** END LICENSE BLOCK *****
 
  * XML parser
+ * @see https://www.w3.org/TR/2008/REC-xml-20081126
+ *      https://www.w3.org/TR/2009/REC-xml-names-20091208
  */
 
 %{

lib/src/clixon_xml_parse.y

@@ -32,6 +32,8 @@
   ***** END LICENSE BLOCK *****
 
  * XML parser
+ * @see https://www.w3.org/TR/2008/REC-xml-20081126
+ *      https://www.w3.org/TR/2009/REC-xml-names-20091208
  */
 %union {
   char *string;
@@ -46,7 +48,7 @@
 %token BCOMMENT ECOMMENT 
 
 
-%type <string> val aid
+%type <string> attvalue attqname
 
 %lex-param     {void *_ya} /* Add this argument to parse() and lex() function */
 %parse-param   {void *_ya}
@@ -68,7 +70,12 @@
 /* clicon */
 #include "clixon_err.h"
 #include "clixon_log.h"
+#include "clixon_queue.h"
+#include "clixon_hash.h"
+#include "clixon_handle.h"
+#include "clixon_yang.h"
 #include "clixon_xml.h"
+#include "clixon_xml_sort.h"
 #include "clixon_xml_parse.h"
 
 void 
@@ -79,8 +86,9 @@ clixon_xml_parseerror(void *_ya, char *s)
   return;
 }
 
-/* note that we dont handle escaped characters correctly 
+/*
-   there may also be some leakage here on NULL return
+ * Note that we dont handle escaped characters correctly 
+ * there may also be some leakage here on NULL return
  */
 static int
 xml_parse_content(struct xml_parse_yacc_arg *ya, 
@@ -92,7 +100,7 @@ xml_parse_content(struct xml_parse_yacc_arg *ya,
 
     ya->ya_xelement = NULL; /* init */
     if (xn == NULL){
-	if ((xn = xml_new("body", xp)) == NULL)
+	if ((xn = xml_new("body", xp, NULL)) == NULL)
 	    goto done; 
 	xml_type_set(xn, CX_BODY);
     }
@@ -105,7 +113,8 @@ xml_parse_content(struct xml_parse_yacc_arg *ya,
 }
 
 static int
-xml_parse_version(struct xml_parse_yacc_arg *ya, char *ver)
+xml_parse_version(struct xml_parse_yacc_arg *ya,
+		  char                      *ver)
 {
     if(strcmp(ver, "1.0")){
 	clicon_err(OE_XML, errno, "Wrong XML version %s expected 1.0\n", ver);
@@ -116,20 +125,35 @@ xml_parse_version(struct xml_parse_yacc_arg *ya, char *ver)
     return 0;
 }
 
+/*! Parse Qualified name
+ * @param[in] ya        XML parser yacc handler struct 
+ * @param[in] prefix    Prefix, namespace, or NULL
+ * @param[in] localpart Name
+ */
 static int
-xml_parse_id(struct xml_parse_yacc_arg *ya, char *name, char *namespace)
+xml_parse_qname(struct xml_parse_yacc_arg *ya,
+		char                      *prefix,
+		char                      *name)
 {
-    if ((ya->ya_xelement=xml_new(name, ya->ya_xparent)) == NULL) {
+    int        retval = -1;
-	if (namespace)
+    cxobj     *x;
-	    free(namespace);
+    yang_stmt *y = NULL;  /* yang node */   
-	free(name);
+    cxobj     *xp;      /* xml parent */ 
-	return -1;
+
-    }
+    xp = ya->ya_xparent;
-    xml_namespace_set(ya->ya_xelement, namespace);
+    if (xml_child_spec(name, xp, ya->ya_yspec, &y) < 0)
-    if (namespace)
+	goto done;
-	free(namespace);
+    if ((x = xml_new(name, xp, y)) == NULL) 
+	goto done;
+    if (xml_namespace_set(x, prefix) < 0)
+	goto done;
+    ya->ya_xelement = x;
+    retval = 0;
+ done:
+    if (prefix)
+	free(prefix);
     free(name);
-    return 0;
+    return retval;
 }
 
 static int
@@ -250,38 +274,45 @@ xml_parse_bslash2(struct xml_parse_yacc_arg *ya,
     return retval;
 }
 
+static int
+xml_parse_attr(struct xml_parse_yacc_arg *ya,
+	       char                      *qname,
+	       char                      *attval)
+{
+    int    retval = -1;
+    cxobj *xa; 
 
+    if ((xa = xml_new(qname, ya->ya_xelement, NULL)) == NULL)
+	goto done;
+    xml_type_set(xa, CX_ATTR);
+    if (xml_value_set(xa, attval) < 0)
+	goto done;
+    retval = 0;
+  done:
+    free(qname); 
+    free(attval);
+    return retval;
+}
+
+/*! Parse Attribue Qualified name, Just transform prefix:name into a new string
+ *
+ */
 static char*
-xml_parse_ida(struct xml_parse_yacc_arg *ya, char *namespace, char *name)
+xml_merge_attqname(struct xml_parse_yacc_arg *ya,
+		   char                      *prefix,
+		   char                      *name)
 {
     char *str;
-    int len = strlen(namespace)+strlen(name)+2;
+    int len = strlen(prefix)+strlen(name)+2;
 
     if ((str=malloc(len)) == NULL)
 	return NULL;
-    snprintf(str, len, "%s:%s", namespace, name);
+    snprintf(str, len, "%s:%s", prefix, name);
-    free(namespace);
+    free(prefix);
     free(name);
     return str;
 }
 
-static int
-xml_parse_attr(struct xml_parse_yacc_arg *ya, char *id, char *val)
-{
-    int retval = -1;
-    cxobj *xa; 
-
-    if ((xa = xml_new(id, ya->ya_xelement)) == NULL)
-	goto done;
-    xml_type_set(xa, CX_ATTR);
-    if (xml_value_set(xa, val) < 0)
-	goto done;
-    retval = 0;
-  done:
-    free(id); 
-    free(val);
-    return retval;
-}
  
 %} 
  
@@ -309,22 +340,22 @@ encode      : ENC '=' '\"' CHAR '\"' {free($4);}
             | ENC '=' '\'' CHAR '\'' {free($4);}
             ;
 
-emnt     : '<' id  attrs emnt1 
+element     : '<' qname  attrs element1 
-                   { clicon_debug(3, "emnt -> < id attrs emnt1"); }
+                   { clicon_debug(3, "element -> < qname attrs element1"); }
 	      ;
 
-id          : NAME            { if (xml_parse_id(_YA, $1, NULL) < 0) YYABORT; 
+qname       : NAME           { if (xml_parse_qname(_YA, NULL, $1) < 0) YYABORT; 
-                                clicon_debug(3, "id -> NAME %s", $1);}
+                                clicon_debug(3, "qname -> NAME %s", $1);}
-            | NAME ':' NAME   { if (xml_parse_id(_YA, $3, $1) < 0) YYABORT; 
+            | NAME ':' NAME  { if (xml_parse_qname(_YA, $1, $3) < 0) YYABORT; 
-                                clicon_debug(3, "id -> NAME : NAME");}
+                                clicon_debug(3, "qname -> NAME : NAME");}
             ;
 
-emnt1    :  ESLASH          {_YA->ya_xelement = NULL; 
+element1    :  ESLASH         {_YA->ya_xelement = NULL; 
-                               clicon_debug(3, "emnt1 -> />");} 
+                               clicon_debug(3, "element1 -> />");} 
-            | '>'              { xml_parse_endslash_pre(_YA); }
+            | '>'             { xml_parse_endslash_pre(_YA); }
-              list             { xml_parse_endslash_mid(_YA); }
+              list            { xml_parse_endslash_mid(_YA); }
               etg             { xml_parse_endslash_post(_YA); 
-                               clicon_debug(3, "emnt1 -> > list etg");} 
+                               clicon_debug(3, "element1 -> > list etg");} 
             ;
 
 etg         : BSLASH NAME '>'          
@@ -339,7 +370,7 @@ list        : list content { clicon_debug(3, "list -> list content"); }
             | content      { clicon_debug(3, "list -> content"); }
             ;
 
-content     : emnt         { clicon_debug(3, "content -> emnt"); }
+content     : element      { clicon_debug(3, "content -> element"); }
             | comment      { clicon_debug(3, "content -> comment"); }
             | CHAR         { if (xml_parse_content(_YA, $1) < 0) YYABORT;  
                              clicon_debug(3, "content -> CHAR %s", $1); }
@@ -350,20 +381,20 @@ comment     : BCOMMENT ECOMMENT
             ;
 
 
-attrs       : attrs att
+attrs       : attrs attr
             |
             ;
 
+attr        : attqname '=' attvalue { if (xml_parse_attr(_YA, $1, $3) < 0) YYABORT; }
+            ;
 
-aid         : NAME   {$$ = $1;}
+attqname    : NAME   {$$ = $1;}
             | NAME ':' NAME  
-                     { if (($$ = xml_parse_ida(_YA, $1, $3)) == NULL) YYABORT; }
+                     { if (($$ = xml_merge_attqname(_YA, $1, $3)) == NULL) YYABORT; }
             ;
 
-att         : aid '=' val { if (xml_parse_attr(_YA, $1, $3) < 0) YYABORT; }
-            ;
 
-val         : '\"' CHAR '\"'   { $$=$2; /* $2 must be consumed */}
+attvalue    : '\"' CHAR '\"'   { $$=$2; /* $2 must be consumed */}
             | '\"'  '\"'       { $$=strdup(""); /* $2 must be consumed */}
             ;
 

lib/src/clixon_xml_sort.c

@@ -0,0 +1,648 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2017 Olof Hagsand and Benny Holmgren
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+ * XML search functions when used with YANG
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <limits.h>
+#include <fnmatch.h>
+#include <stdint.h>
+#include <assert.h>
+#include <syslog.h>
+
+/* cligen */
+#include <cligen/cligen.h>
+
+/* clixon */
+#include "clixon_err.h"
+#include "clixon_log.h"
+#include "clixon_string.h"
+
+#include "clixon_queue.h"
+#include "clixon_hash.h"
+#include "clixon_handle.h"
+#include "clixon_yang.h"
+#include "clixon_xml.h"
+#include "clixon_xml_sort.h"
+
+/*
+ * Variables
+ */
+
+/* Sort and binary search of XML children
+ * Experimental
+ */
+int xml_child_sort = 1;
+
+
+/*! Given an XML object and a child name, return yang stmt of child
+ * If no xml parent, find root yang stmt matching name
+ * @param[in]  name     Name of child
+ * @param[in]  xp       XML parent, can be NULL.
+ * @param[in]  yspec    Yang specification (top level)
+ * @param[out] yresult  Pointer to yang stmt of result, or NULL, if not found
+ */
+int
+xml_child_spec(char       *name,
+	       cxobj      *xp,
+	       yang_spec  *yspec,
+	       yang_stmt **yresult)
+{
+    yang_stmt *y;  /* result yang node */   
+    yang_stmt *yparent; /* parent yang */
+    
+    if (xp && (yparent = xml_spec(xp)) != NULL)
+	y = yang_find_datanode((yang_node*)yparent, name);
+    else if (yspec)
+	y = yang_find_topnode(yspec, name, 0); /* still NULL for config */
+    else
+	y = NULL;
+    *yresult = y;
+    return 0;
+}
+
+/*! Help function to qsort for sorting entries in xml child vector
+ * @param[in]  arg1 - actually cxobj**
+ * @param[in]  arg2 - actually cxobj**
+ * @retval  0  If equal
+ * @retval <0  if arg1 is less than arg2
+ * @retval >0  if arg1 is greater than arg2
+ * @note args are pointer ot pointers, to fit into qsort cmp function
+ * @see xml_cmp1   Similar, but for one object
+ */
+int
+xml_cmp(const void* arg1, 
+	const void* arg2)
+{
+    cxobj *x1 = *(struct xml**)arg1;
+    cxobj *x2 = *(struct xml**)arg2;
+    yang_stmt  *y1;
+    yang_stmt  *y2;
+    int         yi1;
+    int         yi2;
+    cvec       *cvk = NULL; /* vector of index keys */
+    cg_var     *cvi;
+    int         equal = 0;
+    char       *b1;
+    char       *b2;
+    char       *keyname;
+    
+    assert(x1&&x2);
+    y1 = xml_spec(x1);
+    y2 = xml_spec(x2);
+    if (y1==NULL || y2==NULL)
+	return 0; /* just ignore */
+    if (y1 != y2){
+	yi1 = yang_order(y1);
+	yi2 = yang_order(y2);
+	if ((equal = yi1-yi2) != 0)
+	    return equal;
+    }
+    /* Now y1=y2, same Yang spec, can only be list or leaf-list,
+     * sort according to key
+     */
+    if (yang_find((yang_node*)y1, Y_ORDERED_BY, "user") != NULL)
+	return 0; /* Ordered by user: maintain existing order */
+    switch (y1->ys_keyword){
+    case Y_LEAF_LIST: /* Match with name and value */
+	equal = strcmp(xml_body(x1), xml_body(x2));
+	break;
+    case Y_LIST: /* Match with key values 
+		  * Use Y_LIST cache (see struct yang_stmt)
+		  */
+	cvk = y1->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
+	cvi = NULL;
+	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
+	    keyname = cv_string_get(cvi);
+	    b1 = xml_find_body(x1, keyname);
+	    b2 = xml_find_body(x2, keyname);
+	    if ((equal = strcmp(b1,b2)) != 0)
+		goto done;
+	}
+	equal = 0;
+	break;
+    default:
+	break;
+    }
+ done:
+    return equal;
+}
+
+/*!
+ * @param[in] yangi  Yang order
+ * @param[in]  keynr    Length of keyvec/keyval vector when applicable
+ * @param[in]  keyvec   Array of of yang key identifiers
+ * @param[in]  keyval   Array of of yang key values
+ * @param[out] userorder If set, this yang order is user ordered, linear search
+ * @retval  0  If equal (or userorder set)
+ * @retval <0  if arg1 is less than arg2
+ * @retval >0  if arg1 is greater than arg2
+ * @see xml_cmp   Similar, but for two objects
+ */
+static int
+xml_cmp1(cxobj        *x,
+	 yang_stmt    *y,
+	 char         *name,
+	 enum rfc_6020 keyword,   
+	 int           keynr,
+	 char        **keyvec,
+	 char        **keyval,
+	 int          *userorder)
+{
+    char *b;
+    int   i;
+    char *keyname;
+    char *key;
+
+    /* Check if same yang spec (order in yang stmt list) */
+    switch (keyword){
+    case Y_CONTAINER: /* Match with name */
+    case Y_LEAF: /* Match with name */
+	return strcmp(name, xml_name(x));
+	break;
+    case Y_LEAF_LIST: /* Match with name and value */
+	if (userorder && yang_find((yang_node*)y, Y_ORDERED_BY, "user") != NULL)
+	    *userorder=1;
+	b=xml_body(x);
+	return strcmp(keyval[0], b);
+	break;
+    case Y_LIST: /* Match with array of key values */
+	if (userorder && yang_find((yang_node*)y, Y_ORDERED_BY, "user") != NULL)
+	    *userorder=1;
+	for (i=0; i<keynr; i++){
+	    keyname = keyvec[i];
+	    key = keyval[i];
+	    /* Eg return "e0" in <if><name>e0</name></name></if> given "name" */
+	    if ((b = xml_find_body(x, keyname)) == NULL)
+		break; /* error case */
+	    return strcmp(key, b);
+	}
+	return 0;
+	break;
+    default:
+	break;
+    }
+    return 0; /* should not reach here */
+}
+
+/*! Sort children of an XML node
+ * Assume populated by yang spec.
+ * @param[in] x0   XML node
+ * @param[in] arg  Dummy so it can be called by xml_apply()
+ */
+int
+xml_sort(cxobj *x,
+	 void  *arg)
+{
+    qsort(xml_childvec_get(x), xml_child_nr(x), sizeof(cxobj *), xml_cmp);
+    return 0;
+}
+
+/*! Special case search for ordered-by user where linear sort is used
+ */
+static cxobj *
+xml_search_userorder(cxobj        *x0,
+		     yang_stmt    *y,
+		     char         *name,
+		     int           yangi,
+		     int           mid,
+		     enum rfc_6020 keyword,   
+		     int           keynr,
+		     char        **keyvec,
+		     char        **keyval)
+{
+    int    i;
+    cxobj *xc;
+    
+    for (i=mid+1; i<xml_child_nr(x0); i++){ /* First increment */
+	xc = xml_child_i(x0, i);
+	y = xml_spec(xc);
+	if (yangi!=yang_order(y))
+	    break;
+	if (xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, NULL) == 0)
+	    return xc;
+    }
+    for (i=mid-1; i>=0; i--){ /* Then decrement */
+	xc = xml_child_i(x0, i);
+	y = xml_spec(xc);
+	if (yangi!=yang_order(y))
+	    break;
+	if (xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, NULL) == 0)
+	    return xc;
+    }
+    return NULL; /* Not found */
+}
+
+/*!
+ * @param[in] yangi  Yang order
+ * @param[in]  keynr    Length of keyvec/keyval vector when applicable
+ * @param[in]  keyvec   Array of of yang key identifiers
+ * @param[in]  keyval   Array of of yang key values
+ * @param[in] low    Lower bound of childvec search interval 
+ * @param[in] upper  Lower bound of childvec search interval 
+ */
+static cxobj *
+xml_search1(cxobj        *x0,
+	    char         *name,
+	    int           yangi,
+	    enum rfc_6020 keyword,   
+	    int           keynr,
+	    char        **keyvec,
+	    char        **keyval,
+	    int           low, 
+	    int           upper)
+{
+    int        mid;
+    int        cmp;
+    cxobj     *xc;
+    yang_stmt *y;
+    int        userorder= 0;
+    
+    if (upper < low)
+	return NULL; /* not found */
+    mid = (low + upper) / 2;
+    if (mid >= xml_child_nr(x0))  /* beyond range */
+	return NULL;
+    xc = xml_child_i(x0, mid);
+    assert(y = xml_spec(xc));
+    cmp = yangi-yang_order(y);
+    if (cmp == 0){
+	cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, &userorder);
+	if (userorder && cmp)	    /* Look inside this yangi order */
+	    return xml_search_userorder(x0, y, name, yangi, mid, keyword, keynr, keyvec, keyval);
+    }
+    if (cmp == 0)
+	return xc;
+    else if (cmp < 0)
+	return xml_search1(x0, name, yangi, keyword,
+			   keynr, keyvec, keyval, low, mid-1);
+    else 
+	return xml_search1(x0, name, yangi, keyword,
+			   keynr, keyvec, keyval, mid+1, upper);
+    return NULL;
+}
+
+/*! Find XML children using binary search
+ * @param[in] yangi  yang child order
+ * @param[in]  keynr    Length of keyvec/keyval vector when applicable
+ * @param[in]  keyvec   Array of of yang key identifiers
+ * @param[in]  keyval   Array of of yang key values
+ */
+cxobj *
+xml_search(cxobj        *x0,
+	   char         *name,
+	   int           yangi,
+	   enum rfc_6020 keyword,   
+	   int           keynr,
+	   char        **keyvec,
+	   char        **keyval)
+{
+    return xml_search1(x0, name, yangi, keyword, keynr, keyvec, keyval,
+		       0, xml_child_nr(x0));
+}
+
+
+/*! Position where to insert xml object into a list of children nodes
+ * @note EXPERIMENTAL
+ * Insert after position returned
+ * @param[in]  x0       XML parent node.
+ * @param[in]  low       Lower bound
+ * @param[in]  upper     Upper bound (+1)
+ * @retval     position 
+ * XXX: Problem with this is that evrything must be known before insertion
+ */
+int
+xml_insert_pos(cxobj        *x0,
+	       char         *name,
+	       int           yangi,
+	       enum rfc_6020 keyword,   
+	       int           keynr,
+	       char        **keyvec,
+	       char        **keyval,
+	       int           low, 
+	       int           upper)
+{
+    int        mid;
+    cxobj     *xc;
+    yang_stmt *y;
+    int        cmp;
+    int        i;
+    int        userorder= 0;
+    
+    if (upper < low)
+	return low; /* not found */
+    mid = (low + upper) / 2;
+    if (mid >= xml_child_nr(x0))
+	return xml_child_nr(x0); /* upper range */
+    xc = xml_child_i(x0, mid); 
+    y = xml_spec(xc);
+    cmp = yangi-yang_order(y);
+    if (cmp == 0){
+	cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, &userorder);
+	if (userorder){	    /* Look inside this yangi order */
+	    /* Special case: append last of equals if ordered by user */
+	    for (i=mid+1;i<xml_child_nr(x0);i++){
+		xc = xml_child_i(x0, i);
+		if (strcmp(xml_name(xc), name))
+		    break;
+		mid=i; /* still ok */
+	    }
+	    return mid;
+	}
+    }
+    if (cmp == 0)
+	return mid;
+    else if (cmp < 0)
+	return xml_insert_pos(x0, name, yangi, keyword,
+			      keynr, keyvec, keyval, low, mid-1);
+    else
+	return xml_insert_pos(x0, name, yangi, keyword,
+			      keynr, keyvec, keyval, mid+1, upper);
+}
+
+/*! Find matching xml child given name and optional key values
+ * container: x0, y->keyword, name
+ * list:      x0, y->keyword, y->key, name
+ *
+ * The function needs a vector of key values (or single for LEAF_LIST).
+ * What format? 
+ * 1) argc/argv:: "val1","val2" <<==
+ * 2) cv-list?
+ * 3) va-list?
+ *
+ *      yc - LIST (interface) - 
+ *       ^
+ *       |
+ * x0-->x0c-->(name=interface)+->x(name=name)->xb(value="eth0") <==this is
+ *       |
+ *       v
+ *      x1c->name (interface)
+ *      x1c->x(name=name)->xb(value="eth0")
+ *
+ *      CONTAINER:name
+ *      LEAF:     name
+ *      LEAFLIST: name/body... #b0
+ *      LIST:     name/key0/key1... #b2vec+b0 -> x0c
+
+ * <interface><name>eth0</name></interface>
+ * <interface><name>eth1</name></interface>
+ * <interface><name>eth2</name></interface>
+ * @param[in]  x0       XML node. Find child of this node.
+ * @param[in]  keyword  Yang keyword. Relevant: container, list, leaf, leaf_list
+ * @param[in]  keynr    Length of keyvec/keyval vector when applicable
+ * @param[in]  keyvec   Array of of yang key identifiers
+ * @param[in]  keyval   Array of of yang key values
+ * @param[out] xp       Return value on success, pointer to XML child node
+ * @note If keyword is:
+ *   - list, keyvec and keyval should be an array with keynr length
+ *   - leaf_list, keyval should be 1 and keyval should contain one element
+ *   - otherwise, keyval should be 0 and keyval and keyvec should be both NULL.
+ */
+cxobj *
+xml_match(cxobj        *x0,
+	  char         *name,
+	  enum rfc_6020 keyword,   
+	  int           keynr,
+	  char        **keyvec,
+	  char        **keyval)
+{
+    char  *key;
+    char  *keyname;
+    char  *b0;
+    cxobj *x = NULL;
+    int    equal;
+    int    i;
+    
+    x = NULL;
+    switch (keyword){
+    case Y_CONTAINER: /* Match with name */
+    case Y_LEAF: /* Match with name */
+	if (keynr != 0){
+	    clicon_err(OE_XML, EINVAL, "Expected no key argument to CONTAINER or LEAF");
+	    goto ok;
+	}
+	x = xml_find(x0, name);
+	break;
+    case Y_LEAF_LIST: /* Match with name and value */
+	if (keynr != 1)
+	    goto ok;
+	x = xml_find_body_obj(x0, name, keyval[0]);
+	break;
+    case Y_LIST: /* Match with array of key values */
+	i = 0;
+	while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL){
+	    equal = 0;
+	    if (strcmp(xml_name(x), name))
+		continue;
+	    /* Must be inner loop */
+	    for (i=0; i<keynr; i++){
+		keyname = keyvec[i];
+		key = keyval[i];
+		equal = 0;
+		if ((b0 = xml_find_body(x, keyname)) == NULL)
+		    break; /* error case */
+		if (strcmp(b0, key))
+		    break; /* stop as soon as inequal key found */
+		equal=1; /* reaches here for all keynames, x is found. */
+	    }
+	    if (equal) /* x matches, oyherwise look for other */
+		break;
+	} /* while x */
+	break;
+    default: 
+	break;
+    }
+ ok:
+    return x;
+}
+
+/*! Verify all children of XML node are sorted according to xml_sort()
+ * @param[in]   x       XML node. Check its children
+ * @param[in]   arg     Dummy. Ensures xml_apply can be used with this fn
+ @ @retval      0       Sorted
+ @ @retval     -1       Not sorted
+ * @see xml_apply
+ */
+int
+xml_sort_verify(cxobj *x0,
+		void  *arg)
+{
+    int    retval = -1;
+    cxobj *x = NULL;
+    cxobj *xprev = NULL;
+
+    while ((x = xml_child_each(x0, x, -1)) != NULL) {
+	if (xprev != NULL){ /* Check xprev <= x */
+	    if (xml_cmp(&xprev, &x) > 0)
+		goto done;
+	}
+	xprev = x;
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
+/*! Given child tree x1c, find matching child in base tree x0
+ * param[in]  x0   Base tree node
+ * param[in]  x1c  Modification tree child
+ * param[in]  yc   Yang spec of tree child
+ * param[out] x0cp Matching base tree child (if any)
+ * @note XXX: room for optimization? on 1K calls we have 1M body calls and
+	   500K xml_child_each/cvec_each calls. 
+	   The outer loop is large for large lists
+	   The inner loop is small
+	   Major time in xml_find_body()
+	   Can one do a binary search in the x0 list?
+*/
+int
+match_base_child(cxobj     *x0, 
+		 cxobj     *x1c,
+		 cxobj    **x0cp,
+		 yang_stmt *yc)
+{
+    int        retval = -1;
+    cvec      *cvk = NULL; /* vector of index keys */
+    cg_var    *cvi;
+    char      *b;
+    char      *keyname;
+    char       keynr = 0;
+    char     **keyval = NULL;
+    char     **keyvec = NULL;
+    int        i;
+
+    *x0cp = NULL; /* return value */
+    switch (yc->ys_keyword){
+    case Y_CONTAINER: 	/* Equal regardless */
+    case Y_LEAF: 	/* Equal regardless */
+	break;
+    case Y_LEAF_LIST: /* Match with name and value */
+	keynr = 1;
+	if ((keyval = calloc(keynr+1, sizeof(char*))) == NULL){
+	    clicon_err(OE_UNIX, errno, "calloc");
+	    goto done;
+	}
+	if ((keyval[0] = xml_body(x1c)) == NULL)
+	    goto ok;
+	break;
+    case Y_LIST: /* Match with key values */
+	cvk = yc->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
+	/* Count number of key indexes */
+	cvi = NULL; keynr = 0;
+	while ((cvi = cvec_each(cvk, cvi)) != NULL) 
+	    keynr++;
+	if ((keyval = calloc(keynr+1, sizeof(char*))) == NULL){
+	    clicon_err(OE_UNIX, errno, "calloc");
+	    goto done;
+	}
+	if ((keyvec = calloc(keynr+1, sizeof(char*))) == NULL){
+	    clicon_err(OE_UNIX, errno, "calloc");
+	    goto done;
+	}
+	cvi = NULL; i = 0;
+	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
+	    keyname = cv_string_get(cvi);
+	    keyvec[i] = keyname;
+	    if ((b = xml_find_body(x1c, keyname)) == NULL)
+		goto ok; /* not found */
+	    keyval[i++] = b;
+	}
+	break;
+    default:
+	break;
+    }
+    /* Get match */
+    {
+	yang_node *y0;
+	int        yorder;
+	    
+	if ((y0 = yc->ys_parent) == NULL)
+	    goto done;
+	/* XXX: No we cant do this. on uppermost layer it can look like this:
+	 * config 
+	 * ximport-----ymod = ietf 
+	 * interfaces--ymod = example
+	 * Which means yang order can be different for different children in the
+	 * same childvec.
+	 */
+	if (xml_child_sort==0)
+	    *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
+	else{
+#if 1
+	    if (xml_child_nr(x0)==0 || xml_spec(xml_child_i(x0,0))!=NULL){
+		yorder = yang_order(yc);
+		*x0cp = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval);
+	    }
+	    else{
+		clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__);
+		*x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
+	    }
+#else
+	    cxobj *xx;
+
+
+
+	    *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
+	    if (xml_child_nr(x0) && xml_spec(xml_child_i(x0,0))!=NULL){
+		xx = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval);
+		if (xx!=*x0cp){
+		    clicon_log(LOG_WARNING, "%s mismatch", __FUNCTION__);
+		    fprintf(stderr, "mismatch\n");
+		    xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval);
+		    assert(0);
+		}
+	    }
+	    else
+		clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__);
+#endif
+	}
+    }
+ ok:
+    retval = 0;
+ done:
+    if (keyval)
+	free(keyval);
+    if (keyvec)
+	free(keyvec);
+    return retval;
+}
+	   

lib/src/clixon_xsl.c

@@ -102,6 +102,10 @@ in
 #include "clixon_err.h"
 #include "clixon_log.h"
 #include "clixon_string.h"
+#include "clixon_queue.h"
+#include "clixon_hash.h"
+#include "clixon_handle.h"
+#include "clixon_yang.h"
 #include "clixon_xml.h"
 #include "clixon_xsl.h"
 
@@ -1026,7 +1030,7 @@ xpath_each(cxobj *xcur,
  * @retval     -1      error.
  *
  * @code
- *   cxobj **xvec;
+ *   cxobj **xvec = NULL;
  *   size_t  xlen;
  *   if (xpath_vec(xcur, "//symbol/foo", &xvec, &xlen) < 0) 
  *      goto err;
@@ -1170,7 +1174,7 @@ main(int argc, char **argv)
 	usage(argv[0]);
 	return 0;
     }
-    if (clicon_xml_parse_file(0, &x, "</clicon>") < 0){
+    if (xml_parse_file(0, &x, "</clicon>") < 0){
 	fprintf(stderr, "parsing 2\n");
 	return -1;
     }

lib/src/clixon_yang.c

@@ -373,7 +373,7 @@ yn_each(yang_node *yn,
  *
  * @param[in]  yn         Yang node, current context node.
  * @param[in]  keyword    if 0 match any keyword
- * @param[in]  argument   String compare w wrgument. if NULL, match any.
+ * @param[in]  argument   String compare w argument. if NULL, match any.
  * This however means that if you actually want to match only a yang-stmt with 
  * argument==NULL you cannot, but I have not seen any such examples.
  * @see yang_find_datanode
@@ -384,8 +384,8 @@ yang_find(yang_node *yn,
 	  char      *argument)
 {
     yang_stmt *ys = NULL;
-    int i;
+    int        i;
-    int match = 0;
+    int        match = 0;
 
     for (i=0; i<yn->yn_len; i++){
 	ys = yn->yn_stmt[i];
@@ -558,6 +558,54 @@ yang_find_myprefix(yang_stmt *ys)
     return prefix;
 }
 
+/*! Find matching y in yp:s children, return 0 and index or -1 if not found.
+ * @retval 0 not found
+ * @retval 1 found
+ */
+static int
+order1(yang_node *yp,
+       yang_stmt *y,
+       int       *index)
+{
+    yang_stmt  *ys;
+    int         i;
+    
+    for (i=0; i<yp->yn_len; i++){
+	ys = yp->yn_stmt[i];
+	if (!yang_datanode(ys))
+	    continue;
+	if (ys==y)
+	    return 1;
+	(*index)++;
+    }
+    return 0;
+}
+
+/*! Return order of yang statement y in parents child vector
+ * @retval  i  Order of child with specified argument
+ * @retval -1  Not found
+ */
+int
+yang_order(yang_stmt *y)
+{
+    yang_node  *yp;
+    yang_node  *ypp;
+    yang_node  *yn;
+    int         i;
+    int         j=0;
+    
+    yp = y->ys_parent;
+    if (yp->yn_keyword == Y_MODULE ||yp->yn_keyword == Y_SUBMODULE){
+	ypp = yp->yn_parent;
+	for (i=0; i<ypp->yn_len; i++){
+	    yn = (yang_node*)ypp->yn_stmt[i];
+	    if (order1(yn, y, &j) == 1)
+		return j;
+	}
+    }
+    order1(yp, y, &j);
+    return j;
+}
 
 /*! Reset flag in complete tree, arg contains flag */
 static int
@@ -880,6 +928,20 @@ ys_populate_leaf(yang_stmt *ys,
     return retval;
 }
 
+static int
+ys_populate_list(yang_stmt *ys, 
+		 void      *arg)
+{
+    yang_stmt  *ykey;
+    
+    if ((ykey = yang_find((yang_node*)ys, Y_KEY, NULL)) == NULL)
+	return 0;
+    cvec_free(ys->ys_cvec);
+    if ((ys->ys_cvec = yang_arg2cvec(ykey, " ")) == NULL)
+	return -1;
+    return 0;
+}
+
 /*! Populate range and length statements
  *
  * Create cvec variables "range_min" and "range_max". Assume parent is type.
@@ -1061,6 +1123,10 @@ ys_populate(yang_stmt *ys,
 	if (ys_populate_leaf(ys, arg) < 0)
 	    goto done;
 	break;
+    case Y_LIST:
+	if (ys_populate_list(ys, arg) < 0)
+	    goto done;
+	break;
     case Y_RANGE: 
     case Y_LENGTH: 
 	if (ys_populate_range(ys, arg) < 0)
@@ -1527,7 +1593,7 @@ yang_parse_recurse(clicon_handle h,
 	    if ((nr = yang_parse_find_match(h, yang_dir, module, fbuf)) < 0)
 		goto done;
 	    if (nr == 0){
-		clicon_err(OE_YANG, errno, "No matching %s yang files found (expected modulenameor absolute filename)", module);
+		clicon_err(OE_YANG, errno, "No matching %s yang files found (expected module name or absolute filename)", module);
 		goto done;
 	    }
 	}
@@ -1820,7 +1886,6 @@ yang_abs_schema_nodeid(yang_spec  *yspec,
     yang_stmt       *ymod;
     char            *id;
     char            *prefix = NULL;
-
     yang_stmt       *yprefix;
 
     /* check absolute schema_nodeid */
@@ -2024,24 +2089,36 @@ yang_config(yang_stmt *ys)
     return 1;
 }
 
-/*! Utility function for handling yang parsing and translation to key format
+/*! Parse netconf yang spec, used by netconf client and as internal protocol */
+yang_spec *
+yang_spec_netconf(clicon_handle h)
+{
+    yang_spec     *yspec = NULL;
+
+    if ((yspec = yspec_new()) == NULL)
+	goto done;
+    if (yang_parse(h, CLIXON_DATADIR, "ietf-netconf", NULL, yspec) < 0){
+	yspec_free(yspec); yspec = NULL;
+	goto done;
+    }
+    clicon_netconf_yang_set(h, yspec);	
+ done:
+    return yspec;
+}
+
+/*! Read, parse and save application yang specification as option
  * @param h          clicon handle
  * @param f          file to print to (if printspec enabled)
  * @param printspec  print database (YANG) specification as read from file
  */
-int
+yang_spec*
-yang_spec_main(clicon_handle h, 
+yang_spec_main(clicon_handle h)
-	       FILE         *f, 
-	       int           printspec)
 {
-    yang_spec      *yspec;
+    yang_spec      *yspec = NULL;
     char           *yang_dir;
     char           *yang_module;
     char           *yang_revision;
-    int             retval = -1;
 
-    if ((yspec = yspec_new()) == NULL)
-	goto done;
     if ((yang_dir    = clicon_yang_dir(h)) == NULL){
 	clicon_err(OE_FATAL, 0, "CLICON_YANG_DIR option not set");
 	goto done;
@@ -2052,14 +2129,15 @@ yang_spec_main(clicon_handle h,
 	goto done;
     }
     yang_revision = clicon_yang_module_revision(h);
-    if (yang_parse(h, yang_dir, yang_module, yang_revision, yspec) < 0)
+    if ((yspec = yspec_new()) == NULL)
 	goto done;
+    if (yang_parse(h, yang_dir, yang_module, yang_revision, yspec) < 0){
+	yspec_free(yspec); yspec = NULL;
+	goto done;
+    }
     clicon_dbspec_yang_set(h, yspec);	
-    if (printspec)
-	yang_print(f, (yang_node*)yspec);
-    retval = 0;
   done:
-    return retval;
+    return yspec;
 }
 
 /*! Given a yang node, translate the argument string to a cv vector
@@ -2077,7 +2155,7 @@ yang_spec_main(clicon_handle h,
  *         ...cv_string_get(cv);
  *    cvec_free(cvv);
  * @endcode
- * Note: must free return value after use w cvec_free
+ * @note must free return value after use w cvec_free
  */
 cvec *
 yang_arg2cvec(yang_stmt *ys, 

lib/src/clixon_yang_parse.y

@@ -184,7 +184,8 @@ clicon_yang_debug(int d)
    also called from yacc generated code *
 */
 void 
-clixon_yang_parseerror(void *_yy, char *s) 
+clixon_yang_parseerror(void *_yy,
+		       char *s) 
 { 
     clicon_err(OE_YANG, 0, "%s on line %d: %s at or before: '%s'", 
 	       _YY->yy_name,
@@ -195,7 +196,8 @@ clixon_yang_parseerror(void *_yy, char *s)
 }
 
 int
-yang_parse_init(struct clicon_yang_yacc_arg *yy, yang_spec *ysp)
+yang_parse_init(struct clicon_yang_yacc_arg *yy,
+		yang_spec                   *ysp)
 {
     return 0;
 }
@@ -219,7 +221,8 @@ ystack_pop(struct clicon_yang_yacc_arg *yy)
 }
 
 struct ys_stack *
-ystack_push(struct clicon_yang_yacc_arg *yy, yang_node *yn)
+ystack_push(struct clicon_yang_yacc_arg *yy,
+	    yang_node                   *yn)
 {
     struct ys_stack *ystack; 
 
@@ -240,8 +243,8 @@ ystack_push(struct clicon_yang_yacc_arg *yy, yang_node *yn)
  */
 static yang_stmt *
 ysp_add(struct clicon_yang_yacc_arg *yy, 
-	enum rfc_6020 keyword, 
+	enum rfc_6020                keyword, 
-	char *argument)
+	char                        *argument)
 {
     struct ys_stack *ystack = yy->yy_stack;
     yang_stmt       *ys = NULL;
@@ -272,7 +275,9 @@ ysp_add(struct clicon_yang_yacc_arg *yy,
 
 /*! combination of ysp_add and ysp_push for sub-modules */
 static yang_stmt *
-ysp_add_push(struct clicon_yang_yacc_arg *yy, int keyword, char *argument)
+ysp_add_push(struct clicon_yang_yacc_arg *yy,
+	     int                          keyword,
+	     char                        *argument)
 {
     yang_stmt *ys;
 
@@ -286,7 +291,8 @@ ysp_add_push(struct clicon_yang_yacc_arg *yy, int keyword, char *argument)
 /* identifier-ref-arg-str has a [prefix :] id and prefix_id joins the id with an 
    optional prefix into a single string */
 static char*
-prefix_id_join(char *prefix, char *id)
+prefix_id_join(char *prefix,
+	       char *id)
 {
     char *str;
     int   len;
@@ -333,7 +339,7 @@ module_stmt   : K_MODULE id_arg_str
                         } 
                 '{' module_substmts '}' 
                         { if (ystack_pop(_yy) < 0) _YYERROR("2");
-			  clicon_debug(2,"module -> id-arg-str { module-substmts }");} 
+			  clicon_debug(2,"module_stmt -> id-arg-str { module-substmts }");} 
               ;
 
 module_substmts : module_substmts module_substmt 
@@ -693,7 +699,7 @@ anyxml_substmt  : when_stmt          { clicon_debug(2,"anyxml-substmt -> when-st
               | status_stmt          { clicon_debug(2,"anyxml-substmt -> status-stmt"); }
               | description_stmt     { clicon_debug(2,"anyxml-substmt -> description-stmt"); }
               | reference_stmt       { clicon_debug(2,"anyxml-substmt -> reference-stmt"); }
-              | ustring ':' ustring ';' { clicon_debug(2,"anyxml-substmt -> anyxml extension"); }
+              | ustring ':' ustring ';' { free($1); free($3); clicon_debug(2,"anyxml-substmt -> anyxml extension"); }
               ;
 
 /* uses */

lib/src/json_xpath.c

@@ -98,7 +98,7 @@ main(int argc, char **argv)
 	return -1;
     }
     else
-      if (clicon_xml_parse_str(buf, &x) < 0)
+      if (xml_parse_string(buf, &x) < 0)
 	return -1;
 
     if (xpath_vec(x, xpath, &xv, &xlen) < 0)

test/lib.sh

@@ -2,7 +2,6 @@
 
 testnr=0
 testname=
-clixon_cf=/usr/local/etc/routing.xml
 # error and exit, arg is optional extra errmsg
 err(){
   echo "Error in Test$testnr [$testname]:"

test/test_cli.sh

@@ -9,6 +9,7 @@
 
 # include err() and new() functions
 . ./lib.sh
+cfg=/usr/local/etc/routing.xml
 
 # For memcheck
 #clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"
@@ -16,83 +17,80 @@ clixon_cli=clixon_cli
 
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -z -f $cfg
 if [ $? -ne 0 ]; then
     err
 fi
-new "start backend"
+new "start backend -s init -f $cfg"
-# start new backend
+sudo clixon_backend -s init -f $cfg 
-sudo clixon_backend -s init -f $clixon_cf 
 if [ $? -ne 0 ]; then
     err
 fi
 new "cli tests"
 
 new "cli configure top"
-expectfn "$clixon_cli -1f $clixon_cf set interfaces" "^$"
+expectfn "$clixon_cli -1 -f $cfg set interfaces" "^$"
 
 new "cli show configuration top (no presence)"
-expectfn "$clixon_cli -1f $clixon_cf show conf cli" "^$"
+expectfn "$clixon_cli -1 -f $cfg show conf cli" "^$"
 
 new "cli configure delete top"
-expectfn "$clixon_cli -1f $clixon_cf delete interfaces" "^$"
+expectfn "$clixon_cli -1 -f $cfg delete interfaces" "^$"
 
 new "cli show configuration delete top"
-expectfn "$clixon_cli -1f $clixon_cf show conf cli" "^$"
+expectfn "$clixon_cli -1 -f $cfg show conf cli" "^$"
 
 new "cli configure"
-expectfn "$clixon_cli -1f $clixon_cf set interfaces interface eth/0/0" "^$"
+expectfn "$clixon_cli -1 -f $cfg set interfaces interface eth/0/0" "^$"
 
 new "cli show configuration"
-expectfn "$clixon_cli -1f $clixon_cf show conf cli" "^interfaces interface name eth/0/0" "interfaces interface enabled true$"
+expectfn "$clixon_cli -1 -f $cfg show conf cli" "^interfaces interface name eth/0/0" "interfaces interface enabled true$"
 
 new "cli failed validate"
-expectfn "$clixon_cli -1f $clixon_cf -l o validate" "Missing mandatory variable"
+expectfn "$clixon_cli -1 -f $cfg -l o validate" "Missing mandatory variable"
 
 new "cli configure more"
-expectfn "$clixon_cli -1f $clixon_cf set interfaces interface eth/0/0 ipv4 address 1.2.3.4 prefix-length 24" "^$"
+expectfn "$clixon_cli -1 -f $cfg set interfaces interface eth/0/0 ipv4 address 1.2.3.4 prefix-length 24" "^$"
-expectfn "$clixon_cli -1f $clixon_cf set interfaces interface eth/0/0 description mydesc" "^$"
+expectfn "$clixon_cli -1 -f $cfg set interfaces interface eth/0/0 description mydesc" "^$"
-expectfn "$clixon_cli -1f $clixon_cf set interfaces interface eth/0/0 type bgp" "^$"
+expectfn "$clixon_cli -1 -f $cfg set interfaces interface eth/0/0 type bgp" "^$"
 
 new "cli show xpath description"
-expectfn "$clixon_cli -1f $clixon_cf -l o show xpath /interfaces/interface/description" "<description>mydesc</description>"
+expectfn "$clixon_cli -1 -f $cfg -l o show xpath /interfaces/interface/description" "<description>mydesc</description>"
 
 new "cli delete description"
-expectfn "$clixon_cli -1f $clixon_cf -l o delete interfaces interface eth/0/0 description mydesc"
+expectfn "$clixon_cli -1 -f $cfg -l o delete interfaces interface eth/0/0 description mydesc"
 
 new "cli show xpath no description"
-expectfn "$clixon_cli -1f $clixon_cf -l o show xpath /interfaces/interface/description" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o show xpath /interfaces/interface/description" "^$"
 
 new "cli copy interface"
-expectfn "$clixon_cli -1f $clixon_cf copy interface eth/0/0 to eth99" "^$"
+expectfn "$clixon_cli -1 -f $cfg copy interface eth/0/0 to eth99" "^$"
 
 new "cli success validate"
-expectfn "$clixon_cli -1f $clixon_cf -l o validate" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o validate" "^$"
 
 new "cli commit"
-expectfn "$clixon_cli -1f $clixon_cf -l o commit" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o commit" "^$"
 
 new "cli save"
-expectfn "$clixon_cli -1f $clixon_cf -l o save /tmp/foo" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o save /tmp/foo" "^$"
-
-
 
 new "cli delete all"
-expectfn "$clixon_cli -1f $clixon_cf -l o delete all" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o delete all" "^$"
 
 new "cli load"
-expectfn "$clixon_cli -1f $clixon_cf -l o load /tmp/foo" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o load /tmp/foo" "^$"
 
 new "cli check load"
-expectfn "$clixon_cli -1f $clixon_cf -l o show conf cli" "^interfaces interface name eth/0/0" "interfaces interface enabled true$"
+expectfn "$clixon_cli -1 -f $cfg -l o show conf cli" "^interfaces interface name eth/0/0" "interfaces interface enabled true$"
 
 new "cli debug"
-expectfn "$clixon_cli -1f $clixon_cf -l o debug level 1" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o debug level 1" "^$"
 # How to test this?
-expectfn "$clixon_cli -1f $clixon_cf -l o debug level 0" "^$"
+expectfn "$clixon_cli -1 -f $cfg -l o debug level 0" "^$"
 
 new "cli rpc"
-expectfn "$clixon_cli -1f $clixon_cf -l o rpc ipv4" "^<rpc-reply>"
+expectfn "$clixon_cli -1 -f $cfg -l o rpc ipv4" "^<rpc-reply>"
 
 new "Kill backend"
 # Check if still alive
@@ -101,7 +99,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -z -f $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_leafref.sh

@@ -3,13 +3,15 @@
 
 # include err() and new() functions
 . ./lib.sh
+cfg=/usr/local/etc/routing.xml
+fyang=/tmp/leafref.yang
 
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
 clixon_netconf=clixon_netconf
 clixon_cli=clixon_cli
 
-cat <<EOF > /tmp/leafref.yang
+cat <<EOF > $fyang
 module example{
     import ietf-ip {
       prefix ip;
@@ -48,70 +50,70 @@ EOF
 
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf -y /tmp/leafref.yang
+sudo clixon_backend -zf $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "start backend"
 # start new backend
-sudo clixon_backend -s init -f $clixon_cf -y /tmp/leafref.yang
+sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "leafref base config"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><edit-config><target><candidate/></target><config><interfaces>
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><interfaces>
 <interface><name>eth0</name>  <type>eth</type>  <ipv4><address><ip>192.0.2.1</ip></address><address><ip>192.0.2.2</ip></address></ipv4></interface>
 <interface><name>lo</name><type>lo</type><ipv4><address><ip>127.0.0.1</ip></address></ipv4></interface>
 </interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "leafref get config"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply><data><interfaces><interface><name>eth0</name>'
+expecteof "$clixon_netconf -qf $cfg" '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply><data><interfaces><interface><name>eth0</name>'
 
 new "leafref base commit"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "leafref add wrong ref"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><edit-config><target><candidate/></target><config><default-address><absname>eth3</absname><address>10.0.4.6</address></default-address></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><default-address><absname>eth3</absname><address>10.0.4.6</address></default-address></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "leafref validate"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-tag>missing-attribute</error-tag>"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-tag>missing-attribute</error-tag>"
 
 new "leafref discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "leafref add correct absref"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><edit-config><target><candidate/></target><config><default-address><absname>eth0</absname></default-address></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><default-address><absname>eth0</absname></default-address></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "leafref add correct relref"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><edit-config><target><candidate/></target><config><default-address><relname>eth0</relname></default-address></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><default-address><relname>eth0</relname></default-address></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 # XXX add address
 
 new "leafref validate (ok)"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>"
 
 new "leafref delete leaf"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation=\"delete\"><name>eth0</name></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation=\"delete\"><name>eth0</name></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>"
 
 new "leafref validate (should fail)"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/leafref.yang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>"  "^<rpc-reply><rpc-error><error-tag>missing-attribute</error-tag>"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>"  "^<rpc-reply><rpc-error><error-tag>missing-attribute</error-tag>"
 
 new "leafref discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
-
+exit
 new "cli leafref lo"
-expectfn "$clixon_cli -1f $clixon_cf -y /tmp/leafref.yang -l o set default-address absname lo" "^$"
+expectfn "$clixon_cli -1f $cfg -y $fyang -l o set default-address absname lo" "^$"
 
 new "cli leafref validate"
-expectfn "$clixon_cli -1f $clixon_cf -y /tmp/leafref.yang -l o validate" "^$"
+expectfn "$clixon_cli -1f $cfg -y $fyang -l o validate" "^$"
 
 new "cli sender"
-expectfn "$clixon_cli -1f $clixon_cf -y /tmp/leafref.yang -l o set sender a" "^$"
+expectfn "$clixon_cli -1f $cfg -y $fyang -l o set sender a" "^$"
 
 new "cli sender template"
-expectfn "$clixon_cli -1f $clixon_cf -y /tmp/leafref.yang -l o set sender b template a" "^$"
+expectfn "$clixon_cli -1f $cfg -y $fyang -l o set sender b template a" "^$"
 
 new "Kill backend"
 # Check if still alive
@@ -120,7 +122,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_netconf.sh

@@ -3,136 +3,137 @@
 
 # include err() and new() functions
 . ./lib.sh
+cfg=/usr/local/etc/routing.xml
 
 # For memcheck
-# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+#clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
 clixon_netconf=clixon_netconf
 
+echo "clixon_backend -zf $cfg"
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err
 fi
 new "start backend"
 # start new backend
-sudo clixon_backend -s init -f $clixon_cf
+sudo clixon_backend -s init -f $cfg
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "netconf tests"
 
-new "netconf get empty config"
+expecteof "$clixon_netconf -qf $cfg" '<rpc message-id="101"><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply message-id="101"><data/></rpc-reply>]]>]]>$'
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc message-id="101"><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply message-id="101"><data/></rpc-reply>]]>]]>$'
 
 new "Add subtree eth/0/0 using none which should not change anything"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><edit-config><default-operation>none</default-operation><target><candidate/></target><config><interfaces><interface><name>eth/0/0</name></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><edit-config><default-operation>none</default-operation><target><candidate/></target><config><interfaces><interface><name>eth/0/0</name></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "Check nothing added"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc message-id="101"><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply message-id="101"><data/></rpc-reply>]]>]]>$'
+expecteof "$clixon_netconf -qf $cfg" '<rpc message-id="101"><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply message-id="101"><data/></rpc-reply>]]>]]>$'
 
 new "Add subtree eth/0/0 using none and create which should add eth/0/0"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="create"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="create"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>' "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "Check eth/0/0 added using xpath"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/interfaces/interface[name=eth/0/0]"/></get-config></rpc>]]>]]>' "^<rpc-reply><data><interfaces><interface><name>eth/0/0</name><type>eth</type><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/interfaces/interface[name=eth/0/0]"/></get-config></rpc>]]>]]>' "^<rpc-reply><data><interfaces><interface><name>eth/0/0</name><type>eth</type><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
 
 new "Re-create same eth/0/0 which should generate error"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="create"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>' "^<rpc-reply><rpc-error>"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="create"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>' "^<rpc-reply><rpc-error>"
 
 new "Delete eth/0/0 using none config"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="delete"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>'  "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="delete"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>'  "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "Check deleted eth/0/0 (non-presence container)"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc message-id="101"><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply message-id="101"><data/></rpc-reply>]]>]]>$'
+expecteof "$clixon_netconf -qf $cfg" '<rpc message-id="101"><get-config><source><candidate/></source></get-config></rpc>]]>]]>' '^<rpc-reply message-id="101"><data/></rpc-reply>]]>]]>$'
 
 new "Re-Delete eth/0/0 using none should generate error"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="delete"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>'  "^<rpc-reply><rpc-error>"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><edit-config><target><candidate/></target><config><interfaces><interface operation="delete"><name>eth/0/0</name><type>eth</type></interface></interfaces></config><default-operation>none</default-operation> </edit-config></rpc>]]>]]>'  "^<rpc-reply><rpc-error>"
 
 new "netconf edit config"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface><name>eth/0/0</name></interface><interface><name>eth1</name><enabled>true</enabled><ipv4><address><ip>9.2.3.4</ip><prefix-length>24</prefix-length></address></ipv4></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface><name>eth/0/0</name></interface><interface><name>eth1</name><enabled>true</enabled><ipv4><address><ip>9.2.3.4</ip><prefix-length>24</prefix-length></address></ipv4></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf get config xpath"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/interfaces/interface[name=eth1]/enabled"/></get-config></rpc>]]>]]>' "^<rpc-reply><data><interfaces><interface><name>eth1</name><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/interfaces/interface[name=eth1]/enabled"/></get-config></rpc>]]>]]>' "^<rpc-reply><data><interfaces><interface><name>eth1</name><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
 
 new "netconf get config xpath parent"
-expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/interfaces/interface[name=eth1]/enabled/../.."/></get-config></rpc>]]>]]>' "^<rpc-reply><data><interfaces><interface><name>eth/0/0</name><enabled>true</enabled></interface><interface><name>eth1</name><enabled>true</enabled><ipv4><enabled>true</enabled><forwarding>false</forwarding><address><ip>9.2.3.4</ip><prefix-length>24</prefix-length></address></ipv4></interface></interfaces></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/interfaces/interface[name=eth1]/enabled/../.."/></get-config></rpc>]]>]]>' "^<rpc-reply><data><interfaces><interface><name>eth/0/0</name><enabled>true</enabled></interface><interface><name>eth1</name><enabled>true</enabled><ipv4><enabled>true</enabled><forwarding>false</forwarding><address><ip>9.2.3.4</ip><prefix-length>24</prefix-length></address></ipv4></interface></interfaces></data></rpc-reply>]]>]]>$"
 
 new "netconf validate missing type"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><rpc-error>"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><rpc-error>"
 
 new "netconf discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf get empty config2"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data/></rpc-reply>]]>]]>$"
 
 new "netconf edit extra xml"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><edit-config><target><candidate/></target><config><interfaces><extra/></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><rpc-error>"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><edit-config><target><candidate/></target><config><interfaces><extra/></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><rpc-error>"
 
 new "netconf discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf edit config eth1"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface><name>eth1</name><type>eth</type></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface><name>eth1</name><type>eth</type></interface></interfaces></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf validate"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf commit"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf edit config replace XXX is merge?"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface><name>eth2</name><type>eth</type></interface></interfaces></config><default-operation>merge</default-operation></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><edit-config><target><candidate/></target><config><interfaces><interface><name>eth2</name><type>eth</type></interface></interfaces></config><default-operation>merge</default-operation></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf get replaced config"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><interfaces><interface><name>eth1</name><type>eth</type><enabled>true</enabled></interface><interface><name>eth2</name><type>eth</type><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><interfaces><interface><name>eth1</name><type>eth</type><enabled>true</enabled></interface><interface><name>eth2</name><type>eth</type><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
 
 new "netconf discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf edit state operation should fail"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><edit-config><target><candidate/></target><config><interfaces-state><interface><name>eth1</name><type>eth</type></interface></interfaces-state></config></edit-config></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-tag>invalid-value</error-tag>"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><edit-config><target><candidate/></target><config><interfaces-state><interface><name>eth1</name><type>eth</type></interface></interfaces-state></config></edit-config></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-tag>invalid-value</error-tag>"
 
 new "netconf get state operation"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><get><filter type=\"xpath\" select=\"/interfaces-state\"/></get></rpc>]]>]]>" "^<rpc-reply><data><interfaces-state><interface><name>eth0</name><type>eth</type><if-index>42</if-index></interface></interfaces-state></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><get><filter type=\"xpath\" select=\"/interfaces-state\"/></get></rpc>]]>]]>" "^<rpc-reply><data><interfaces-state><interface><name>eth0</name><type>eth</type><if-index>42</if-index></interface></interfaces-state></data></rpc-reply>]]>]]>$"
 
 new "netconf lock/unlock"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><lock><target><candidate/></target></lock></rpc>]]>]]><rpc><unlock><target><candidate/></target></unlock></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]><rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><lock><target><candidate/></target></lock></rpc>]]>]]><rpc><unlock><target><candidate/></target></unlock></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]><rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf lock/lock"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><lock><target><candidate/></target></lock></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><lock><target><candidate/></target></lock></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf lock" 
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><lock><target><candidate/></target></lock></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><lock><target><candidate/></target></lock></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "close-session"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><close-session/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><close-session/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "kill-session"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><kill-session><session-id>44</session-id></kill-session></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><kill-session><session-id>44</session-id></kill-session></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "copy startup"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><copy-config><target><startup/></target><source><candidate/></source></copy-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><copy-config><target><startup/></target><source><candidate/></source></copy-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf get startup"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><interfaces><interface><name>eth1</name><type>eth</type><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><interfaces><interface><name>eth1</name><type>eth</type><enabled>true</enabled></interface></interfaces></data></rpc-reply>]]>]]>$"
 
 new "netconf delete startup"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><delete-config><target><startup/></target></delete-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><delete-config><target><startup/></target></delete-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf check empty startup"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data/></rpc-reply>]]>]]>$"
 
 new "netconf rpc"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><rt:fib-route><routing-instance-name>ipv4</routing-instance-name><destination-address><address-family>ipv4</address-family></destination-address></rt:fib-route></rpc>]]>]]>" "^<rpc-reply><route><address-family>ipv4</address-family><next-hop><next-hop-list>"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><rt:fib-route><routing-instance-name>ipv4</routing-instance-name><destination-address><address-family>ipv4</address-family></destination-address></rt:fib-route></rpc>]]>]]>" "^<rpc-reply><route><address-family>ipv4</address-family><next-hop><next-hop-list>"
 
 new "netconf rpc w/o namespace"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><fib-route><routing-instance-name>ipv4</routing-instance-name><destination-address><address-family>ipv4</address-family></destination-address></fib-route></rpc>]]>]]>" "^<rpc-reply><route><address-family>ipv4</address-family><next-hop><next-hop-list>"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><fib-route><routing-instance-name>ipv4</routing-instance-name><destination-address><address-family>ipv4</address-family></destination-address></fib-route></rpc>]]>]]>" "^<rpc-reply><route><address-family>ipv4</address-family><next-hop><next-hop-list>"
 
 new "netconf subscription"
-expectwait "$clixon_netconf -qf $clixon_cf" "<rpc><create-subscription><stream>ROUTING</stream></create-subscription></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]><notification><event>Routing notification</event></notification>]]>]]>$" 30
+expectwait "$clixon_netconf -qf $cfg" "<rpc><create-subscription><stream>ROUTING</stream></create-subscription></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]><notification><event>Routing notification</event></notification>]]>]]>$" 30
 
 new "Kill backend"
 # Check if still alive
@@ -141,7 +142,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_order.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+# Order test. test ordered-by user and ordered-by system.
+# For each leaf and leaf-lists, there are two lists,
+# one ordered-by user and one ordered by system.
+# The ordered-by user MUST be the order it is entered.
+# No test of ordered-by system is done yet
+# (we may want to sort them alphabetically for better performance).
+
+# include err() and new() functions
+. ./lib.sh
+cfg=/tmp/conf_yang.xml
+fyang=/tmp/order.yang
+
+# For memcheck
+# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+clixon_netconf=clixon_netconf
+clixon_cli=clixon_cli
+dbdir=/tmp/order
+
+new "Set up $dbdir"
+rm -rf $dbdir
+mkdir $dbdir
+
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>/tmp/conf_yang.xml</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>$dbdir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+  <CLICON_XML_SORT>true</CLICON_XML_SORT>
+</config>
+EOF
+
+cat <<EOF > $fyang
+module example{
+    container c{
+      leaf d{
+         type string;
+      }
+    }
+    leaf l{
+       type string;
+    }
+    leaf-list y0 {
+      ordered-by user;
+      type string;
+    }
+    leaf-list y1 {
+      ordered-by system;
+      type string;
+    }
+    list y2 {
+      ordered-by user;
+      key "k";
+      leaf k {
+        type string;
+      }
+      leaf a {
+        type string;
+      }   
+    }
+    list y3 {
+      ordered-by system;
+      key "k";
+      leaf k {
+        type string;
+      }
+      leaf a {
+        type string;
+      }   
+    }
+}
+EOF
+
+cat <<EOF > $dbdir/running_db
+<config>
+  <y0>d</y0>
+  <y1>d</y1>
+  <y2><k>d</k><a>bar</a></y2>
+  <y3><k>d</k><a>bar</a></y3>
+  <y0>b</y0>
+  <y1>b</y1>
+  <c><d>hej</d></c>
+  <y0>c</y0>
+  <y1>c</y1>
+  <y2><k>a</k><a>bar</a></y2>
+  <y3><k>a</k><a>bar</a></y3>
+  <l>hopp</l>
+  <y0>a</y0>
+  <y1>a</y1>
+  <y2><k>c</k><a>bar</a></y2>
+  <y3><k>c</k><a>bar</a></y3>
+  <y2><k>b</k><a>bar</a></y2>
+  <y3><k>b</k><a>bar</a></y3>
+</config>
+EOF
+
+# kill old backend (if any)
+new "kill old backend"
+sudo clixon_backend -zf $cfg -y $fyang
+if [ $? -ne 0 ]; then
+    err
+fi
+
+new "start backend"
+# start new backend
+sudo clixon_backend -s running -f $cfg -y $fyang
+if [ $? -ne 0 ]; then
+    err
+fi
+
+# Check as file
+new "verify running from start, should be: l,c,y0,y1,y2,y3; y1 and y3 sorted. Note this fails if XML_SORT set to false"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><c><d>hej</d></c><l>hopp</l><y0>d</y0><y0>b</y0><y0>c</y0><y0>a</y0><y1>a</y1><y1>b</y1><y1>c</y1><y1>d</y1><y2><k>d</k><a>bar</a></y2><y2><k>a</k><a>bar</a></y2><y2><k>c</k><a>bar</a></y2><y2><k>b</k><a>bar</a></y2><y3><k>a</k><a>bar</a></y3><y3><k>b</k><a>bar</a></y3><y3><k>c</k><a>bar</a></y3><y3><k>d</k><a>bar</a></y3></data></rpc-reply>]]>]]>$"
+
+new "get each ordered-by user leaf-list"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><running/></source><filter type=\"xpath\" select=\"/y2[k=a]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><y2><k>a</k><a>bar</a></y2></data></rpc-reply>]]>]]>$"
+
+new "get each ordered-by user leaf-list"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><running/></source><filter type=\"xpath\" select=\"/y3[k=a]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><y3><k>a</k><a>bar</a></y3></data></rpc-reply>]]>]]>$"
+
+new "get each ordered-by user leaf-list"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><running/></source><filter type=\"xpath\" select=\"/y2[k=b]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><y2><k>b</k><a>bar</a></y2></data></rpc-reply>]]>]]>$"
+
+new "get each ordered-by user leaf-list"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><running/></source><filter type=\"xpath\" select=\"/y3[k=b]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><y3><k>b</k><a>bar</a></y3></data></rpc-reply>]]>]]>$"
+
+new "delete candidate"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><delete-config><target><candidate/></target></delete-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+# LEAF_LISTS
+
+new "add two entries to leaf-list user order"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><y0>c</y0><y0>b</y0></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "add one entry to leaf-list user order"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><y0>a</y0></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf commit"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "add one entry to leaf-list user order after commit"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><y0>0</y0></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "netconf commit"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "verify leaf-list user order in running (as entered)"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><running/></source><filter type=\"xpath\" select=\"/y0\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><y0>c</y0><y0>b</y0><y0>a</y0><y0>0</y0></data></rpc-reply>]]>]]>$"
+
+# LISTS
+
+new "add two entries to list user order"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><y2><k>c</k><a>bar</a></y2><y2><k>b</k><a>foo</a></y2></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "add one entry to list user order"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><y2><k>a</k><a>fie</a></y2></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+new "verify list user order (as entered)"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/y2\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><y2><k>c</k><a>bar</a></y2><y2><k>b</k><a>foo</a></y2><y2><k>a</k><a>fie</a></y2></data></rpc-reply>]]>]]>$"
+
+pid=`pgrep clixon_backend`
+if [ -z "$pid" ]; then
+    err "backend already dead"
+fi
+# kill backend
+sudo clixon_backend -zf $cfg
+if [ $? -ne 0 ]; then
+    err "kill backend"
+fi

test/test_perf.sh

@@ -1,21 +1,27 @@
 #!/bin/bash
 # Scaling test
 
+number=1000
+req=100
 if [ $# = 0 ]; then
     number=1000
 elif [ $# = 1 ]; then
     number=$1
+elif [ $# = 2 ]; then
+    number=$1
+    req=$2
 else
-    echo "Usage: $0 [<number>]"
+    echo "Usage: $0 [<number> [<requests>]]"
     exit 1
 fi
-
+cfg=/tmp/scaling-conf.xml
 fyang=/tmp/scaling.yang
 fconfig=/tmp/config
 
 # include err() and new() functions
 . ./lib.sh
 
+
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
 # clixon_netconf="valgrind --tool=callgrind clixon_netconf 
@@ -40,16 +46,29 @@ module ietf-ip{
 }
 EOF
 
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>$fyang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>ietf-ip</CLICON_YANG_MODULE_MAIN>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
+
+
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf -y $fyang
+sudo clixon_backend -zf $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
-new "start backend"
+new "start backend  -s init -f $cfg -y $fyang"
 # start new backend
-sudo clixon_backend -s init -f $clixon_cf -y $fyang
+sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
@@ -61,26 +80,44 @@ for (( i=0; i<$number; i++ )); do
 done
 echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
 
+# Just for manual dbg
+echo "$clixon_netconf -qf $cfg  -y $fyang"
+
 new "netconf edit large config"
-expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+#echo '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' | $clixon_netconf -qf $cfg  -y $fyang 
 
 new "netconf edit large config again"
-expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+#echo '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' | $clixon_netconf -qf $cfg  -y $fyang 
 
 rm $fconfig
 
 new "netconf commit large config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
-new "netconf add small config"
+new "netconf commit same config again"
-expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><y><a>x</a><b>y</b></y></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+exit
+new "netconf add one small config"
+expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><y><a>x</a><b>y</b></y></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
-new "netconf commit small config"
+new "netconf add $req small config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+time -p for (( i=0; i<$req; i++ )); do
+    rnd=$(( ( RANDOM % $number ) ))
+    echo "<rpc><edit-config><target><candidate/></target><config><x><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 
 new "netconf get large config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>0</a><b>0</b></y><y><a>1</a><b>1</b>"
+expecteof "time -p $clixon_netconf -qf $cfg  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>0</a><b>0</b></y><y><a>1</a><b>1</b>"
 
+new "netconf get $req small config"
+time -p for (( i=0; i<$req; i++ )); do
+    rnd=$(( ( RANDOM % $number ) ))
+    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 
 new "generate large leaf-list config"
 echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x>" > $fconfig
@@ -90,21 +127,21 @@ done
 echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
 
 new "netconf replace large list-leaf config"
-expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 rm $fconfig
 
 new "netconf commit large leaf-list config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 new "netconf add small leaf-list config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><c>x</c></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><c>x</c></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 new "netconf commit small leaf-list config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 new "netconf get large leaf-list config"
-expecteof "time -p $clixon_netconf -qf $clixon_cf  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><c>0</c><c>1</c>"
+expecteof "time -p $clixon_netconf -qf $cfg  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><c>0</c><c>1</c>"
 
 new "Kill backend"
 # Check if still alive
@@ -113,7 +150,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_restconf.sh

@@ -4,18 +4,19 @@
 
 # include err() and new() functions
 . ./lib.sh
+cfg=/usr/local/etc/routing.xml
 
 # This is a fixed 'state' implemented in routing_backend. It is always there
 state='{"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}'
 
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err
 fi
 new "start backend"
-sudo clixon_backend -s init -f $clixon_cf
+sudo clixon_backend -s init -f $cfg
 if [ $? -ne 0 ]; then
     err
 fi
@@ -109,7 +110,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_startup.sh

@@ -8,12 +8,36 @@
 
 # include err() and new() functions
 . ./lib.sh
+cfg=/tmp/conf_startup.xml
 
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
 clixon_netconf=clixon_netconf
 clixon_cli=clixon_cli
 
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_BACKEND_DIR>/usr/local/lib/routing/backend</CLICON_BACKEND_DIR>
+  <CLICON_NETCONF_DIR>/usr/local/lib/routing/netconf</CLICON_NETCONF_DIR>
+  <CLICON_RESTCONF_DIR>/usr/local/lib/routing/restconf</CLICON_RESTCONF_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+  <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
+  <CLICON_STARTUP_MODE>init</CLICON_STARTUP_MODE>
+  <CLICON_XML_SORT>true</CLICON_XML_SORT>
+</config>
+
+EOF
+
 run(){
     mode=$1
     expect=$2
@@ -42,7 +66,7 @@ EOF
 EOF
     sudo mv /tmp/db /usr/local/var/routing/startup_db
 
-        cat <<EOF > /tmp/config
+    cat <<EOF > /tmp/config
 <config>
    <interfaces>
       <interface>
@@ -55,20 +79,20 @@ EOF
 
     # kill old backend (if any)
     new "kill old backend"
-    sudo clixon_backend -zf $clixon_cf 
+    sudo clixon_backend -zf $cfg
     if [ $? -ne 0 ]; then
 	err
     fi
 
     new "start backend"
     # start new backend
-    sudo clixon_backend -f $clixon_cf -s $mode -c /tmp/config
+    sudo clixon_backend -f $cfg -s $mode -c /tmp/config
     if [ $? -ne 0 ]; then
 	err
     fi
 
     new "Check $mode"
-    expecteof "$clixon_netconf -qf $clixon_cf" '<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$expect</rpc-reply>]]>]]>$"
+    expecteof "$clixon_netconf -qf $cfg" '<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$expect</rpc-reply>]]>]]>$"
 
     new "Kill backend"
     # Check if still alive
@@ -77,7 +101,7 @@ EOF
 	err "backend already dead"
     fi
     # kill backend
-    sudo clixon_backend -zf $clixon_cf
+    sudo clixon_backend -zf $cfg
     if [ $? -ne 0 ]; then
 	err "kill backend"
     fi
@@ -85,6 +109,6 @@ EOF
 
 run init    '<data/>'
 run none    '<data><interfaces><interface><name>run</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
-run running '<data><interfaces><interface><name>run</name><type>eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>local</type><enabled>true</enabled></interface><interface><name>extra</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
+run running '<data><interfaces><interface><name>extra</name><type>eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>local</type><enabled>true</enabled></interface><interface><name>run</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
-run startup '<data><interfaces><interface><name>startup</name><type>eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>local</type><enabled>true</enabled></interface><interface><name>extra</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
+run startup '<data><interfaces><interface><name>extra</name><type>eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>local</type><enabled>true</enabled></interface><interface><name>startup</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
 

test/test_type.sh

@@ -4,13 +4,15 @@
 
 # include err() and new() functions
 . ./lib.sh
+cfg=/usr/local/etc/routing.xml
+fyang=/tmp/type.yang
 
 # For memcheck
 #clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"
 clixon_cli=clixon_cli
 clixon_netconf=clixon_netconf
 
-cat <<EOF > /tmp/type.yang
+cat <<EOF > $fyang
 module example{
   typedef ab {
        type string {
@@ -63,55 +65,55 @@ EOF
 
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err
 fi
 new "start backend"
 # start new backend
-sudo clixon_backend -s init -f $clixon_cf -y /tmp/type.yang
+sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "cli set ab"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang set list a.b.a.b" "^$"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang set list a.b.a.b" "^$"
 
 new "cli set cd"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang set list c.d.c.d" "^$"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang set list c.d.c.d" "^$"
 
 new "cli set ef"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang set list e.f.e.f" "^$"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang set list e.f.e.f" "^$"
 
 new "cli set ab fail"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang set list a&b&a&b" "^CLI syntax error"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang set list a&b&a&b" "^CLI syntax error"
 
 new "cli set ad fail"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang set list a.b.c.d" "^CLI syntax error"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang set list a.b.c.d" "^CLI syntax error"
 
 new "cli validate"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang -l o validate" "^$"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang -l o validate" "^$"
 
 new "cli commit"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang -l o commit" "^$"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang -l o commit" "^$"
 
 new "netconf validate ok"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/type.yang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf set ab wrong"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/type.yang" "<rpc><edit-config><target><candidate/></target><config><list><ip>a.b&c.d</ip></list></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><list><ip>a.b&c.d</ip></list></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf validate"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/type.yang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><rpc-error>"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><rpc-error>"
 
 new "netconf discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/type.yang" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf commit"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/type.yang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "cli enum value"
-expectfn "$clixon_cli -1f $clixon_cf -l o -y /tmp/type.yang set status down" "^$"
+expectfn "$clixon_cli -1f $cfg -l o -y $fyang set status down" "^$"
 
 new "Kill backend"
 # Check if still alive
@@ -120,7 +122,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_yang.sh

@@ -3,16 +3,17 @@
 
 # include err() and new() functions
 . ./lib.sh
-clixon_cf=/tmp/conf_yang.xml
+cfg=/tmp/conf_yang.xml
+fyang=/tmp/test.yang
 
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
 clixon_netconf=clixon_netconf
 clixon_cli=clixon_cli
 
-cat <<EOF > /tmp/conf_yang.xml
+cat <<EOF > $cfg
 <config>
-  <CLICON_CONFIGFILE>/tmp/test_yang.xml</CLICON_CONFIGFILE>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
   <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
@@ -26,7 +27,7 @@ cat <<EOF > /tmp/conf_yang.xml
 </config>
 EOF
 
-cat <<EOF > /tmp/test.yang
+cat <<EOF > $fyang
 module example{
    container x {
     list y {
@@ -80,75 +81,73 @@ EOF
 
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf -y /tmp/test.yang
+sudo clixon_backend -zf $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
-new "start backend"
+new "start backend -s init -f $cfg -y $fyang"
 # start new backend
-sudo clixon_backend -s init -f $clixon_cf -y /tmp/test.yang
+sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "netconf edit config"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><edit-config><target><candidate/></target><config><x><y><a>1</a><b>2</b><c>5</c></y><d/></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><y><a>1</a><b>2</b><c>5</c></y><d/></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf commit"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 # text empty type in running
 new "netconf commit 2nd"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf get config xpath"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=1][b=2]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>1</a><b>2</b><c>5</c></y></x></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=1][b=2]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>1</a><b>2</b><c>5</c></y></x></data></rpc-reply>]]>]]>$"
 
 new "netconf edit leaf-list"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><edit-config><target><candidate/></target><config><x><f><e>hej</e><e>hopp</e></f></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><f><e>hej</e><e>hopp</e></f></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf get leaf-list"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/f/e\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><f><e>hej</e><e>hopp</e></f></x></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/f/e\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><f><e>hej</e><e>hopp</e></f></x></data></rpc-reply>]]>]]>$"
 
 new "netconf get leaf-list path"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/f[e=hej]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><f><e>hej</e><e>hopp</e></f></x></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/f[e=hej]\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><f><e>hej</e><e>hopp</e></f></x></data></rpc-reply>]]>]]>$"
 
 new "netconf get (should be some)"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><get><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>1</a><b>2</b><c>5</c></y><d/></x></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><get><filter type=\"xpath\" select=\"/\"/></get></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>1</a><b>2</b><c>5</c></y><d/></x></data></rpc-reply>]]>]]>$"
 
 new "cli set leaf-list"
-expectfn "$clixon_cli -1f $clixon_cf -y /tmp/test.yang set x f e foo" ""
+expectfn "$clixon_cli -1f $cfg -y $fyang set x f e foo" ""
 
 new "cli show leaf-list"
-expectfn "$clixon_cli -1f $clixon_cf -y /tmp/test.yang show xpath /x/f/e" "<e>foo</e>"
+expectfn "$clixon_cli -1f $cfg -y $fyang show xpath /x/f/e" "<e>foo</e>"
 new "netconf set state data (not allowed)"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><edit-config><target><candidate/></target><config><state><op>42</op></state></config></edit-config></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-tag>invalid-value"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><state><op>42</op></state></config></edit-config></rpc>]]>]]>" "^<rpc-reply><rpc-error><error-tag>invalid-value"
 
 new "netconf set presence and not present"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><edit-config><target><candidate/></target><config><x><nopresence/><presence/></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><nopresence/><presence/></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
-new "netconf get"
+new "netconf get presence only"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/*presence\"/></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><presence/></x></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" '<rpc><get-config><source><candidate/></source><filter type="xpath" select="/x/*presence"/></get-config></rpc>]]>]]>' "^<rpc-reply><data><x><presence/></x></data></rpc-reply>]]>]]>$"
 
 new "netconf discard-changes"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
-
 
 new "netconf anyxml"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><edit-config><target><candidate/></target><config><x><any><foo><bar a=\"nisse\"/></foo></any></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><any><foo><bar a=\"nisse\"/></foo></any></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 new "netconf validate anyxml"
-expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidate/></source></validate></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
-new "Kill backend"
 # Check if still alive
 pid=`pgrep clixon_backend`
 if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-sudo clixon_backend -zf $clixon_cf
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

yang/Makefile.in

@@ -38,8 +38,9 @@ bindir 		= @bindir@
 includedir	= @includedir@
 datarootdir     = @datarootdir@
 
-YANGSPECS	 = clixon-config@2017-07-02.yang
+YANGSPECS	 = clixon-config@2017-12-27.yang
 YANGSPECS	+= ietf-netconf@2011-06-01.yang
+YANGSPECS	+= ietf-inet-types@2013-07-15.yang
 
 APPNAME	        = clixon  # subdir ehere these files are installed
 

yang/clixon-config@2017-07-02.yang

@@ -38,7 +38,7 @@
 
        ***** END LICENSE BLOCK *****";
 
-    revision 2017-11-12 {
+    revision 2017-07-02 {
       description
         "Added startup config";
     }

yang/clixon-config@2017-12-03.yang

@@ -0,0 +1,256 @@
+module clixon-config {
+
+    prefix cc;
+
+    organization
+	"Clicon / Clixon";
+
+    contact
+	"Olof Hagsand <olof@hagsand.se>";
+
+    description
+      "Clixon configuration file
+       ***** BEGIN LICENSE BLOCK *****
+       Copyright (C) 2009-2017 Olof Hagsand and Benny Holmgren
+       
+       This file is part of CLIXON
+
+       Licensed under the Apache License, Version 2.0 (the \"License\");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+            http://www.apache.org/licenses/LICENSE-2.0
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an \"AS IS\" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
+
+       Alternatively, the contents of this file may be used under the terms of
+       the GNU General Public License Version 3 or later (the \"GPL\"),
+       in which case the provisions of the GPL are applicable instead
+       of those above. If you wish to allow use of your version of this file only
+       under the terms of the GPL, and not to allow others to
+       use your version of this file under the terms of Apache License version 2, 
+       indicate your decision by deleting the provisions above and replace them with
+       the notice and other provisions required by the GPL. If you do not delete
+       the provisions above, a recipient may use your version of this file under
+       the terms of any one of the Apache License version 2 or the GPL.
+
+       ***** END LICENSE BLOCK *****";
+
+    revision 2017-12-03 {
+	description
+	    "Added startup config for Clixon 1.3.3 and xmldb_cache in 1.4.0";
+    }
+    typedef startup_mode{
+	description
+	    "Which method to boot/start clicon backend.
+             The methods differ in how they reach a running state
+             Which source database to commit from, if any.";
+	type enumeration{
+	    enum none{
+		description
+		"Do not touch running state
+                 Typically after crash when running state and db are synched";
+	    }
+	    enum init{
+		description
+		"Initialize running state.
+                 Start with a completely clean running state";
+	    }
+	    enum running{
+		description
+		"Commit running db configuration into running state
+                 After reboot if a persistent running db exists";
+	    }
+	    enum startup{
+		description
+		"Commit startup configuration into running state
+                 After reboot when no persistent running db exists";
+	    }
+	}
+    }
+    container config {
+	leaf CLICON_CONFIGFILE{
+	    type string;
+	    description
+		"Location of configuration-file for default values (this file)";
+	}
+	leaf CLICON_YANG_DIR {
+	    type string;
+	    mandatory true;
+	    description
+		"Location of YANG module and submodule files.";
+	}
+	leaf CLICON_YANG_MODULE_MAIN {
+	    type string;
+            default "clicon";
+	    description
+		"Option used to construct initial yang file: 
+                 <module>[@<revision>]";
+	}
+	leaf CLICON_YANG_MODULE_REVISION {
+	    type string;
+	    description
+		"Option used to construct initial yang file: 
+                 <module>[@<revision>]";
+	}
+	leaf CLICON_BACKEND_DIR {
+	    type string;
+	    description
+		"Location of backend .so plugins. Load all .so 
+       	         plugins in this dir as backend plugins";
+	}
+	leaf CLICON_NETCONF_DIR {
+	    type string;
+	    description "Location of netconf (frontend) .so plugins";
+	}
+	leaf CLICON_RESTCONF_DIR {
+	    type string;
+	    description
+		"Location of restconf (frontend) .so plugins. Load all .so
+       	         plugins in this dir as restconf code plugins";
+	}
+	leaf CLICON_RESTCONF_PATH {
+	    type string;
+	    default "/www-data/fastcgi_restconf.sock";
+	    description
+		"FastCGI unix socket. Should be specified in webserver
+         	 Eg in nginx: fastcgi_pass unix:/www-data/clicon_restconf.sock";
+	}
+	leaf CLICON_CLI_DIR {
+	    type string;
+	    description
+		"Location of cli frontend .so plugins. Load all .so 
+                 plugins in this dir as CLI object plugins";
+	}
+	leaf CLICON_CLISPEC_DIR {
+	    type string;
+	    description
+		"Location of frontend .cli cligen spec files. Load all .cli 
+       	         files in this dir as CLI specification files";
+	}
+	leaf CLICON_CLISPEC_FILE {
+	    type string;
+	    description "Specific frontend .cli cligen spec file.";
+	}
+	leaf CLICON_CLI_MODE {
+	    type string;
+	    default "base";
+	    description
+		"Startup CLI mode. This should match a CLICON_MODE set in
+                 one of the clispec files";
+	}
+	leaf CLICON_CLI_GENMODEL {
+	    type int32;
+	    default 1;
+	    description
+		"Generate code for CLI completion of existing db symbols. 
+      	         Example: Add name=\"myspec\" in datamodel spec and reference
+	         as @myspec";
+	}
+	leaf CLICON_CLI_GENMODEL_COMPLETION {
+	    type int32;
+	    default 1;
+	    description "Generate code for CLI completion of existing db symbols";
+	}
+	leaf CLICON_CLI_GENMODEL_TYPE {
+	    type string;
+	    default "VARS";
+	    description "How to generate and show CLI syntax: VARS|ALL";
+	}
+	leaf CLICON_CLI_VARONLY {
+	    type int32;
+	    default 1;
+	    description
+		"Dont include keys in cvec in cli vars callbacks, 
+          	 ie a & k in 'a <b> k <c>' ignored";
+	}
+	leaf CLICON_CLI_LINESCROLLING {
+	    type int32;
+	    default 1;
+	    description
+		"Set to 0 if you want CLI to wrap to next line.
+                 Set to 1 if you  want CLI to scroll sideways when approaching 
+                      right margin";
+	}
+	leaf CLICON_SOCK_FAMILY {
+	    type string;
+	    default "UNIX";
+	    description
+		"Address family for communicating with clixon_backend 
+                 (UNIX|IPv4|IPv6)";
+	}
+	leaf CLICON_SOCK {
+	    type string;
+	    mandatory true;
+	    description
+		"If family above is AF_UNIX: Unix socket for communicating 
+       	         with clixon_backend. If family is AF_INET: IPv4 address";
+	}
+	leaf CLICON_SOCK_PORT {
+	    type int32;
+	    default 4535;
+	    description
+		"Inet socket port for communicating with clixon_backend 
+                 (only IPv4|IPv6)";
+	}
+	leaf CLICON_SOCK_GROUP {
+	    type string;
+	    default "clicon";
+	    description "Group membership to access clixon_backend unix socket";
+	}
+	leaf CLICON_BACKEND_PIDFILE {
+	    type string;
+	    mandatory true;
+	    description "Process-id file of backend daemon";
+	}
+	leaf CLICON_AUTOCOMMIT {
+	    type int32;
+	    default 0;
+	    description
+		"Set if all configuration changes are committed automatically 
+                 on every edit change. Explicit commit commands unnecessary";
+	}
+	leaf CLICON_MASTER_PLUGIN {
+	    type string;
+	    default "master";
+	    description
+		"Name of master plugin (both frontend and backend). 
+                 Master plugin has special callbacks for frontends. 
+	         See clicon user manual for more info. (Obsolete?)";
+	}
+	leaf CLICON_XMLDB_DIR {
+	    type string;
+	    mandatory true;
+	    description
+		"Directory where \"running\", \"candidate\" and \"startup\" are placed";
+	}
+	leaf CLICON_XMLDB_PLUGIN {
+	    type string;
+	    mandatory true;
+	    description
+		"XMLDB datastore plugin filename 
+	         (see datastore/ and clixon_xml_db.[ch])";
+	}
+	leaf CLICON_XMLDB_CACHE {
+	    type boolean;
+	    default true;
+	    description
+		"XMLDB datastore cache. 
+                 If set, XML parsed tree is stored in memory";
+	}
+	leaf CLICON_USE_STARTUP_CONFIG {
+	    type int32;
+	    default 0;
+	    description
+		"Enabled uses \"startup\" configuration on boot. It is called
+                 startup_db and exists in XMLDB_DIR. 
+	         NOTE: Obsolete with 1.3.3 and CLICON_STARTUP_MODE";
+	}
+	leaf CLICON_STARTUP_MODE {
+	    type startup_mode;
+	    description "Which method to boot/start clicon backend";
+	}
+    }
+}

yang/clixon-config@2017-12-27.yang

@@ -0,0 +1,291 @@
+module clixon-config {
+
+    prefix cc;
+
+    organization
+	"Clicon / Clixon";
+
+    contact
+	"Olof Hagsand <olof@hagsand.se>";
+
+    description
+      "Clixon configuration file
+       ***** BEGIN LICENSE BLOCK *****
+       Copyright (C) 2009-2017 Olof Hagsand and Benny Holmgren
+       
+       This file is part of CLIXON
+
+       Licensed under the Apache License, Version 2.0 (the \"License\");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+            http://www.apache.org/licenses/LICENSE-2.0
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an \"AS IS\" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
+
+       Alternatively, the contents of this file may be used under the terms of
+       the GNU General Public License Version 3 or later (the \"GPL\"),
+       in which case the provisions of the GPL are applicable instead
+       of those above. If you wish to allow use of your version of this file only
+       under the terms of the GPL, and not to allow others to
+       use your version of this file under the terms of Apache License version 2, 
+       indicate your decision by deleting the provisions above and replace them with
+       the notice and other provisions required by the GPL. If you do not delete
+       the provisions above, a recipient may use your version of this file under
+       the terms of any one of the Apache License version 2 or the GPL.
+
+       ***** END LICENSE BLOCK *****";
+
+    revision 2017-12-27 {
+	description
+	    "Added xml sort option for 1.4.0";
+    }
+    typedef startup_mode{
+	description
+	    "Which method to boot/start clicon backend.
+             The methods differ in how they reach a running state
+             Which source database to commit from, if any.";
+	type enumeration{
+	    enum none{
+		description
+		"Do not touch running state
+                 Typically after crash when running state and db are synched";
+	    }
+	    enum init{
+		description
+		"Initialize running state.
+                 Start with a completely clean running state";
+	    }
+	    enum running{
+		description
+		"Commit running db configuration into running state
+                 After reboot if a persistent running db exists";
+	    }
+	    enum startup{
+		description
+		"Commit startup configuration into running state
+                 After reboot when no persistent running db exists";
+	    }
+	}
+    }
+    typedef xmldb_format{
+	description
+	    "Format of TEXT xml database format.";
+	type enumeration{
+	    enum xml{
+		description "Save and load xmldb as XML";
+	    }
+	    enum json{
+		description "Save and load xmldb as JSON";
+	    }
+	}
+    }
+    container config {
+	leaf CLICON_CONFIGFILE{
+	    type string;
+	    description
+		"Location of configuration-file for default values (this file)";
+	}
+	leaf CLICON_YANG_DIR {
+	    type string;
+	    mandatory true;
+	    description
+		"Location of YANG module and submodule files.";
+	}
+	leaf CLICON_YANG_MODULE_MAIN {
+	    type string;
+            default "clicon";
+	    description
+		"Option used to construct initial yang file: 
+                 <module>[@<revision>]";
+	}
+	leaf CLICON_YANG_MODULE_REVISION {
+	    type string;
+	    description
+		"Option used to construct initial yang file: 
+                 <module>[@<revision>]";
+	}
+	leaf CLICON_BACKEND_DIR {
+	    type string;
+	    description
+		"Location of backend .so plugins. Load all .so 
+       	         plugins in this dir as backend plugins";
+	}
+	leaf CLICON_NETCONF_DIR {
+	    type string;
+	    description "Location of netconf (frontend) .so plugins";
+	}
+	leaf CLICON_RESTCONF_DIR {
+	    type string;
+	    description
+		"Location of restconf (frontend) .so plugins. Load all .so
+       	         plugins in this dir as restconf code plugins";
+	}
+	leaf CLICON_RESTCONF_PATH {
+	    type string;
+	    default "/www-data/fastcgi_restconf.sock";
+	    description
+		"FastCGI unix socket. Should be specified in webserver
+         	 Eg in nginx: fastcgi_pass unix:/www-data/clicon_restconf.sock";
+	}
+	leaf CLICON_CLI_DIR {
+	    type string;
+	    description
+		"Location of cli frontend .so plugins. Load all .so 
+                 plugins in this dir as CLI object plugins";
+	}
+	leaf CLICON_CLISPEC_DIR {
+	    type string;
+	    description
+		"Location of frontend .cli cligen spec files. Load all .cli 
+       	         files in this dir as CLI specification files";
+	}
+	leaf CLICON_CLISPEC_FILE {
+	    type string;
+	    description "Specific frontend .cli cligen spec file.";
+	}
+	leaf CLICON_CLI_MODE {
+	    type string;
+	    default "base";
+	    description
+		"Startup CLI mode. This should match a CLICON_MODE set in
+                 one of the clispec files";
+	}
+	leaf CLICON_CLI_GENMODEL {
+	    type int32;
+	    default 1;
+	    description
+		"Generate code for CLI completion of existing db symbols. 
+      	         Example: Add name=\"myspec\" in datamodel spec and reference
+	         as @myspec";
+	}
+	leaf CLICON_CLI_GENMODEL_COMPLETION {
+	    type int32;
+	    default 1;
+	    description "Generate code for CLI completion of existing db symbols";
+	}
+	leaf CLICON_CLI_GENMODEL_TYPE {
+	    type string;
+	    default "VARS";
+	    description "How to generate and show CLI syntax: VARS|ALL";
+	}
+	leaf CLICON_CLI_VARONLY {
+	    type int32;
+	    default 1;
+	    description
+		"Dont include keys in cvec in cli vars callbacks, 
+          	 ie a & k in 'a <b> k <c>' ignored";
+	}
+	leaf CLICON_CLI_LINESCROLLING {
+	    type int32;
+	    default 1;
+	    description
+		"Set to 0 if you want CLI to wrap to next line.
+                 Set to 1 if you  want CLI to scroll sideways when approaching 
+                      right margin";
+	}
+	leaf CLICON_SOCK_FAMILY {
+	    type string;
+	    default "UNIX";
+	    description
+		"Address family for communicating with clixon_backend 
+                 (UNIX|IPv4|IPv6)";
+	}
+	leaf CLICON_SOCK {
+	    type string;
+	    mandatory true;
+	    description
+		"If family above is AF_UNIX: Unix socket for communicating 
+       	         with clixon_backend. If family is AF_INET: IPv4 address";
+	}
+	leaf CLICON_SOCK_PORT {
+	    type int32;
+	    default 4535;
+	    description
+		"Inet socket port for communicating with clixon_backend 
+                 (only IPv4|IPv6)";
+	}
+	leaf CLICON_SOCK_GROUP {
+	    type string;
+	    default "clicon";
+	    description "Group membership to access clixon_backend unix socket";
+	}
+	leaf CLICON_BACKEND_PIDFILE {
+	    type string;
+	    mandatory true;
+	    description "Process-id file of backend daemon";
+	}
+	leaf CLICON_AUTOCOMMIT {
+	    type int32;
+	    default 0;
+	    description
+		"Set if all configuration changes are committed automatically 
+                 on every edit change. Explicit commit commands unnecessary";
+	}
+	leaf CLICON_MASTER_PLUGIN {
+	    type string;
+	    default "master";
+	    description
+		"Name of master plugin (both frontend and backend). 
+                 Master plugin has special callbacks for frontends. 
+	         See clicon user manual for more info. (Obsolete?)";
+	}
+	leaf CLICON_XMLDB_DIR {
+	    type string;
+	    mandatory true;
+	    description
+		"Directory where \"running\", \"candidate\" and \"startup\" are placed";
+	}
+	leaf CLICON_XMLDB_PLUGIN {
+	    type string;
+	    mandatory true;
+	    description
+		"XMLDB datastore plugin filename 
+	         (see datastore/ and clixon_xml_db.[ch])";
+	}
+	leaf CLICON_XMLDB_CACHE {
+	    type boolean;
+	    default true;
+	    description
+		"XMLDB datastore cache. 
+                 If set, XML candidate/running parsed tree is stored in memory
+                 If not set, candidate/running is always accessed via disk.";
+	}
+	leaf CLICON_XMLDB_FORMAT {
+	    type xmldb_format;
+	    default xml;
+	    description	"XMLDB datastore format.";
+	}
+	leaf CLICON_XMLDB_PRETTY {
+	    type boolean;
+	    default true;
+	    description
+		"XMLDB datastore pretty print. 
+                 If set, insert spaces and line-feeds making the XML/JSON human
+                 readable. If not set, make the XML/JSON more compact.";
+	}
+	leaf CLICON_XML_SORT {
+	    type boolean;
+	    default true;
+	    description
+		"If set, sort XML lists and leaf-lists alphabetically and uses binary
+                 search. Unless ordered-by user is used. 
+                 Only works for Yang specified XML.
+                 If not set, all lists accessed via linear search.";
+	}
+	leaf CLICON_USE_STARTUP_CONFIG {
+	    type int32;
+	    default 0;
+	    description
+		"Enabled uses \"startup\" configuration on boot. It is called
+                 startup_db and exists in XMLDB_DIR. 
+	         NOTE: Obsolete with 1.3.3 and CLICON_STARTUP_MODE";
+	}
+	leaf CLICON_STARTUP_MODE {
+	    type startup_mode;
+	    description "Which method to boot/start clicon backend";
+	}
+    }
+}

yang/ietf-inet-types@2013-07-15.yang

@@ -0,0 +1,457 @@
+   module ietf-inet-types {
+
+     namespace "urn:ietf:params:xml:ns:yang:ietf-inet-types";
+     prefix "inet";
+
+     organization
+      "IETF NETMOD (NETCONF Data Modeling Language) Working Group";
+
+     contact
+      "WG Web:   <http://tools.ietf.org/wg/netmod/>
+       WG List:  <mailto:netmod@ietf.org>
+
+       WG Chair: David Kessens
+                 <mailto:david.kessens@nsn.com>
+
+       WG Chair: Juergen Schoenwaelder
+                 <mailto:j.schoenwaelder@jacobs-university.de>
+
+       Editor:   Juergen Schoenwaelder
+                 <mailto:j.schoenwaelder@jacobs-university.de>";
+
+     description
+      "This module contains a collection of generally useful derived
+       YANG data types for Internet addresses and related things.
+
+       Copyright (c) 2013 IETF Trust and the persons identified as
+       authors of the code.  All rights reserved.
+
+       Redistribution and use in source and binary forms, with or
+       without modification, is permitted pursuant to, and subject
+       to the license terms contained in, the Simplified BSD License
+       set forth in Section 4.c of the IETF Trust's Legal Provisions
+       Relating to IETF Documents
+       (http://trustee.ietf.org/license-info).
+
+       This version of this YANG module is part of RFC 6991; see
+       the RFC itself for full legal notices.";
+
+     revision 2013-07-15 {
+       description
+        "This revision adds the following new data types:
+         - ip-address-no-zone
+         - ipv4-address-no-zone
+         - ipv6-address-no-zone";
+       reference
+        "RFC 6991: Common YANG Data Types";
+     }
+
+     revision 2010-09-24 {
+       description
+        "Initial revision.";
+       reference
+        "RFC 6021: Common YANG Data Types";
+     }
+
+     /*** collection of types related to protocol fields ***/
+
+     typedef ip-version {
+       type enumeration {
+         enum unknown {
+           value "0";
+           description
+            "An unknown or unspecified version of the Internet
+             protocol.";
+         }
+         enum ipv4 {
+           value "1";
+           description
+            "The IPv4 protocol as defined in RFC 791.";
+         }
+         enum ipv6 {
+           value "2";
+           description
+            "The IPv6 protocol as defined in RFC 2460.";
+         }
+       }
+       description
+        "This value represents the version of the IP protocol.
+
+         In the value set and its semantics, this type is equivalent
+         to the InetVersion textual convention of the SMIv2.";
+       reference
+        "RFC  791: Internet Protocol
+         RFC 2460: Internet Protocol, Version 6 (IPv6) Specification
+         RFC 4001: Textual Conventions for Internet Network Addresses";
+     }
+
+     typedef dscp {
+       type uint8 {
+         range "0..63";
+       }
+       description
+        "The dscp type represents a Differentiated Services Code Point
+         that may be used for marking packets in a traffic stream.
+         In the value set and its semantics, this type is equivalent
+         to the Dscp textual convention of the SMIv2.";
+       reference
+        "RFC 3289: Management Information Base for the Differentiated
+                   Services Architecture
+         RFC 2474: Definition of the Differentiated Services Field
+                   (DS Field) in the IPv4 and IPv6 Headers
+         RFC 2780: IANA Allocation Guidelines For Values In
+                   the Internet Protocol and Related Headers";
+     }
+
+     typedef ipv6-flow-label {
+       type uint32 {
+         range "0..1048575";
+       }
+       description
+        "The ipv6-flow-label type represents the flow identifier or Flow
+         Label in an IPv6 packet header that may be used to
+         discriminate traffic flows.
+
+         In the value set and its semantics, this type is equivalent
+         to the IPv6FlowLabel textual convention of the SMIv2.";
+       reference
+        "RFC 3595: Textual Conventions for IPv6 Flow Label
+         RFC 2460: Internet Protocol, Version 6 (IPv6) Specification";
+     }
+
+     typedef port-number {
+       type uint16 {
+         range "0..65535";
+       }
+       description
+        "The port-number type represents a 16-bit port number of an
+         Internet transport-layer protocol such as UDP, TCP, DCCP, or
+         SCTP.  Port numbers are assigned by IANA.  A current list of
+         all assignments is available from <http://www.iana.org/>.
+
+         Note that the port number value zero is reserved by IANA.  In
+         situations where the value zero does not make sense, it can
+         be excluded by subtyping the port-number type.
+         In the value set and its semantics, this type is equivalent
+         to the InetPortNumber textual convention of the SMIv2.";
+       reference
+        "RFC  768: User Datagram Protocol
+         RFC  793: Transmission Control Protocol
+         RFC 4960: Stream Control Transmission Protocol
+         RFC 4340: Datagram Congestion Control Protocol (DCCP)
+         RFC 4001: Textual Conventions for Internet Network Addresses";
+     }
+
+     /*** collection of types related to autonomous systems ***/
+
+     typedef as-number {
+       type uint32;
+       description
+        "The as-number type represents autonomous system numbers
+         which identify an Autonomous System (AS).  An AS is a set
+         of routers under a single technical administration, using
+         an interior gateway protocol and common metrics to route
+         packets within the AS, and using an exterior gateway
+         protocol to route packets to other ASes.  IANA maintains
+         the AS number space and has delegated large parts to the
+         regional registries.
+
+         Autonomous system numbers were originally limited to 16
+         bits.  BGP extensions have enlarged the autonomous system
+         number space to 32 bits.  This type therefore uses an uint32
+         base type without a range restriction in order to support
+         a larger autonomous system number space.
+
+         In the value set and its semantics, this type is equivalent
+         to the InetAutonomousSystemNumber textual convention of
+         the SMIv2.";
+       reference
+        "RFC 1930: Guidelines for creation, selection, and registration
+                   of an Autonomous System (AS)
+         RFC 4271: A Border Gateway Protocol 4 (BGP-4)
+         RFC 4001: Textual Conventions for Internet Network Addresses
+         RFC 6793: BGP Support for Four-Octet Autonomous System (AS)
+                   Number Space";
+     }
+
+     /*** collection of types related to IP addresses and hostnames ***/
+
+     typedef ip-address {
+       type union {
+         type inet:ipv4-address;
+         type inet:ipv6-address;
+       }
+       description
+        "The ip-address type represents an IP address and is IP
+         version neutral.  The format of the textual representation
+         implies the IP version.  This type supports scoped addresses
+         by allowing zone identifiers in the address format.";
+       reference
+        "RFC 4007: IPv6 Scoped Address Architecture";
+     }
+
+     typedef ipv4-address {
+       type string {
+         pattern
+           '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}'
+         +  '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'
+         + '(%[\p{N}\p{L}]+)?';
+       }
+       description
+         "The ipv4-address type represents an IPv4 address in
+          dotted-quad notation.  The IPv4 address may include a zone
+          index, separated by a % sign.
+
+          The zone index is used to disambiguate identical address
+          values.  For link-local addresses, the zone index will
+          typically be the interface index number or the name of an
+          interface.  If the zone index is not present, the default
+          zone of the device will be used.
+
+          The canonical format for the zone index is the numerical
+          format";
+     }
+
+     typedef ipv6-address {
+       type string {
+         pattern '((:|[0-9a-fA-F]{0,4}):)([0-9a-fA-F]{0,4}:){0,5}'
+               + '((([0-9a-fA-F]{0,4}:)?(:|[0-9a-fA-F]{0,4}))|'
+               + '(((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}'
+               + '(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])))'
+               + '(%[\p{N}\p{L}]+)?';
+         pattern '(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|'
+               + '((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)'
+               + '(%.+)?';
+       }
+       description
+        "The ipv6-address type represents an IPv6 address in full,
+         mixed, shortened, and shortened-mixed notation.  The IPv6
+         address may include a zone index, separated by a % sign.
+
+         The zone index is used to disambiguate identical address
+         values.  For link-local addresses, the zone index will
+         typically be the interface index number or the name of an
+         interface.  If the zone index is not present, the default
+         zone of the device will be used.
+
+         The canonical format of IPv6 addresses uses the textual
+         representation defined in Section 4 of RFC 5952.  The
+         canonical format for the zone index is the numerical
+         format as described in Section 11.2 of RFC 4007.";
+       reference
+        "RFC 4291: IP Version 6 Addressing Architecture
+         RFC 4007: IPv6 Scoped Address Architecture
+         RFC 5952: A Recommendation for IPv6 Address Text
+                   Representation";
+     }
+
+     typedef ip-address-no-zone {
+       type union {
+         type inet:ipv4-address-no-zone;
+         type inet:ipv6-address-no-zone;
+       }
+       description
+        "The ip-address-no-zone type represents an IP address and is
+         IP version neutral.  The format of the textual representation
+         implies the IP version.  This type does not support scoped
+         addresses since it does not allow zone identifiers in the
+         address format.";
+       reference
+        "RFC 4007: IPv6 Scoped Address Architecture";
+     }
+
+     typedef ipv4-address-no-zone {
+       type inet:ipv4-address {
+         pattern '[0-9\.]*';
+       }
+       description
+         "An IPv4 address without a zone index.  This type, derived from
+          ipv4-address, may be used in situations where the zone is
+          known from the context and hence no zone index is needed.";
+     }
+
+     typedef ipv6-address-no-zone {
+       type inet:ipv6-address {
+         pattern '[0-9a-fA-F:\.]*';
+       }
+       description
+         "An IPv6 address without a zone index.  This type, derived from
+          ipv6-address, may be used in situations where the zone is
+          known from the context and hence no zone index is needed.";
+       reference
+        "RFC 4291: IP Version 6 Addressing Architecture
+         RFC 4007: IPv6 Scoped Address Architecture
+         RFC 5952: A Recommendation for IPv6 Address Text
+                   Representation";
+     }
+
+     typedef ip-prefix {
+       type union {
+         type inet:ipv4-prefix;
+         type inet:ipv6-prefix;
+       }
+       description
+        "The ip-prefix type represents an IP prefix and is IP
+         version neutral.  The format of the textual representations
+         implies the IP version.";
+     }
+
+     typedef ipv4-prefix {
+       type string {
+         pattern
+            '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}'
+          +  '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'
+          + '/(([0-9])|([1-2][0-9])|(3[0-2]))';
+       }
+       description
+        "The ipv4-prefix type represents an IPv4 address prefix.
+         The prefix length is given by the number following the
+         slash character and must be less than or equal to 32.
+
+         A prefix length value of n corresponds to an IP address
+         mask that has n contiguous 1-bits from the most
+         significant bit (MSB) and all other bits set to 0.
+
+         The canonical format of an IPv4 prefix has all bits of
+         the IPv4 address set to zero that are not part of the
+         IPv4 prefix.";
+     }
+
+     typedef ipv6-prefix {
+       type string {
+         pattern '((:|[0-9a-fA-F]{0,4}):)([0-9a-fA-F]{0,4}:){0,5}'
+               + '((([0-9a-fA-F]{0,4}:)?(:|[0-9a-fA-F]{0,4}))|'
+               + '(((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}'
+               + '(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])))'
+               + '(/(([0-9])|([0-9]{2})|(1[0-1][0-9])|(12[0-8])))';
+         pattern '(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|'
+               + '((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)'
+               + '(/.+)';
+       }
+       description
+        "The ipv6-prefix type represents an IPv6 address prefix.
+         The prefix length is given by the number following the
+         slash character and must be less than or equal to 128.
+
+         A prefix length value of n corresponds to an IP address
+         mask that has n contiguous 1-bits from the most
+         significant bit (MSB) and all other bits set to 0.
+
+         The IPv6 address should have all bits that do not belong
+         to the prefix set to zero.
+
+         The canonical format of an IPv6 prefix has all bits of
+         the IPv6 address set to zero that are not part of the
+         IPv6 prefix.  Furthermore, the IPv6 address is represented
+         as defined in Section 4 of RFC 5952.";
+       reference
+        "RFC 5952: A Recommendation for IPv6 Address Text
+                   Representation";
+     }
+
+     /*** collection of domain name and URI types ***/
+
+     typedef domain-name {
+       type string {
+         pattern
+           '((([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.)*'
+         + '([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.?)'
+         + '|\.';
+         length "1..253";
+       }
+       description
+        "The domain-name type represents a DNS domain name.  The
+         name SHOULD be fully qualified whenever possible.
+
+         Internet domain names are only loosely specified.  Section
+         3.5 of RFC 1034 recommends a syntax (modified in Section
+         2.1 of RFC 1123).  The pattern above is intended to allow
+         for current practice in domain name use, and some possible
+         future expansion.  It is designed to hold various types of
+         domain names, including names used for A or AAAA records
+         (host names) and other records, such as SRV records.  Note
+         that Internet host names have a stricter syntax (described
+         in RFC 952) than the DNS recommendations in RFCs 1034 and
+         1123, and that systems that want to store host names in
+         schema nodes using the domain-name type are recommended to
+         adhere to this stricter standard to ensure interoperability.
+
+         The encoding of DNS names in the DNS protocol is limited
+         to 255 characters.  Since the encoding consists of labels
+         prefixed by a length bytes and there is a trailing NULL
+         byte, only 253 characters can appear in the textual dotted
+         notation.
+
+         The description clause of schema nodes using the domain-name
+         type MUST describe when and how these names are resolved to
+         IP addresses.  Note that the resolution of a domain-name value
+         may require to query multiple DNS records (e.g., A for IPv4
+         and AAAA for IPv6).  The order of the resolution process and
+         which DNS record takes precedence can either be defined
+         explicitly or may depend on the configuration of the
+         resolver.
+
+         Domain-name values use the US-ASCII encoding.  Their canonical
+         format uses lowercase US-ASCII characters.  Internationalized
+         domain names MUST be A-labels as per RFC 5890.";
+       reference
+        "RFC  952: DoD Internet Host Table Specification
+         RFC 1034: Domain Names - Concepts and Facilities
+         RFC 1123: Requirements for Internet Hosts -- Application
+                   and Support
+         RFC 2782: A DNS RR for specifying the location of services
+                   (DNS SRV)
+         RFC 5890: Internationalized Domain Names in Applications
+                   (IDNA): Definitions and Document Framework";
+     }
+
+     typedef host {
+       type union {
+         type inet:ip-address;
+         type inet:domain-name;
+       }
+       description
+        "The host type represents either an IP address or a DNS
+         domain name.";
+     }
+
+     typedef uri {
+       type string;
+       description
+        "The uri type represents a Uniform Resource Identifier
+         (URI) as defined by STD 66.
+
+         Objects using the uri type MUST be in US-ASCII encoding,
+         and MUST be normalized as described by RFC 3986 Sections
+         6.2.1, 6.2.2.1, and 6.2.2.2.  All unnecessary
+         percent-encoding is removed, and all case-insensitive
+         characters are set to lowercase except for hexadecimal
+         digits, which are normalized to uppercase as described in
+         Section 6.2.2.1.
+
+         The purpose of this normalization is to help provide
+         unique URIs.  Note that this normalization is not
+         sufficient to provide uniqueness.  Two URIs that are
+         textually distinct after this normalization may still be
+         equivalent.
+
+         Objects using the uri type may restrict the schemes that
+         they permit.  For example, 'data:' and 'urn:' schemes
+         might not be appropriate.
+
+         A zero-length URI is not a valid URI.  This can be used to
+         express 'URI absent' where required.
+
+         In the value set and its semantics, this type is equivalent
+         to the Uri SMIv2 textual convention defined in RFC 5017.";
+       reference
+        "RFC 3986: Uniform Resource Identifier (URI): Generic Syntax
+         RFC 3305: Report from the Joint W3C/IETF URI Planning Interest
+                   Group: Uniform Resource Identifiers (URIs), URLs,
+                   and Uniform Resource Names (URNs): Clarifications
+                   and Recommendations
+         RFC 5017: MIB Textual Conventions for Uniform Resource
+                   Identifiers (URIs)";
+     }
+
+   }