Merge branch 'master' of https://github.com/clicon/clixon

.gitignore

@@ -14,6 +14,7 @@ docker/Makefile
 docker/*/Makefile
 etc/Makefile
 example/Makefile
+example/*/Makefile
 lib/Makefile
 lib/*/Makefile
 test/Makefile

CHANGELOG.md

@@ -32,9 +32,19 @@
   * Two config options control:
     * CLICON_XML_CHANGELOG enables the yang changelog feature
     * CLICON_XML_CHANGELOG_FILE where the changelog resides
+* Optimization work
+  * Improved performance of validation of (large) lists
+  * A scaling of [large lists](doc/scaling) report is added
+  * New xmldb_get1() returning actual cache - not a copy. This has lead to some householding instead of just deleting the copy
+  * xml_diff rewritten to work linearly instead of O(2)
+  * New xml_insert function using tree search. The new code uses this in insertion xmldb_put and defaults. (Note previous xml_insert renamed to xml_wrap_all)
+  * A yang type regex cache added, this helps the performance by avoiding re-running the `regcomp` command on every iteration.
+  * An XML namespace cache added (see `xml2ns()`)
+  * Better performance of XML whitespace parsing/scanning.
 	
 ### API changes on existing features (you may need to change your code)
 
+* The directory `docker/system` has been moved to `docker/main`, to reflect that it runs the main example.
 * xmldb_get() removed "config" parameter:
   * Change all calls to dbget from: `xmldb_get(h, db, xpath, 0|1, &xret, msd)` to `xmldb_get(h, db, xpath, &xret, msd)`
 * Structural change: removed datastore plugin and directory, and merged into regular clixon lib code.
@@ -51,9 +61,8 @@
     * Change all y->ys_keyword to yang_keyword_get(y)
     * Change all y->ys_argument to yang_argument_get(y)
     * Change all y->ys_cv to yang_cv_get(y)
-    * Change all y->ys_cvec to yang_cvec_get(y)
+    * Change all y->ys_cvec to yang_cvec_get(y) or yang_cvec_set(y, cvv)
   * Removed external direct access to the yang_stmt struct.
-
 * xmldb_get() removed unnecessary config option:
   * Change all calls to dbget from: `xmldb_get(h, db, xpath, 0|1, &xret, msd)` to `xmldb_get(h, db, xpath, &xret, msd)`
 
@@ -101,11 +110,12 @@
 ```
 
 ### Minor changes
+
+* A new "hello world" example is added
 * Experimental customized error output strings, see [lib/clixon/clixon_err_string.h]
 * Empty leaf values, eg <a></a> are now checked at validation.
   * Empty values were skipped in validation.
   * They are now checked and invalid for ints, dec64, etc, but are treated as empty string "" for string types.
-* Optimized validation by making xml_diff work on raw cache tree (not copies)
 * Added syntactic check for yang status: current, deprecated or obsolete.
 * Added `xml_wrap` function that adds an XML node above a node as a wrapper
   * also renamed `xml_insert` to `xml_wrap_all`.
@@ -128,6 +138,8 @@
 * Added libgen.h for baseline()
 	
 ### Corrected Bugs
+* Failure in startup with -m startup or running left running_db cleared.
+  * Running-db should not be changed on failure. Unless failure-db defined. Or if SEGV, etc. In those cases, tmp_db should include the original running-db.
 * Backend plugin returning NULL was still installed - is now logged and skipped.
 * [Parent list key is not validated if not provided via RESTCONF #83](https://github.com/clicon/clixon/issues/83), thanks achernavin22.
 * [Invalid JSON if GET /operations via RESTCONF #82](https://github.com/clicon/clixon/issues/82), thanks achernavin22

README.md

@@ -8,6 +8,7 @@ support.
 
   * [Background](#background)
   * [Frequently asked questions (FAQ)](doc/FAQ.md)
+  * [Hello world](example/hello/README.md)
   * [Changelog](CHANGELOG.md)
   * [Installation](#installation)
   * [Licenses](#licenses)
@@ -26,6 +27,7 @@ support.
   * [Runtime](#runtime)
   * [Clixon project page](http://www.clicon.org)
   * [Tests and CI](test/README.md)
+  * [Scaling: large lists](doc/scaling/large-lists.md)
   * [Containers](docker/README.md)
   * [Roadmap](doc/ROADMAP.md)
   * [Reference manual](#reference) 

apps/backend/backend_commit.c

@@ -146,7 +146,7 @@ generic_validate(yang_stmt          *yspec,
  * and call application callback validations.
  * @param[in]  h       Clicon handle
  * @param[in]  db      The startup database. The wanted backend state
- * @param[out] xtr     Transformed XML
+ * @param[in]  td      Transaction
  * @param[out] cbret   CLIgen buffer w error stmt if retval = 0
  * @retval    -1       Error - or validation failed (but cbret not set)
  * @retval     0       Validation failed (with cbret set)
@@ -180,8 +180,13 @@ startup_common(clicon_handle       h,
     if (clicon_option_bool(h, "CLICON_XMLDB_MODSTATE"))
 	if ((msd = modstate_diff_new()) == NULL)
 	    goto done;
+    clicon_debug(1, "Reading startup config from %s", db);
     if (xmldb_get(h, db, "/", &xt, msd) < 0)
 	goto done;
+    if (xml_child_nr(xt) == 0){     /* If empty skip */
+	td->td_target = xt;
+	goto ok;
+    }
     if (msd){
 	if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0)
 	    goto done;
@@ -211,6 +216,7 @@ startup_common(clicon_handle       h,
 
     /* 5. Make generic validation on all new or changed data.
        Note this is only call that uses 3-values */
+    clicon_debug(1, "Validating startup %s", db);
     if ((ret = generic_validate(yspec, td, cbret)) < 0)
 	goto done;
     if (ret == 0)
@@ -223,6 +229,7 @@ startup_common(clicon_handle       h,
     /* 7. Call plugin transaction complete callbacks */
     if (plugin_transaction_complete(h, td) < 0)
 	goto done;
+ ok:
     retval = 1;
  done:
     if (msd)
@@ -282,6 +289,7 @@ startup_validate(clicon_handle  h,
  * @retval    -1       Error - or validation failed (but cbret not set)
  * @retval     0       Validation failed (with cbret set)
  * @retval     1       Validation OK       
+ * Only called from startup_mode_startup
  */
 int
 startup_commit(clicon_handle  h, 
@@ -292,6 +300,10 @@ startup_commit(clicon_handle  h,
     int                 ret;
     transaction_data_t *td = NULL;
 
+    if (strcmp(db,"running")==0){
+	clicon_err(OE_FATAL, 0, "Invalid startup db: %s", db);
+	goto done;
+    }
     /* Handcraft a transition with only target and add trees */
     if ((td = transaction_new()) == NULL)
 	goto done;
@@ -302,6 +314,13 @@ startup_commit(clicon_handle  h,
      /* 8. Call plugin transaction commit callbacks */
      if (plugin_transaction_commit(h, td) < 0)
 	 goto done;
+     /* [Delete and] create running db */
+     if (xmldb_exists(h, "running") == 1){
+	if (xmldb_delete(h, "running") != 0 && errno != ENOENT) 
+	    goto done;;
+    }
+    if (xmldb_create(h, "running") < 0)
+	goto done;
      /* 9, write (potentially modified) tree to running
       * XXX note here startup is copied to candidate, which may confuse everything
       */

apps/backend/backend_main.c

@@ -638,16 +638,27 @@ main(int    argc,
 	status = STARTUP_OK;
 	break;
     case SM_RUNNING: /* Use running as startup */
-	/* Copy original running to startup and treat as startup */
+	/* Copy original running to tmp as backup (restore if error) */
 	if (xmldb_copy(h, "running", "tmp") < 0)
 	    goto done;
 	ret = startup_mode_startup(h, "tmp", cbret);
+	/* If ret fails, copy tmp back to running */
+	if (ret != 1)
+	    if (xmldb_copy(h, "tmp", "running") < 0)
+		goto done;
 	if (ret2status(ret, &status) < 0)
 	    goto done;
 	break;
     case SM_STARTUP: 
+	/* Copy original running to tmp as backup (restore if error) */
+	if (xmldb_copy(h, "running", "tmp") < 0)
+	    goto done;
 	/* Load and commit from startup */
 	ret = startup_mode_startup(h, "startup", cbret);
+	/* If ret fails, copy tmp back to running */
+	if (ret != 1)
+	    if (xmldb_copy(h, "tmp", "running") < 0)
+		goto done;
 	if (ret2status(ret, &status) < 0)
 	    goto done;
 	/* if status = STARTUP_INVALID, cbret contains info */
@@ -683,8 +694,9 @@ main(int    argc,
     /* Call backend plugin_start with user -- options */
     if (clixon_plugin_start(h) < 0)
 	goto done;
+    /* -1 option to run only once */
     if (once)
-	goto done;
+	goto ok;
 
     /* Daemonize and initiate logging. Note error is initiated here to make
        demonized errors OK. Before this stage, errors are logged on stderr 
@@ -722,6 +734,7 @@ main(int    argc,
 	goto done;
     if (event_loop() < 0)
 	goto done;
+ ok:
     retval = 0;
   done:
     if (cbret)

apps/backend/backend_startup.c

@@ -114,6 +114,7 @@ db_merge(clicon_handle h,
 
 /*! Clixon startup startup mode: Commit startup configuration into running state
  * @param[in]  h       Clixon handle
+ * @param[in]  db      tmp or startup
  * @param[out] cbret   If status is invalid contains error message
  * @retval    -1       Error
  * @retval     0       Validation failed
@@ -150,9 +151,10 @@ startup_mode_startup(clicon_handle        h,
     int         retval = -1;
     int         ret;
     
-    /* [Delete and] create running db */
+    if (strcmp(db, "running")==0){
-    if (startup_db_reset(h, "running") < 0)
+	clicon_err(OE_FATAL, 0, "Invalid startup db: %s", db);
 	goto done;
+    }
     /* Load plugins and call plugin_init() */
     if (backend_plugin_initiate(h) != 0) 
 	goto done;
@@ -258,6 +260,8 @@ startup_extraxml(clicon_handle        h,
 	goto done;
     if (ret == 0)
 	goto fail;
+    if (xt==NULL || xml_child_nr(xt)==0) 
+	goto ok;
     /* Write (potentially modified) xml tree xt back to tmp
      */
     if ((ret = xmldb_put(h, "tmp", OP_REPLACE, xt,
@@ -268,6 +272,7 @@ startup_extraxml(clicon_handle        h,
 	goto fail;
     if (ret == 0)
 	goto fail;
+ ok:
     retval = 1;
  done:
     if (xt)
@@ -299,15 +304,22 @@ startup_failsafe(clicon_handle h)
 	clicon_err(OE_XML, errno, "cbuf_new");
 	goto done;
     }
-    if (startup_db_reset(h, "running") < 0)
-	goto done;
     if ((ret = xmldb_exists(h, db)) < 0)
 	goto done;
     if (ret == 0){ /* No it does not exist, fail */
 	clicon_err(OE_DB, 0, "Startup failed and no Failsafe database found, exiting");
 	goto done;
     }
-    if ((ret = candidate_commit(h, db, cbret)) < 0) /* diff */
+    /* Copy original running to tmp as backup (restore if error) */
+    if (xmldb_copy(h, "running", "tmp") < 0)
+	goto done;
+    if (startup_db_reset(h, "running") < 0)
+	goto done;
+    ret = candidate_commit(h, db, cbret);
+    if (ret != 1)
+	if (xmldb_copy(h, "tmp", "running") < 0)
+	    goto done;
+    if (ret < 0)
 	goto done;
     if (ret == 0){
 	clicon_err(OE_DB, 0, "Startup failed, Failsafe database validation failed %s", cbuf_get(cbret));

configure

@@ -4447,7 +4447,7 @@ _ACEOF
 
 
 
-ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile"
+ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile example/hello/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -5155,9 +5155,10 @@ do
     "etc/clixonrc") CONFIG_FILES="$CONFIG_FILES etc/clixonrc" ;;
     "example/Makefile") CONFIG_FILES="$CONFIG_FILES example/Makefile" ;;
     "example/main/Makefile") CONFIG_FILES="$CONFIG_FILES example/main/Makefile" ;;
+    "example/hello/Makefile") CONFIG_FILES="$CONFIG_FILES example/hello/Makefile" ;;
     "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;;
     "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;;
-    "docker/system/Makefile") CONFIG_FILES="$CONFIG_FILES docker/system/Makefile" ;;
+    "docker/main/Makefile") CONFIG_FILES="$CONFIG_FILES docker/main/Makefile" ;;
     "docker/base/Makefile") CONFIG_FILES="$CONFIG_FILES docker/base/Makefile" ;;
     "util/Makefile") CONFIG_FILES="$CONFIG_FILES util/Makefile" ;;
     "yang/Makefile") CONFIG_FILES="$CONFIG_FILES yang/Makefile" ;;

configure.ac

@@ -249,9 +249,10 @@ AC_OUTPUT(Makefile
 	  etc/clixonrc
 	  example/Makefile
   	  example/main/Makefile
+	  example/hello/Makefile	
 	  extras/rpm/Makefile
 	  docker/Makefile
-  	  docker/system/Makefile
+  	  docker/main/Makefile
     	  docker/base/Makefile
   	  util/Makefile 
 	  yang/Makefile

doc/FAQ.md

@@ -6,6 +6,7 @@
   * [Is Clixon extendible?](#is-clixon-extendible)
   * [Which programming language is used?](#which-programming-language-is-used)
   * [How to best understand Clixon?](#how-to-best-understand-clixon)
+  * [Hello world?](#hello-world)
   * [How do you build and install Clixon (and the example)?](how-do-you-build-and-install-clixon)
   * [How do I run Clixon example commands?](#how-do-i-run-clixon-example-commands)
   * [Do I need to setup anything? (IMPORTANT)](#do-i-need-to-setup-anything))
@@ -67,6 +68,10 @@ specification uses [CLIgen](http://github.com/olofhagsand/cligen)
 ## How to best understand Clixon?
 Run the Clixon example, in the [example](../example) directory.
 
+## Hello world?
+
+One of the examples is [a hello world example](../example/hello). Please start with that.
+
 ## How do you build and install Clixon?
 Clixon: 
 ```

doc/scaling/large-lists.md

@@ -0,0 +1,225 @@
+# Large lists in Clixon
+
+Olof Hagsand, 2019-04-17
+
+  * [1. Background](#1-background)
+  * [2. Overview](#2-overview)
+  * [3. Tests](#3-tests)
+  * [4. Results](#4-results)
+  * [5. Discussion](#5-discussion)
+  * [6. Future work](#6-future-work)
+  * [7. References](#7-references)
+
+## 1. Background
+
+Clixon can handle large configurations. Here, measurements using a
+large number of elements in a simple "flat" list is analysed. This
+includes starting up with alarge existing database; initializing an
+empty database with a large number of entries, accessing single
+entries with a large database, etc.
+
+In short, the results show a linear dependency on the number of
+entries. This is OK for startup scenarions, but single-enrty (transactional) operations need improvement.
+
+There are other scaling usecases, such as large configuratin "depth",
+large number of requesting clients, etc.
+
+Thanks to [Netgate](www.netgate.com) for supporting this work.
+
+## 2. Overview
+
+The basic case is a large list, according to the following Yang specification:
+```
+   container x {
+      description "top-level container";
+      list y {
+         description "List with potential large number of elements";
+         key "a";
+         leaf a {
+            description "key in list";
+            type int32;
+         }
+         leaf b {
+            description "payload data";
+            type string;
+         }
+      }
+   }
+```
+where `a` is a unique key and `b` is a payload, useful in replace operations.
+
+With this XML lists with `N` elements are generated based on
+this configuration, eg for `N=10`:
+```
+   <y><a>0</a><b>0</b></y>
+   <y><a>1</a><b>1</b></y>
+   <y><a>2</a><b>2</b></y>
+   <y><a>3</a><b>3</b></y>
+   <y><a>4</a><b>4</b></y>
+   <y><a>5</a><b>5</b></y>
+   <y><a>6</a><b>6</b></y>
+   <y><a>7</a><b>7</b></y>
+   <y><a>8</a><b>8</b></y>
+   <y><a>9</a><b>9</b></y>
+```
+
+Requests are either made over the _whole_ dataset, or for one specific element. The following example shows a Restconf GET operation of a single element:
+```
+   curl -X GET http://localhost/restconf/data/scaling:x/y=3
+   {"scaling:y": [{"a": 3,"b": "3"}]}
+
+```
+
+Operations of single elements (transactions) are made in a burst of
+random elements, typically 100. 
+
+
+## 3. Tests
+
+All details of the setup are in the [test script](../../test/plot_perf.sh).
+
+### Testcases
+
+All tests measure the "real" time of a command on a lightly loaded
+machine using the Linux command `time(1)`.
+
+The following tests were made (for each architecture and protocol):
+* Write `N` entries into the startup configuration. The clixon_backend was started with options `-1s startup`.
+* Write `N` entries in one single operation. (With an empty datastore)
+* Read `N` entries in one single operation. (With a datastore of `N` entries)
+* Commit `N` entries (With a candidate of `N` entries and empty running)
+* Read 1 entry (In a datastore of `N` entries)
+* Write/Replace 1 entry (In a datastore of `N` entries)
+* Delete 1 entry (In a datastore of `N` entries)
+
+The tests are made using Netconf and Restconf, except commit which is made only for Netconf and startup where protocol is irrelevant.
+
+### Architecture and OS
+
+The tests were made on the following hardware, all running Ubuntu Linux:
+
+#### i686
+
+* IBM Thinkpad X60
+* Dual Intel Core Duo processor 
+* Ubuntu 16.04.6 LTS
+* Linux version 4.4.0-143-generic (buildd@lgw01-amd64-037)
+  * gcc version 5.4.0 20160609 (Ubuntu 5.4.0-6ubuntu1~16.04.10)
+  * #169-Ubuntu SMP Thu Feb 7 07:56:51 UTC 2019
+
+#### ARM
+
+* Raspberry PI 2 Model B
+* ARMv7 Processor rev 5 (v7l) 
+* Raspbian GNU/Linux 9
+* Linux version 4.14.79-v7+ (dc4@dc4-XPS13-9333)
+  * gcc version 4.9.3 (crosstool-NG crosstool-ng-1.22.0-88-g8460611))
+  * #1159 SMP Sun Nov 4 17:50:20 GMT 2018
+
+#### x86_64
+
+* Intel NUC Coffee Lake
+* Intel Quad-core I5-8259U 
+* Ubuntu 18.04.1 LTS
+* Linux version 4.15.0-47-generic (buildd@lgw01-amd64-001)
+  * gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)
+  * #50-Ubuntu SMP Wed Mar 13 10:44:52 UTC 2019
+
+## 4. Results
+
+This section shows the results of the measurements as defined in [Tests](#tests).
+### Startup
+
+![Startup](clixon-startup.png "Startup")
+
+### Access of the whole datastore
+
+![Get config](clixon-get-0.png "Get config")
+
+![Put config](clixon-put-0.png "Put config")
+
+![Commit config](clixon-commit-0.png "Commit config")
+
+### Access of single entries
+
+![Get single entry](clixon-get-100.png "Get single entry")
+
+![Put single entry](clixon-put-100.png "Put single entry")
+
+![Delete single entry](clixon-delete-100.png "Delete single entry")
+
+### Profiling
+
+An example profiling of the most demanding case was made: Put single restconf for with 5000 existing entries.
+The tool used is valgrind/callgrind with the following approximate result (percentage of total cycles):
+* from_rpc_callback 100%
+  * from_client_commit 65%
+    * candidate_commit 65%
+      * from_validate_common 30%
+        * xml_diff 13%
+	* xml_yang_validate_all 10%
+      * xmldb_copy 29%
+        * xml_copy 22%
+  * from_client_edit_config 35%
+    * xmldb_put 35%
+      * clicon_xml2file 30%
+        * fprintf 13%
+        * xml_chardata_encode 12%
+
+It can be seen that most cycles are spend in file copying and writing
+the existing datastore to file. This explains the linear behaviour, ie
+the larger existing datastore, the larger number of cycles spent.
+
+Why is the existing datastore accessed in this way? When a small PUT request is received, several things happen:
+* The existing candidate database is modified with the change and written to disk. (35%)
+* The difference between candidate and running is computed (13%)
+* The new candidate is validated (10%)
+* The candidate db is copied to running (29%)
+
+## 5. Discussion
+
+All measurements show clear performance differences between the
+architectures, which was expected.
+
+By looking at top and other tools, it seems clear that the main
+bootleneck is CPU for the `clixon_backend`. The clients, eg `nginx`,
+clixon_restconf` and `clixon_netconf` seem negligable. Memory
+footprint is also not limiting.
+
+Accessing the whole configuration is similar between protocols and
+linear in time. This is to be expected since the tranfer is dependent
+on the size of the database.
+
+Accessing a single entry is also similar in all cases and shows large
+differences. As expected, the CPU architecture is significant, but
+there is also a large difference between Netconf and Restconf.
+
+The Netconf and restconf setups differ somewhat which may explain
+differences in performance. Primarily, the Netconf input is piped to a
+single Netconf client while a curl is re-started for each Restconf
+call. Also, the Restconf PUT includes a commit.
+
+Further, the single entry access is linear wrt number of entries. This
+means it is possible to run large scale applications on high
+performance CPUs, such as 100K entries on a x86_64 in the results, but
+it also means that very large lists may not be supported, and that the
+system degrades with the size of the lists.
+
+Examining the profiling of the most demanding Restconf PUT case, most
+cycles are spent on handling writing and copying the existing datastore.
+
+Note that the experiments here contains _very_ simple
+data-structures. A more realistic complex example will require more
+CPU effort. Ad-hoc measurement of a more complex datastructure,
+generated four times the duration of the simple yang model in this work.
+
+## 6. Future work
+
+* Improve access of individual elements to sub-linear performance.
+* CLI access on large lists (not included in this study)
+
+## 7. References
+
+* [RFC6241](https://tools.ietf.org/html/rfc6241) "Network Configuration Protocol (NETCONF)"
+* [RFC8040](https://tools.ietf.org/html/rfc8040) "RESTCONF Protocol"
+* [plot_perf.sh](../../test/plot_perf.sh) Test script

doc/startup.md

@@ -254,7 +254,8 @@ When the startup process is completed, a startup status is set and is accessible
 
 If the startup fails, the backend looks for a `failsafe` configuration
 in `CLICON_XMLDB_DIR/failsafe_db`. If such a config is not found, the
-backend terminates.
+backend terminates. In this mode, running and startup mode should be
+unchanged.
 
 If the failsafe is found, the failsafe config is loaded and
 committed into the running db.
@@ -405,6 +406,7 @@ running   |--------+------------> GOTO EXTRA XML
 
 ### Running mode
 
+On failure, running is restored to initial state
 ```
 running   ----+                   |----------+--------> GOTO EXTRA XML
                \ copy   parse  validate OK  / commit 
@@ -420,7 +422,7 @@ running                         |--------+------------> GOTO EXTRA XML
 startup -------+--+-------+------------+          
 ```
 
-### Failure
+### Failure if failsafe
 ```
 failsafe      ----------------------+
                             reset    \ commit

docker/Makefile.in

@@ -41,7 +41,7 @@ LIBS    	= @LIBS@
 SHELL	= /bin/sh
 
 SUBDIRS  = base
-SUBDIRS  += system
+SUBDIRS  += main
 #SUBDIRS += cluster
 
 .PHONY: all clean distclean depend install-include install uninstall test $(SUBDIRS)

docker/README.md

@@ -2,6 +2,6 @@
 
 This directory contains sub-directories with examples of Clixon docker images:
 
-  * [base](base/README.md)      Clixon base image 
+  * [base](base/README.md)  Clixon base image 
-  * [system](system/README.md)  Example and test application 
+  * [main](main/README.md)  Main example and test application 
 

docker/base/README.md

@@ -8,7 +8,7 @@ The clixon docker base image can be used to build clixon
 applications. It has all the whole code for a clixon release which it
 downloads from git.
 
-See [clixon-system](../system/README.md) for a more complete clixon image.
+See [clixon-system](../main/README.md) for a more complete clixon image.
 
 ## Build and push
 
@@ -20,7 +20,7 @@ You may also do `make push` if you want to push the image, but you may then cons
 
 ## Example run
 
-The base container is a minimal and primitive example. Look at the [clixon-system](../system) for a more stream-lined application.
+The base container is a minimal and primitive example. Look at the [clixon-system](../main) for a more stream-lined application.
 
 The following shows a simple example of how to run the example
 application. First, the container is started with the backend running:

docker/main/README.md

@@ -54,3 +54,10 @@ To check status and then kill it:
 ```
 
 You trigger the test scripts inside the container using `make test`.
+
+## Changing code
+
+If you want to edit clixon code so it runs in the container?
+You either
+(1) "persistent": make your changes in the actual clixon code and commit; make clean to remove the local clone;  make test again
+(2) "volatile" edit the local clone; make test.

example/Makefile.in

@@ -40,7 +40,7 @@ LIBS    	= @LIBS@
 
 SHELL	= /bin/sh
 
-SUBDIRS  = main
+SUBDIRS  = main hello
 
 .PHONY: all clean depend install $(SUBDIRS)
 

example/README.md

@@ -1,4 +1,5 @@
 # Clixon examples
 
 Clixon have the following examples:
+  * [Hello world](hello/README.md)
   * [Main example](main/README.md)

example/hello/Makefile.in

@@ -0,0 +1,167 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# 
+# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
+#
+# This file is part of CLIXON
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 3 or later (the "GPL"),
+# in which case the provisions of the GPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of the GPL, and not to allow others to
+# use your version of this file under the terms of Apache License version 2, 
+# indicate your decision by deleting the provisions above and replace them with
+# the notice and other provisions required by the GPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the Apache License version 2 or the GPL.
+#
+# ***** END LICENSE BLOCK *****
+#
+VPATH       	= @srcdir@
+srcdir  	= @srcdir@
+top_srcdir  	= @top_srcdir@
+prefix 		= @prefix@
+bindir 		= @bindir@
+includedir	= @includedir@
+datarootdir     = @datarootdir@
+sysconfdir      = @sysconfdir@
+datarootdir	= @datarootdir@
+localstatedir   = @localstatedir@
+libdir          = @exec_prefix@/lib
+
+APPNAME	        = hello
+
+# Here is where example yang appears
+CLIXON_DATADIR  = @CLIXON_DATADIR@
+# Install here if you want default clixon location:
+CLIXON_DEFAULT_CONFIG = @CLIXON_DEFAULT_CONFIG@
+
+CC		= @CC@
+CFLAGS  	= @CFLAGS@ -rdynamic -fPIC 
+INSTALLFLAGS  	= @INSTALLFLAGS@ 
+with_restconf	= @with_restconf@
+
+INCLUDES 	= -I$(includedir) @INCLUDES@ 
+CPPFLAGS  	= @CPPFLAGS@ -fPIC
+
+BE_PLUGIN       = $(APPNAME)_backend.so
+CLI_PLUGIN      = $(APPNAME)_cli.so
+NETCONF_PLUGIN  = $(APPNAME)_netconf.so
+RESTCONF_PLUGIN = $(APPNAME)_restconf.so
+
+PLUGINS = $(BE_PLUGIN) $(CLI_PLUGIN) $(NETCONF_PLUGIN)
+ifeq ($(with_restconf),yes)
+PLUGINS += $(RESTCONF_PLUGIN)
+endif
+
+.PHONY: all clean depend install 
+
+all:	$(PLUGINS)
+
+.SUFFIXES: .c .o
+
+# implicit rule
+.c.o:
+	$(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -c $<
+
+CLISPECS = $(APPNAME)_cli.cli
+
+YANGSPECS	 = clixon-hello@2019-04-17.yang
+
+# CLI frontend plugin (see also install rule)
+#CLI_SRC  = $(APPNAME)_cli.c
+CLI_OBJ  = $(CLI_SRC:%.c=%.o)
+$(CLI_PLUGIN): $(CLI_OBJ)
+	$(CC) -Wall -shared -o $@ -lc $^ 
+
+# Backend plugin (see also install rule)
+#BE_SRC   = $(APPNAME)_backend.c
+BE_OBJ   = $(BE_SRC:%.c=%.o)
+$(BE_PLUGIN): $(BE_OBJ)
+	$(CC) -Wall -shared -o $@ -lc $<
+
+# NETCONF frontend plugin (see also install rule)
+#NETCONF_SRC  = $(APPNAME)_netconf.c
+NETCONF_OBJ  = $(NETCONF_SRC:%.c=%.o)
+$(NETCONF_PLUGIN): $(NETCONF_OBJ)
+	$(CC) -Wall -shared -o $@ -lc $^
+
+# See configure.ac for disabling restconf
+# RESTCONF frontend plugin (see also install rule)
+#RESTCONF_SRC  = $(APPNAME)_restconf.c
+RESTCONF_OBJ  = $(RESTCONF_SRC:%.c=%.o)
+$(RESTCONF_PLUGIN): $(RESTCONF_OBJ)
+	$(CC) -Wall -shared -o $@ -lc $^ 
+
+SRC     = $(BE_SRC) $(CLI_SRC) $(NETCONF_SRC)
+SRC    +=  $(RESTCONF_SRC)
+
+OBJS    = $(BE_OBJ) $(CLI_OBJ) $(NETCONF_OBJ)
+OBJS   += $(RESTCONF_OBJ)
+
+clean:
+	rm -f $(PLUGINS) $(OBJS) 
+
+distclean: clean
+	rm -f Makefile *~ .depend
+
+
+install: $(YANGSPECS) $(CLISPECS) $(PLUGINS) $(APPNAME).xml
+	install -m 0644 $(APPNAME).xml $(DESTDIR)$(CLIXON_DEFAULT_CONFIG)
+	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)
+	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/clispec
+	install -m 0644 $(CLISPECS) $(DESTDIR)$(libdir)/$(APPNAME)/clispec
+	install -d -m 0755 $(DESTDIR)$(datarootdir)/$(APPNAME)/yang
+	install -m 0644 $(YANGSPECS) $(DESTDIR)$(DESTDIR)$(CLIXON_DATADIR)
+	install -d -m 0755 $(DESTDIR)$(localstatedir)/$(APPNAME)
+
+# Uncomment for installing config file in /usr/local/etc instead
+#	install -d -m 0755 $(DESTDIR)$(sysconfdir)
+#	install -m 0644 $(APPNAME).xml $(DESTDIR)$(sysconfdir)
+
+# Uncomment for installing cli plugin (see CLI_SRC)
+#	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/cli
+#	install -m 0644 $(INSTALLFLAGS) $(CLI_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/cli
+
+# Uncomment for installing backend plugin (see BE_SRC)
+#	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/backend
+#	install -m 0644 $(INSTALLFLAGS) $(BE_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/backend
+
+# Uncomment for installing netconf plugin (see NETCONF_SRC)
+#	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/netconf
+#	install -m 0644 $(INSTALLFLAGS) $(NETCONF_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/netconf
+
+# Uncomment for installing restconf plugin (see RESTCONF_SRC). The conditional is
+# because it is an option to disable restconf
+#ifeq ($(with_restconf),yes)
+#	install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/restconf
+#	install -m 0644 $(INSTALLFLAGS) $(RESTCONF_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/restconf
+#endif
+
+uninstall: 
+	rm -rf $(DESTDIR)$(CLIXON_DEFAULT_CONFIG)
+	rm -rf $(DESTDIR)$(libdir)/$(APPNAME)
+	rm -rf $(DESTDIR)$(localstatedir)/$(APPNAME)
+	rm -rf $(DESTDIR)$(datarootdir)/$(APPNAME)
+
+
+install-include:	
+
+depend:
+	$(CC) $(DEPENDFLAGS) $(INCLUDES) $(CFLAGS) -MM $(SRC) > .depend
+
+#include .depend
+

example/hello/README.md

@@ -0,0 +1,111 @@
+# Clixon hello world example
+
+  * [Content](#content)
+  * [Compile and run](#compile)
+  * [Using the CLI](#using-the-cli)
+  * [Netconf](#netconf)	
+  * [Restconf](#restconf)
+  * [Next steps](#next-steps)
+  
+## Content
+
+This directory contains a Clixon example which includes a simple example. It contains the following files:
+* `hello.xml`       The configuration file. See [yang/clixon-config@<date>.yang](../../yang/clixon-config@2019-03-05.yang) for the documentation of all available fields.
+* `clixon-hello@2019-04-17.yang` The yang spec of the example.
+* `hello_cli.cli`                CLIgen specification.
+* `Makefile.in`                  Example makefile where plugins are built and installed
+* `README.md`                    This file
+
+
+## Compile and run
+
+Before you start,
+* Make [group setup](../../doc/FAQ.md#do-i-need-to-setup-anything-important)
+
+```
+    make && sudo make install
+```
+Start backend in the background:
+```
+    sudo clixon_backend
+```
+Start cli:
+```
+    clixon_cli
+```
+
+## Using the CLI
+
+The example CLI allows you to modify and view the data model using `set`, `delete` and `show` via generated code.
+
+The following example shows how to add a very simple configuration `hello world` using the generated CLI. The config is added to the candidate database, shown, committed to running, and then deleted.
+```
+   olof@vandal> clixon_cli
+   cli> set <?>
+     hello                 
+   cli> set hello world 
+   cli> show configuration 
+   hello world;
+   cli> commit 
+   cli> delete <?>
+     all                   Delete whole candidate configuration
+     hello                 
+   cli> delete hello 
+   cli> show configuration 
+   cli> commit 
+   cli> quit
+   olof@vandal> 
+```
+
+## Netconf
+
+Clixon also provides a Netconf interface. The following example starts a netconf client form the shell, adds the hello world config, commits it, and shows it:
+```
+   olof@vandal> clixon_netconf -q
+   <rpc><edit-config><target><candidate/></target><config><hello xmlns="urn:example:hello"><world/></hello></config></edit-config></rpc>]]>]]>
+   <rpc-reply><ok/></rpc-reply>]]>]]>
+   <rpc><commit/></rpc>]]>]]>
+   <rpc-reply><ok/></rpc-reply>]]>]]>
+   <rpc><get-config><source><running/></source></get-config></rpc>]]>]]>
+   <rpc-reply><data><hello xmlns="urn:example:hello"><world/></hello></data></rpc-reply>]]>]]>
+olof@vandal> 
+```
+
+## Restconf
+
+Clixon also provides a Restconf interface. A reverse proxy needs to be configured. There are [instructions how to setup Nginx](../../doc/FAQ.md#how-do-i-use-restconf) for Clixon.
+
+Start restconf daemon
+```
+   sudo su -c "/www-data/clixon_restconf" -s /bin/sh www-data &
+```
+
+Start sending restconf commands (using Curl):
+```
+   olof@vandal> curl -X POST http://localhost/restconf/data -d '{"clixon-hello:hello":{"world":null}}'
+   olof@vandal> curl -X GET http://localhost/restconf/data 
+   {
+     "data": {
+       "clixon-hello:hello": {
+         "world": null
+       }
+     }
+   }
+```
+
+## Next steps
+
+The hello world example only has a Yang spec and a template CLI
+spec. For more advanced applications, customized backend, cli, netconf
+and restconf code callbacks becomes necessary.
+
+Further, you may want to add upgrade, RPC:s, state data, notification
+streams, authentication and authorization. The [main example](../main)
+contains examples for such capabilities.
+
+There are also [container examples](../../docker) and lots more.
+
+
+
+
+

example/hello/clixon-hello@2019-04-17.yang

@@ -0,0 +1,14 @@
+module clixon-hello {
+    yang-version 1.1;
+    namespace "urn:example:hello";
+    prefix he;
+    revision 2019-04-17 {
+	description
+	    "Clixon hello world example";
+    }
+    container hello{
+	container world{
+	    presence true;
+	}
+    }
+}

example/hello/hello.xml

@@ -0,0 +1,13 @@
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>/usr/local/etc/example.xml</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>*:*</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>clixon-hello</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLI_MODE>hello</CLICON_CLI_MODE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/hello/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_SOCK>/usr/local/var/hello.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/hello.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/hello</CLICON_XMLDB_DIR>
+  <CLICON_STARTUP_MODE>init</CLICON_STARTUP_MODE>
+  <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
+</clixon-config>

example/hello/hello_cli.cli

@@ -0,0 +1,54 @@
+# Common CLI syntax for both server and PMNode operatio mode
+CLICON_MODE="hello";
+CLICON_PROMPT="cli> ";
+
+# Reference generated data model
+set @datamodel, cli_set();
+merge @datamodel, cli_merge();
+create @datamodel, cli_create();
+delete("Delete a configuration item") @datamodel, cli_del();
+
+validate("Validate changes"), cli_validate();
+commit("Commit the changes"), cli_commit();
+quit("Quit"), cli_quit();
+delete("Delete a configuration item")  all("Delete whole candidate configuration"), delete_all("candidate");
+
+startup("Store running as startup config"), db_copy("running", "startup");
+no("Negate or remove") debug("Debugging parts of the system"), cli_debug_cli((int32)0);
+debug("Debugging parts of the system"), cli_debug_cli((int32)1);{
+    level("Set debug level: 1..n") <level:int32>("Set debug level (0..n)"), cli_debug_backend();
+}
+discard("Discard edits (rollback 0)"), discard_changes();
+compare("Compare running and candidate"), compare_dbs((int32)1);
+
+show("Show a particular state of the system"){
+    xpath("Show configuration") <xpath:string>("XPATH expression"), show_conf_xpath("candidate");
+    version("Show version"), cli_show_version("candidate", "text", "/");
+    compare("Compare candidate and running databases"), compare_dbs((int32)0);{
+    		     xml("Show comparison in xml"), compare_dbs((int32)0);
+		     text("Show comparison in text"), compare_dbs((int32)1);
+    }
+    configuration("Show configuration"), cli_show_config("candidate", "text", "/");{
+	    xml("Show configuration as XML"), cli_show_config("candidate", "xml", "/");{
+	        @datamodel, cli_show_auto("candidate", "xml");		   
+	    }
+	    cli("Show configuration as CLI commands"), cli_show_config("candidate", "cli", "/");{
+	        @datamodel, cli_show_auto("candidate", "cli");
+	    }
+	    netconf("Show configuration as netconf edit-config operation"), cli_show_config("candidate", "netconf", "/");{
+	        @datamodel, cli_show_auto("candidate", "netconf");
+	    }
+	    text("Show configuration as text"), cli_show_config("candidate","text","/");{
+	        @datamodel, cli_show_auto("candidate", "text");
+	    }
+	    json("Show configuration as JSON"), cli_show_config("candidate", "json", "/");{
+	    	@datamodel, cli_show_auto("candidate", "json");
+	    }
+    }
+}
+
+save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename");
+load("Load configuration from XML file") <filename:string>("Filename (local filename)"),load_config_file("filename", "replace");{
+    replace("Replace candidate with file contents"), load_config_file("filename", "replace");
+    merge("Merge file with existent candidate"), load_config_file("filename", "merge");
+}

example/main/README.md

@@ -15,7 +15,7 @@
 ## Content
 
 This directory contains a Clixon example which includes a simple example. It contains the following files:
-* `example.xml`       The configuration file. See [yang/clixon-config@<date>.yang](../../yang/clixon-config@2018-10-21.yang) for the documentation of all available fields.
+* `example.xml`       The configuration file. See [yang/clixon-config@<date>.yang](../../yang/clixon-config@2019-03-05.yang) for the documentation of all available fields.
 * `clixon-example@2019-01-13.yang`      The yang spec of the example.
 * `example_cli.cli`   CLIgen specification.
 * `example_cli.c`     CLI callback plugin containing functions called in the cli file above: a generic callback (`mycallback`) and an example RPC call (`example_client_rpc`).

include/clixon_custom.h

@@ -41,4 +41,6 @@
  */
 #undef RPC_USERNAME_ASSERT
 
-
+/* Use new xml_insert code on sorted xml lists
+ */
+#define USE_XML_INSERT

lib/clixon/clixon_string.h

@@ -86,6 +86,7 @@ int xml_chardata_encode(char **escp, char *fmt, ...);
 int uri_percent_decode(char *enc, char **str);
 const char *clicon_int2str(const map_str2int *mstab, int i);
 int clicon_str2int(const map_str2int *mstab, char *str);
+int clicon_str2int_search(const map_str2int *mstab, char *str, int upper);
 int nodeid_split(char *nodeid,	char **prefix, char **id);
 char *clixon_trim(char *str);
 int regexp_xsd2posix(char *xsd, char **posix);

lib/clixon/clixon_xml.h

@@ -118,8 +118,9 @@ cxobj    *xml_child_i_type(cxobj *xn, int i, enum cxobj_type type);
 cxobj    *xml_child_i_set(cxobj *xt, int i, cxobj *xc);
 cxobj    *xml_child_each(cxobj *xparent, cxobj *xprev,  enum cxobj_type type);
 
-cxobj   **xml_childvec_get(cxobj *x);
+int       xml_child_insert_pos(cxobj *x, cxobj *xc, int i);
 int       xml_childvec_set(cxobj *x, int len);
+cxobj   **xml_childvec_get(cxobj *x);
 cxobj    *xml_new(char *name, cxobj *xn_parent, yang_stmt *spec);
 yang_stmt *xml_spec(cxobj *x);
 int       xml_spec_set(cxobj *x, yang_stmt *spec);
@@ -130,7 +131,6 @@ cxobj    *xml_find(cxobj *xn_parent, char *name);
 int       xml_addsub(cxobj *xp, cxobj *xc);
 cxobj    *xml_wrap_all(cxobj *xp, char *tag);
 cxobj    *xml_wrap(cxobj *xc, char *tag);
-#define xml_insert(x,t) xml_wrap_all((x),(t))
 int       xml_purge(cxobj *xc);
 int       xml_child_rm(cxobj *xp, int i);
 int       xml_rm(cxobj *xc);

lib/clixon/clixon_xml_sort.h

@@ -40,8 +40,10 @@
  * Prototypes
  */
 int xml_child_spec(cxobj *x, cxobj *xp, yang_stmt *yspec, yang_stmt **yp);
+int xml_cmp(cxobj *x1, cxobj *x2, int enm);
 int xml_sort(cxobj *x0, void *arg);
-int    xml_sort_verify(cxobj *x, void *arg);
+int xml_insert(cxobj *xp, cxobj *xc);
-int    match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp);
+int xml_sort_verify(cxobj *x, void *arg);
+int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp);
 
 #endif /* _CLIXON_XML_SORT_H */

lib/clixon/clixon_yang.h

@@ -161,6 +161,9 @@ enum rfc_6020 yang_keyword_get(yang_stmt *ys);
 char      *yang_argument_get(yang_stmt *ys);
 cg_var    *yang_cv_get(yang_stmt *ys);
 cvec      *yang_cvec_get(yang_stmt *ys);
+int        yang_cvec_set(yang_stmt *ys, cvec *cvv);
+void      *yang_regex_cache_get(yang_stmt *ys);
+int        yang_regex_cache_set(yang_stmt *ys, void *regex);
 
 /* Other functions */
 yang_stmt *yspec_new(void);

lib/src/clixon_datastore.c

@@ -188,14 +188,15 @@ xmldb_copy(clicon_handle h,
     int                 retval = -1;
     char               *fromfile = NULL;
     char               *tofile = NULL;
-    db_elmnt           *de1 = NULL;
+    db_elmnt           *de1 = NULL; /* from */
-    db_elmnt           *de2 = NULL;
+    db_elmnt           *de2 = NULL; /* to */
     db_elmnt            de0 = {0,};
-    cxobj              *x1 = NULL;
+    cxobj              *x1 = NULL;  /* from */
-    cxobj              *x2 = NULL;
+    cxobj              *x2 = NULL;  /* to */
 		
     /* XXX lock */
     if (clicon_option_bool(h, "CLICON_XMLDB_CACHE")){
+	/* Copy in-memory cache */
 	/* 1. "to" xml tree in x1 */
 	if ((de1 = clicon_db_elmnt_get(h, from)) != NULL)
 	    x1 = de1->de_xml;
@@ -208,7 +209,7 @@ xmldb_copy(clicon_handle h,
 	    xml_free(x2);
 	    x2 = NULL;
 	}
-	else  if (x2 == NULL){ /* create x2 and copy x1 to it */
+	else  if (x2 == NULL){ /* create x2 and copy from x1 */
 	    if ((x2 = xml_new(xml_name(x1), NULL, xml_spec(x1))) == NULL)
 		goto done;
 	    if (xml_copy(x1, x2) < 0) 
@@ -221,12 +222,13 @@ xmldb_copy(clicon_handle h,
 	    if (xml_copy(x1, x2) < 0) 
 		goto done;
 	}
-	if (x1 || x2){
+	/* always set cache although not strictly necessary in case 1
-	    if (de2)
+	 * above, but logic gets complicated due to differences with
-		de0 = *de2;
+	 * de and de->de_xml */
-	    de0.de_xml = x2; /* The new tree */
+	if (de2)
-	    clicon_db_elmnt_set(h, to, &de0);
+	    de0 = *de2;
-	}
+	de0.de_xml = x2; /* The new tree */
+	clicon_db_elmnt_set(h, to, &de0);
     }
     /* Copy the files themselves (above only in-memory cache) */
     if (xmldb_db2file(h, from, &fromfile) < 0)

lib/src/clixon_datastore_write.c

@@ -129,8 +129,6 @@ text_modify(clicon_handle       h,
     int        ret;
     int        changed = 0; /* Only if x0p's children have changed-> sort is necessary */
     
-    assert(x1 && xml_type(x1) == CX_ELMNT);
-    assert(y0);
     /* Check for operations embedded in tree according to netconf */
     if ((opstr = xml_find_value(x1, "operation")) != NULL)
 	if (xml_operation(opstr, &op) < 0)
@@ -157,8 +155,16 @@ text_modify(clicon_handle       h,
 		    permit = 1;
 		}
 		//		int iamkey=0;
+
+#ifdef USE_XML_INSERT
+		/* Add new xml node but without parent - insert when node fully
+		   copied (see changed conditional below) */
+		if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL)
+		    goto done;
+#else
 		if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		    goto done;		
+#endif
 		changed++;
 
 		/* Copy xmlns attributes  */
@@ -204,6 +210,12 @@ text_modify(clicon_handle       h,
 		    }
 		}
 	    }
+#ifdef USE_XML_INSERT
+	    if (changed){
+		if (xml_insert(x0p, x0) < 0)
+		    goto done;
+	    }
+#endif
 	    break;
 	case OP_DELETE:
 	    if (x0==NULL){
@@ -283,8 +295,15 @@ text_modify(clicon_handle       h,
 			goto fail;
 		    permit = 1;
 		}
+#ifdef USE_XML_INSERT
+		/* Add new xml node but without parent - insert when node fully
+		   copied (see changed conditional below) */
+		if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL)
+		    goto done;
+#else
 		if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
 		    goto done;
+#endif
 		changed++;
 		/* Copy xmlns attributes  */
 		x1a = NULL;
@@ -346,6 +365,12 @@ text_modify(clicon_handle       h,
 		if (ret == 0)
 		    goto fail;
 	    }
+#ifdef USE_XML_INSERT
+	    if (changed){
+		if (xml_insert(x0p, x0) < 0)
+		    goto done;
+	    }
+#endif
 	    break;
 	case OP_DELETE:
 	    if (x0==NULL){
@@ -363,15 +388,16 @@ text_modify(clicon_handle       h,
 		}
 		if (xml_purge(x0) < 0)
 		    goto done;
-		changed++;
 	    }
 	    break;
 	default:
 	    break;
 	} /* CONTAINER switch op */
     } /* else Y_CONTAINER  */
+#ifndef USE_XML_INSERT
     if (changed)
 	xml_sort(x0p, NULL);
+#endif
     retval = 1;
  done:
     if (x0vec)
@@ -418,8 +444,8 @@ text_modify_top(clicon_handle       h,
     int        ret;
 
     /* Assure top-levels are 'config' */
-    assert(x0 && strcmp(xml_name(x0),"config")==0);
+    //    assert(x0 && strcmp(xml_name(x0),"config")==0);
-    assert(x1 && strcmp(xml_name(x1),"config")==0);
+    //    assert(x1 && strcmp(xml_name(x1),"config")==0);
 
     /* Check for operations embedded in tree according to netconf */
     if ((opstr = xml_find_value(x1, "operation")) != NULL)
@@ -584,11 +610,11 @@ xml_container_presence(cxobj  *x,
  */
 int
 xmldb_put(clicon_handle       h,
-	 const char         *db, 
+	  const char         *db, 
-	 enum operation_type op,
+	  enum operation_type op,
-	 cxobj              *x1,
+	  cxobj              *x1,
-	 char               *username,
+	  char               *username,
-    	 cbuf               *cbret)
+	  cbuf               *cbret)
 {
     int                 retval = -1;
     char               *dbfile = NULL;

lib/src/clixon_hash.c

@@ -112,6 +112,7 @@ hash_bucket(const char *str)
  *
  * @retval  hash  Pointer to new hash table.
  * @retval  NULL  Error
+ * @see hash_free  For freeing the hash-table
  */
 clicon_hash_t *
 hash_init(void)

lib/src/clixon_json.c

@@ -915,79 +915,4 @@ json_parse_file(int        fd,
     return retval;    
 }
 
-/*
- * Turn this on to get a json parse and pretty print test program
- * Usage: json
- * read json from input
- * Example compile:
- gcc -g -o json -I. -I../clixon ./clixon_json.c -lclixon -lcligen
- * Example run:
-    echo '{"foo": -23}' | ./json
-*/
-#if 0 /* Test program */
-
-static int
-usage(char *argv0)
-{
-    fprintf(stderr, "usage:%s.\n\tInput on stdin\n", argv0);
-    exit(0);
-}
-
-int
-main(int    argc,
-     char **argv)
-{
-    cxobj *xt;
-    cxobj *xc;
-    cbuf  *cb = cbuf_new();
-    char  *buf = NULL;
-    int    i;
-    int    c;
-    int    len;
-    FILE  *f = stdin;
-    
-    if (argc != 1){
-	usage(argv[0]);
-	return 0;
-    }
-    clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR); 
-    len = 1024; /* any number is fine */
-    if ((buf = malloc(len)) == NULL){
-	perror("malloc");
-	return -1;
-    }
-    memset(buf, 0, len);
-
-    i = 0; /* position in buf */
-    while (1){ /* read the whole file */
-	if ((c =  fgetc(f)) == EOF)
-	    break;
-	if (len==i){
-	    if ((buf = realloc(buf, 2*len)) == NULL){
-		fprintf(stderr, "%s: realloc: %s\n", __FUNCTION__, strerror(errno));
-		goto done;
-	    }	    
-	    memset(buf+len, 0, len);
-	    len *= 2;
-	}
-	buf[i++] = (char)(c&0xff);
-    } /* read a line */
-
-    if (json_parse_str(buf, &xt) < 0)
-	return -1;
-    xc = NULL;
-    while ((xc = xml_child_each(xt, xc, -1)) != NULL) {
-	xmltree2cbuf(cb, xc, 0);       /* dump data structures */
-		//clicon_xml2cbuf(cb, xc, 0, 1); /* print xml */
-    }
-    fprintf(stdout, "%s", cbuf_get(cb));
-    if (xt)
-	xml_free(xt);
-    if (cb)
-	cbuf_free(cb);
- done:
-    return 0;
-}
-
-#endif /* Test program */
 

lib/src/clixon_string.c

@@ -552,7 +552,7 @@ clicon_int2str(const map_str2int *mstab,
  * @param[in] str  Input string
  * @retval    int  Value
  * @retval   -1    Error, not found
- * @note linear search
+ * @see clicon_str2int_search for optimized lookup, but strings must be sorted
  */
 int
 clicon_str2int(const map_str2int *mstab, 
@@ -566,6 +566,65 @@ clicon_str2int(const map_str2int *mstab,
     return -1;
 }
 
+/*! Map from string to int using binary (alphatical) search
+ * @param[in]  ms    String, integer map
+ * @param[in]  str   Input string
+ * @param[in]  low   Lower bound index
+ * @param[in]  upper Upper bound index
+ * @param[in]  len   Length of array (max)
+ * @param[out] found Integer found (can also be negative)
+ * @retval    0      Not found
+ * @retval    1      Found with "found" value set.
+ * @note Assumes sorted strings, tree search
+ */
+static int
+str2int_search1(const map_str2int *mstab, 
+		char              *str,
+		int                low,
+		int                upper,
+		int                len,
+		int               *found)
+{
+    const struct map_str2int *ms;
+    int                       mid;
+    int                       cmp;
+
+    if (upper < low)
+	return 0; /* not found */
+    mid = (low + upper) / 2;
+    if (mid >= len)  /* beyond range */
+	return 0; /* not found */
+    ms = &mstab[mid];
+    if ((cmp = strcmp(str, ms->ms_str)) == 0){
+	*found = ms->ms_int;
+	return 1; /* found */
+    }
+    else if (cmp < 0)
+	return str2int_search1(mstab, str, low, mid-1, len, found);
+    else
+	return str2int_search1(mstab, str, mid+1, upper, len, found);
+}
+
+/*! Map from string to int using str2int map
+ * @param[in] ms   String, integer map
+ * @param[in] str  Input string
+ * @retval    int  Value
+ * @retval   -1    Error, not found
+ * @note Assumes sorted strings, tree search
+ * @note -1 can not be value
+ */
+int
+clicon_str2int_search(const map_str2int *mstab, 
+		      char              *str,
+		      int                len)
+{
+    int found;
+
+    if (str2int_search1(mstab, str, 0, len, len, &found)) 
+	return found;
+    return -1; /* not found */
+}
+
 /*! Split colon-separated node identifier into prefix and name
  * @param[in]  node-id
  * @param[out] prefix  Malloced string. May be NULL.

lib/src/clixon_xml.c

@@ -127,6 +127,7 @@ struct xml{
 				       reference, dont free */
     cg_var           *x_cv;         /* Cached value as cligen variable 
                                        (eg xml_cmp) */
+    char             *x_ns_cache;   /* Cached namespace */
     int              _x_vector_i;   /* internal use: xml_child_each */
     int              _x_i;          /* internal use for sorting: 
 				       see xml_enumerate and xml_cmp */
@@ -234,6 +235,7 @@ xml_prefix_set(cxobj *xn,
  * @retval     0          OK
  * @retval    -1          Error
  * @see xmlns_check XXX can these be merged?
+ * @note, this function uses a cache. Any case where cache should be cleared?
  */
 int
 xml2ns(cxobj *x,
@@ -241,9 +243,11 @@ xml2ns(cxobj *x,
        char **namespace)
 {
     int    retval = -1;
-    char  *ns;
+    char  *ns = NULL;
     cxobj *xp;
     
+    if ((ns = x->x_ns_cache) != NULL)
+	goto ok;
     if (prefix != NULL) /* xmlns:<prefix>="<uri>" */
 	ns = xml_find_type_value(x, "xmlns", prefix, CX_ATTR);
     else                /* xmlns="<uri>" */
@@ -261,6 +265,11 @@ xml2ns(cxobj *x,
 	    ns = DEFAULT_XML_RPC_NAMESPACE;
 #endif
     }
+    if (ns && (x->x_ns_cache = strdup(ns)) == NULL){
+	clicon_err(OE_XML, errno, "strdup");
+	goto done;
+    }
+ ok:
     if (namespace)
 	*namespace = ns;
     retval = 0;
@@ -562,6 +571,7 @@ xml_child_nr_type(cxobj          *xn,
  * @param[in]  i     the number of the child, eg order in children vector
  * @retval     xml   The child xml node
  * @retval     NULL  if no such child, or empty child
+ * @see xml_child_i_type
  */
 cxobj *
 xml_child_i(cxobj *xn, 
@@ -653,7 +663,7 @@ xml_child_each(cxobj           *xparent,
 }
 
 /*! Extend child vector with one and insert xml node there
- * Note: does not do anything with child, you may need to set its parent, etc
+ * @note does not do anything with child, you may need to set its parent, etc
  */
 static int
 xml_child_append(cxobj *x, 
@@ -669,7 +679,31 @@ xml_child_append(cxobj *x,
     return 0;
 }
 
-/*! Set a a childvec to a specific size, fill with children after
+/*! Insert child xc at position i under parent xp
+ * 
+ * @see xml_child_append
+ * @note does not do anything with child, you may need to set its parent, etc
+ */
+int
+xml_child_insert_pos(cxobj *xp,
+		     cxobj *xc,
+		     int    i)
+{
+    size_t size;
+   
+    xp->x_childvec_len++;
+    xp->x_childvec = realloc(xp->x_childvec, xp->x_childvec_len*sizeof(cxobj*));
+    if (xp->x_childvec == NULL){
+	clicon_err(OE_XML, errno, "realloc");
+	return -1;
+    }
+    size = (xml_child_nr(xp) - i - 1)*sizeof(cxobj *);
+    memmove(&xp->x_childvec[i+1], &xp->x_childvec[i], size);
+    xp->x_childvec[i] = xc;
+    return 0;
+}
+
+/*! Set a childvec to a specific size, fill with children after
  * @code
  *   xml_childvec_set(x, 2);
  *   xml_child_i_set(x, 0, xc0)
@@ -710,8 +744,10 @@ xml_childvec_get(cxobj *x)
  *   ...
  *   xml_free(x);
  * @endcode
- * @note yspec may be NULL either because it is not known or it is irrelevant, 
+ * @note As a rule, yspec should be given in normal Clixon calls to enable
- *       eg for body or attribute
+ *       proper sorting and insert functionality. Except as follows:
+ *         - type is body or attribute
+ *         - Yang is unknown
  * @see xml_sort_insert
  */
 cxobj *
@@ -732,6 +768,7 @@ xml_new(char      *name,
 	xml_parent_set(x, xp);
 	if (xml_child_append(xp, x) < 0) 
 	    return NULL;
+	x->_x_i = xml_child_nr(xp)-1;
     }
     x->x_spec = yspec; /* Can be NULL */
     return x;
@@ -788,11 +825,12 @@ xml_cv_set(cxobj  *x,
  * name "name".
  *
  * @param[in]  x_up   Base XML object
- * @param[in]  name   shell wildcard pattern to match with node name
+ * @param[in]  name   Node name
  *
  * @retval xmlobj     if found.
  * @retval NULL       if no such node found.
  * @see xml_find_type  A more generic function
+ * @note Linear scalability and relies on strcmp
  */
 cxobj *
 xml_find(cxobj *x_up, 
@@ -967,15 +1005,14 @@ xml_child_rm(cxobj *xp,
 int
 xml_rm(cxobj *xc)
 {
-    int    retval = 0;
+    int    retval = -1;
     cxobj *xp;
     cxobj *x;
     int    i;
 
     if ((xp = xml_parent(xc)) == NULL)
-	goto done;
+	goto ok;
-    retval = -1;
+    /* Find child in parent XXX: search? */
-    /* Find child in parent */
     x = NULL; i = 0;
     while ((x = xml_child_each(xp, x, -1)) != NULL) {
 	if (x == xc)
@@ -983,7 +1020,10 @@ xml_rm(cxobj *xc)
 	i++;
     }
     if (x != NULL)
-	retval = xml_child_rm(xp, i);
+	if (xml_child_rm(xp, i) < 0)
+	    goto done;
+ ok:
+    retval = 0;
  done:
     return retval;
 }
@@ -1331,6 +1371,8 @@ xml_free(cxobj *x)
 	free(x->x_childvec);
     if (x->x_cv)
 	cv_free(x->x_cv);
+    if (x->x_ns_cache)
+	free(x->x_ns_cache);
     free(x);
     return 0;
 }

lib/src/clixon_xml_map.c

@@ -1073,6 +1073,13 @@ cvec2xml_1(cvec   *cvv,
  * @param[out] changed_x0  Pointervector to XML nodes changed orig value
  * @param[out] changed_x1  Pointervector to XML nodes changed wanted value
  * @param[out] changedlen Length of changed vector
+ * Algorithm to compare two sorted lists A, B:
+ * A 0 1 2 3 5 6
+ * B 0 2 4 5 6
+ * Let a,b be first elements of A,B respectively
+ * a = b : recurse; get next a,b
+ * a < b : add a in x0, get next a 
+ * a > b : add b in x1, get next b
  */
 static int
 xml_diff1(yang_stmt *ys, 
@@ -1092,35 +1099,54 @@ xml_diff1(yang_stmt *ys,
     yang_stmt *yc;
     char      *b1;
     char      *b2;
+    int        eq;
 
-    clicon_debug(2, "%s: %s", __FUNCTION__, ys->ys_argument?ys->ys_argument:"yspec");
+    /* Traverse x0 and x1 in lock-step */
-    /* Check nodes present in x0 and x1 + nodes only in x0
+    x0c = x1c = NULL;    
-     * Loop over x0
+    x0c = xml_child_each(x0, x0c, CX_ELMNT);
-     * XXX: room for improvement. Compare with match_base_child()
+    x1c = xml_child_each(x1, x1c, CX_ELMNT);
-     */
+    for (;;){
-    x0c = NULL;
+	if (x0c == NULL && x1c == NULL)
-    while ((x0c = xml_child_each(x0, x0c, CX_ELMNT)) != NULL){
+	    goto ok;
-	if ((yc = xml_spec(x0c)) == NULL){
+	else if (x0c == NULL){
-	    clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x0c));
+	    if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
-	    goto done;
+		goto done;
+	    x1c = xml_child_each(x1, x1c, CX_ELMNT);
+	    continue;
 	}
-	/* Does x1 have a child matching x0c? */
+	else if (x1c == NULL){
-	if (match_base_child(x1, x0c, yc, &x1c) < 0)
-	    goto done;
-	if (x1c == NULL){
 	    if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
 		goto done;
+	    x0c = xml_child_each(x0, x0c, CX_ELMNT);
+	    continue;
 	}
-	else if (yang_choice(yc)){
+	/* Both x0c and x1c exists, check if they are equal. */
-	    /* if x0c and x1c are choice/case, then they are changed */
+	eq = xml_cmp(x0c, x1c, 0);
-	    if (cxvec_append(x0c, changed_x0, changedlen) < 0) 
+	if (eq < 0){
-		goto done;
+	    if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
-	    (*changedlen)--; /* append two vectors */
-	    if (cxvec_append(x1c, changed_x1, changedlen) < 0) 
 		goto done;
+	    x0c = xml_child_each(x0, x0c, CX_ELMNT);
 	}
-	else{  /* if x0c and x1c are leafs w bodies, then they are changed */
+	else if (eq > 0){
-	    if (yc->ys_keyword == Y_LEAF){
+	    if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
+		goto done;
+	    x1c = xml_child_each(x1, x1c, CX_ELMNT);
+	}
+	else{ /* equal */
+	    if ((yc = xml_spec(x0c)) == NULL){
+		clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x0c));
+		goto done;
+	    }
+	    if (yang_choice(yc)){
+		/* if x0c and x1c are choice/case, then they are changed */
+		if (cxvec_append(x0c, changed_x0, changedlen) < 0) 
+		    goto done;
+		(*changedlen)--; /* append two vectors */
+		if (cxvec_append(x1c, changed_x1, changedlen) < 0) 
+		    goto done;
+	    }
+	    else if (yc->ys_keyword == Y_LEAF){
+		/* if x0c and x1c are leafs w bodies, then they are changed */
 		if ((b1 = xml_body(x0c)) == NULL) /* empty type */
 		    break;
 		if ((b2 = xml_body(x1c)) == NULL) /* empty type */
@@ -1133,29 +1159,16 @@ xml_diff1(yang_stmt *ys,
 			goto done;
 		}
 	    }
-	    if (xml_diff1(yc, x0c, x1c,   
+	    else if (xml_diff1(yc, x0c, x1c,   
-			  x0vec, x0veclen, 
+			       x0vec, x0veclen, 
-			  x1vec, x1veclen, 
+			       x1vec, x1veclen, 
-			  changed_x0, changed_x1, changedlen)< 0)
+			       changed_x0, changed_x1, changedlen)< 0)
 		goto done;
 	}
-    } /* while x0 */
+	x0c = xml_child_each(x0, x0c, CX_ELMNT);
-    /* Check nodes present only in x1
+	x1c = xml_child_each(x1, x1c, CX_ELMNT);
-     * Loop over x1
+    }
-     */
+ ok:
-    x1c = NULL;
-    while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL){
-	if ((yc = xml_spec(x1c)) == NULL){
-	    clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x1c));
-	    goto done;
-	}
-	/* Does x0 have a child matching x1c? */
-        if (match_base_child(x0, x1c, yc, &x0c) < 0)
-	    goto done;
-	if (x0c == NULL)
-	    if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
-		goto done;
-    } /* while x0 */
     retval = 0;
  done:
     return retval;
@@ -1635,6 +1648,7 @@ xml_default(cxobj *xt,
     cxobj     *xc;
     cxobj     *xb;
     char      *str;
+    int        added=0;
     
     if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){
 	retval = 0;
@@ -1650,8 +1664,14 @@ xml_default(cxobj *xt,
 	    assert(y->ys_cv);
 	    if (!cv_flag(y->ys_cv, V_UNSET)){  /* Default value exists */
 		if (!xml_find(xt, y->ys_argument)){
+
+#ifdef USE_XML_INSERT
+		    if ((xc = xml_new(y->ys_argument, NULL, y)) == NULL)
+			goto done;
+#else
 		    if ((xc = xml_new(y->ys_argument, xt, y)) == NULL)
 			goto done;
+#endif
 		    xml_flag_set(xc, XML_FLAG_DEFAULT);
 		    if ((xb = xml_new("body", xc, NULL)) == NULL)
 			goto done;
@@ -1663,11 +1683,19 @@ xml_default(cxobj *xt,
 		    if (xml_value_set(xb, str) < 0)
 			goto done;
 		    free(str);
+		    added++;
+#ifdef USE_XML_INSERT
+		    if (xml_insert(xt, xc) < 0)
+			goto done;
+#endif
 		}
 	    }
 	}
     }
-    xml_sort(xt, NULL);
+#ifndef USE_XML_INSERT
+    if (added)
+	xml_sort(xt, NULL);
+#endif
     retval = 0;
  done:
     return retval;
@@ -1959,8 +1987,8 @@ api_path2xpath(yang_stmt *yspec,
  * @param[out]  xpathp    Resulting xml tree 
  * @param[out]  ypathp    Yang spec matching xpathp
  * @retval      1         OK
- * @retval     0          Invalid api_path or associated XML, clicon_err called
+ * @retval      0         Invalid api_path or associated XML, clicon_err called
- * @retval    -1          Fatal error, clicon_err called
+ * @retval     -1         Fatal error, clicon_err called
  *
  * @note both retval 0 and -1 set clicon_err, but the later is fatal
  * @see api_path2xpath For api-path to xml xpath translation
@@ -1994,6 +2022,7 @@ api_path2xml_vec(char             **vec,
     cxobj     *x = NULL;
     yang_stmt *y = NULL;
     yang_stmt *ymod;
+    yang_stmt *ykey;
     char      *namespace = NULL;
 
     if ((nodeid = vec[0]) == NULL || strlen(nodeid)==0){
@@ -2077,7 +2106,12 @@ api_path2xml_vec(char             **vec,
 	/* Create keys */
 	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
 	    keyname = cv_string_get(cvi);
-	    if ((xn = xml_new(keyname, x, NULL)) == NULL)
+	    if ((ykey = yang_find(y, Y_LEAF, keyname)) == NULL){
+		clicon_err(OE_XML, 0, "List statement \"%s\" has no key leaf \"%s\"", 
+			   yang_argument_get(y), keyname);
+		goto done;
+	    }
+	    if ((xn = xml_new(keyname, x, ykey)) == NULL)
 		goto done; 
 	    xml_type_set(xn, CX_ELMNT);
 	    if ((xb = xml_new("body", xn, NULL)) == NULL)

lib/src/clixon_xml_parse.l

@@ -132,10 +132,10 @@ ncname     {namestart}{namechar}*
 <STATEA>"<?"          { BEGIN(PIDECL); return BQMARK; }
 <STATEA>\<            { BEGIN(START); return *clixon_xml_parsetext; }
 <STATEA>&             { _YA->ya_lex_state =STATEA;BEGIN(AMPERSAND);}
-<STATEA>[ \t]         { clixon_xml_parselval.string = yytext;return WHITESPACE; }
+<STATEA>[ \t]+        { clixon_xml_parselval.string = yytext;return WHITESPACE; }
-<STATEA>\r\n          { clixon_xml_parselval.string = "\n";return WHITESPACE; }
+<STATEA>\r\n          { clixon_xml_parselval.string = "\n"; _YA->ya_linenum++; return WHITESPACE; }
 <STATEA>\r            { clixon_xml_parselval.string = "\n";return WHITESPACE; }
-<STATEA>\n            { clixon_xml_parselval.string = yytext; _YA->ya_linenum++;return WHITESPACE; }
+<STATEA>\n            { clixon_xml_parselval.string = "\n"; _YA->ya_linenum++;return WHITESPACE; }
 <STATEA>.             { clixon_xml_parselval.string = yytext; return CHARDATA; }
 
 	/* @see xml_chardata_encode */

lib/src/clixon_xml_parse.y

@@ -112,6 +112,44 @@ xml_parse_content(struct xml_parse_yacc_arg *ya,
     return retval;
 }
 
+/*! Add whitespace
+ * If text, ie only body, keep as is.
+ * But if there is an element, then skip all whitespace.
+ */
+static int
+xml_parse_whitespace(struct xml_parse_yacc_arg *ya,
+		     char                      *str)
+{
+    cxobj *xn = ya->ya_xelement;
+    cxobj *xp = ya->ya_xparent;
+    int    retval = -1;
+    int    i;
+
+    ya->ya_xelement = NULL; /* init */
+    /* If there is an element already, only add one whitespace child 
+     * otherwise, keep all whitespace.
+     */
+#if 1
+    for (i=0; i<xml_child_nr(xp); i++){
+	if (xml_type(xml_child_i(xp, i)) == CX_ELMNT)
+	    goto ok; /* Skip if already element */
+    }
+#endif
+    if (xn == NULL){
+	if ((xn = xml_new("body", xp, NULL)) == NULL)
+	    goto done; 
+	xml_type_set(xn, CX_BODY);
+    }
+    if (xml_value_append(xn, str)==NULL)
+	goto done; 
+    ya->ya_xelement = xn;
+ ok:
+    retval = 0;
+  done:
+    return retval;
+}
+
+ 
 static int
 xml_parse_version(struct xml_parse_yacc_arg *ya,
 		  char                      *ver)
@@ -243,10 +281,16 @@ xml_parse_bslash1(struct xml_parse_yacc_arg *ya,
 	while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) 
 	    break;
 	if (xc != NULL){ /* at least one element */
-	    xc = NULL;
+	    int i;
-	    while ((xc = xml_child_each(x, xc, CX_BODY)) != NULL) {
+	    for (i=0; i<xml_child_nr(x);){
-		xml_purge(xc);
+		xc = xml_child_i(x, i);
-		xc = NULL; /* reset iterator */
+		if (xml_type(xc) != CX_BODY){
+		    i++;
+		    continue;
+		}
+		if (xml_child_rm(x, i) < 0)
+		    goto done;
+		xml_free(xc);
 	    }
 	}
     }
@@ -425,7 +469,7 @@ content     : element      { clicon_debug(2, "content -> element"); }
             | pi           { clicon_debug(2, "content -> pi"); }
             | CHARDATA     { if (xml_parse_content(_YA, $1) < 0) YYABORT;  
                              clicon_debug(2, "content -> CHARDATA %s", $1); }
-            | WHITESPACE   { if (xml_parse_content(_YA, $1) < 0) YYABORT;  
+            | WHITESPACE   { if (xml_parse_whitespace(_YA, $1) < 0) YYABORT;  
                              clicon_debug(2, "content -> WHITESPACE %s", $1); }
             |              { clicon_debug(2, "content -> "); }
             ;

lib/src/clixon_xml_sort.c

@@ -89,7 +89,8 @@ xml_cv_cache(cxobj   *x,
     uint8_t      fraction = 0;
     char        *body;
 		 
-    body = xml_body(x);
+    if ((body = xml_body(x)) == NULL)
+	body="";
     if ((cv = xml_cv(x)) != NULL)
 	goto ok;
     if ((y = xml_spec(x)) == NULL)
@@ -187,19 +188,30 @@ xml_child_spec(cxobj      *x,
 }
 
 /*! Help function to qsort for sorting entries in xml child vector same parent
- * @param[in]  xml object 1
+ * @param[in]  x1    object 1
- * @param[in]  xml object 2
+ * @param[in]  x2    object 2
- * @retval  0  If equal
+ * @param[in]  same  If set, x1 and x2 are member of same parent & enumeration 
- * @retval <0  if x1 is less than x2
+ *                   is used (see explanation below)
- * @retval >0  if x1 is greater than x2
+ * @retval     0     If equal
+ * @retval    <0     If x1 is less than x2
+ * @retval    >0     If x1 is greater than x2
  * @see xml_cmp1   Similar, but for one object
+ *
+ * There are distinct calls for this function:
+ * 1. For sorting in an existing list of XML children
+ * 2. For searching of an existing element in a list
+ * In the first case, there is a special case for "ordered-by-user", where
+ * if they have the same yang-spec, the existing order is used as tie-breaker.
+ * In other words, if order-by-system, or if the case (2) above, the existing
+ * order is ignored and the actual xml element contents is examined.
  * @note empty value/NULL is smallest value
- * @note xml_enumerate_children must have been called prior to this call
+ * @note some error cases return as -1 (qsort cant handle errors)
  * @note some error cases return as -1 (qsort cant handle errors)
  */
-static int
+int
 xml_cmp(cxobj *x1,
-	cxobj *x2)
+	cxobj *x2,
+	int    same)
 {
     yang_stmt  *y1;
     yang_stmt  *y2;
@@ -217,18 +229,18 @@ xml_cmp(cxobj *x1,
     int         nr2 = 0;
     cxobj      *x1b;
     cxobj      *x2b;
-    int         e;
 
-    e=0;
     if (x1==NULL || x2==NULL)
 	goto done; /* shouldnt happen */
-    e=1;
     y1 = xml_spec(x1);
     y2 = xml_spec(x2);
-    nr1 = xml_enumerate_get(x1);
+    if (same){
-    nr2 = xml_enumerate_get(x2);
+	nr1 = xml_enumerate_get(x1);
+	nr2 = xml_enumerate_get(x2);
+    }
     if (y1==NULL && y2==NULL){
-	equal = nr1-nr2;
+	if (same)
+	    equal = nr1-nr2;
 	goto done;
     }
     if (y1==NULL){
@@ -239,24 +251,24 @@ xml_cmp(cxobj *x1,
         equal = 1;
         goto done;
     }
-    e=2;
     if (y1 != y2){ 
 	yi1 = yang_order(y1);
 	yi2 = yang_order(y2);
 	if ((equal = yi1-yi2) != 0)
 	    goto done;
     }
-    e=3;
     /* Now y1==y2, same Yang spec, can only be list or leaf-list,
      * But first check exceptions, eg config false or ordered-by user
      * otherwise sort according to key
+     * If the two elements are in the same list, and they are ordered-by user
+     * then do not look more into equivalence, use the enumeration in the
+     * existing list.
      */
-    if (yang_config(y1)==0 ||
+    if (same &&
-	yang_find(y1, Y_ORDERED_BY, "user") != NULL){
+	(yang_config(y1)==0 || yang_find(y1, Y_ORDERED_BY, "user") != NULL)){
-	equal = nr1-nr2;
+	    equal = nr1-nr2;
-	goto done; /* Ordered by user or state data : maintain existing order */
+	    goto done; /* Ordered by user or state data : maintain existing order */
-    }
+	}
-    e=4;
     switch (yang_keyword_get(y1)){
     case Y_LEAF_LIST: /* Match with name and value */
 	if ((b1 = xml_body(x1)) == NULL)
@@ -287,8 +299,10 @@ xml_cmp(cxobj *x1,
 	    else{
 		if (xml_cv_cache(x1b, &cv1) < 0) /* error case */
 		    goto done;
+		assert(cv1); 		
 		if (xml_cv_cache(x2b, &cv2) < 0) /* error case */
 		    goto done;
+		assert(cv2);
 		if ((equal = cv_cmp(cv1, cv2)) != 0)
 		    goto done;
 	    }
@@ -298,9 +312,8 @@ xml_cmp(cxobj *x1,
     default:
 	break;
     }
-    e=5;
  done:
-    clicon_debug(2, "%s %s %s %d %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, e, nr1, nr2, yi1, yi2);
+    clicon_debug(2, "%s %s %s %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, nr1, nr2, yi1, yi2);
     return equal;
 }
 
@@ -311,95 +324,9 @@ static int
 xml_cmp_qsort(const void* arg1, 
 	      const void* arg2)
 {
-    return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2);
+    return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1);
 }
 
-/*! Compare xml object
- * @param[in]  x        XML node to compare with
- * @param[in]  y        The yang spec of x
- * @param[in]  name     Name to compare with x
- * @param[in]  keyword  Yang keyword (stmt type) to compare w x/y
- * @param[in]  keynr    Length of keyvec/keyval vector when applicable
- * @param[in]  keyvec   Array of of yang key identifiers
- * @param[in]  keyval   Array of of yang key values
- * @param[out] userorder If set, this yang order is user ordered, linear search
- * @retval  0  If equal (or userorder set)
- * @retval <0  if arg1 is less than arg2
- * @retval >0  if arg1 is greater than arg2
- * @see xml_cmp   Similar, but for two objects
- * @note Does not care about y type of value as xml_cmp
- */
-static int
-xml_cmp1(cxobj        *x,
-	 yang_stmt    *y,
-	 char         *name,
-	 enum rfc_6020 keyword,   
-	 int           keynr,
-	 char        **keyvec,
-	 char        **keyval,
-	 cg_var      **keycvec,
-	 int          *userorder)
-{
-    char      *b;
-    cxobj     *xb;
-    int        i;
-    char      *keyname;
-    char      *key;
-    int        match = 0;
-    cg_var    *cv;
-
-    /* state data = userorder */
-    if (userorder && yang_config(y)==0)
-	*userorder=1;
-    /* Check if same yang spec (order in yang stmt list) */
-    switch (keyword){
-    case Y_CONTAINER: /* Match with name */
-    case Y_LEAF: /* Match with name */
-	match = strcmp(name, xml_name(x));
-	break;
-    case Y_LEAF_LIST: /* Match with name and value */
-	if (userorder && yang_find(y, Y_ORDERED_BY, "user") != NULL)
-	    *userorder=1;
-	if ((b=xml_body(x)) == NULL)
-	    match = 1;
-	else{
-	    if (keycvec[0]){
-		if (xml_cv_cache(x, &cv) < 0) /* error case */
-		    goto done;
-		match = cv_cmp(keycvec[0], cv);
-	    }
-	    else
-		match = strcmp(keyval[0], b);
-	}
-	break;
-    case Y_LIST: /* Match with array of key values */
-	if (userorder && yang_find(y, Y_ORDERED_BY, "user") != NULL)
-	    *userorder=1;
-	/* All must match */
-	for (i=0; i<keynr; i++){
-	    keyname = keyvec[i];
-	    key = keyval[i];
-	    if ((xb = xml_find(x, keyname)) == NULL)
-		break; /* error case */
-	    if ((b = xml_body(xb)) == NULL)
-		break; /* error case */
-	    if (xml_cv_cache(xb, &cv) < 0) /* error case */
-		goto done;
-	    if (keycvec[i]){
-		if ((match = cv_cmp(keycvec[i], cv)) != 0)
-		    break;
-	    }
-	    else
-		if ((match = strcmp(key, b)) != 0)
-		    break;
-	}
-	break;
-    default:
-	break;
-    }
- done:
-    return match;
-}
 
 /*! Sort children of an XML node 
  * Assume populated by yang spec.
@@ -427,40 +354,37 @@ xml_sort(cxobj *x,
 /*! Special case search for ordered-by user where linear sort is used
  */
 static cxobj *
-xml_search_userorder(cxobj        *x0,
+xml_search_userorder(cxobj        *xp,
+		     cxobj        *x1,
 		     yang_stmt    *y,
-		     char         *name,
 		     int           yangi,
-		     int           mid,
+		     int           mid)
-		     enum rfc_6020 keyword,   
+
-		     int           keynr,
-		     char        **keyvec,
-		     char        **keyval,
-		     cg_var      **keycvec)
 {
     int    i;
     cxobj *xc;
     
-    for (i=mid+1; i<xml_child_nr(x0); i++){ /* First increment */
+    for (i=mid+1; i<xml_child_nr(xp); i++){ /* First increment */
-	xc = xml_child_i(x0, i);
+	xc = xml_child_i(xp, i);
 	y = xml_spec(xc);
 	if (yangi!=yang_order(y))
 	    break;
-	if (xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, NULL) == 0)
+	if (xml_cmp(xc, x1, 0) == 0)
 	    return xc;
     }
     for (i=mid-1; i>=0; i--){ /* Then decrement */
-	xc = xml_child_i(x0, i);
+	xc = xml_child_i(xp, i);
 	y = xml_spec(xc);
 	if (yangi!=yang_order(y))
 	    break;
-	if (xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, NULL) == 0)
+	if (xml_cmp(xc, x1, 0) == 0)
 	    return xc;
     }
     return NULL; /* Not found */
 }
 
 /*!
+ * @param[in] xp       Parent xml node. 
  * @param[in] yangi    Yang order
  * @param[in] keynr    Length of keyvec/keyval vector when applicable
  * @param[in] keyvec   Array of of yang key identifiers
@@ -469,14 +393,10 @@ xml_search_userorder(cxobj        *x0,
  * @param[in] upper    Lower bound of childvec search interval 
  */
 static cxobj *
-xml_search1(cxobj        *x0,
+xml_search1(cxobj        *xp,
-	    char         *name,
+	    cxobj        *x1,
+	    int           userorder,
 	    int           yangi,
-	    enum rfc_6020 keyword,   
-	    int           keynr,
-	    char        **keyvec,
-	    char        **keyval,
-	    cg_var      **keycvec,
 	    int           low, 
 	    int           upper)
 {
@@ -484,122 +404,200 @@ xml_search1(cxobj        *x0,
     int        cmp;
     cxobj     *xc;
     yang_stmt *y;
-    int        userorder= 0;
 
     if (upper < low)
 	return NULL; /* not found */
     mid = (low + upper) / 2;
-    if (mid >= xml_child_nr(x0))  /* beyond range */
+    if (mid >= xml_child_nr(xp))  /* beyond range */
 	return NULL;
-    xc = xml_child_i(x0, mid);
+    xc = xml_child_i(xp, mid);
     if ((y = xml_spec(xc)) == NULL)
 	return NULL;
     cmp = yangi-yang_order(y);
     /* Here is right yang order == same yang? */
     if (cmp == 0){
-	cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, &userorder);
+	if (userorder){
-	if (userorder && cmp)	    /* Look inside this yangi order */
+	    return xml_search_userorder(xp, x1, y, yangi, mid);	    
-	    return xml_search_userorder(x0, y, name, yangi, mid, keyword, keynr, keyvec, keyval, keycvec);
+	}
+	else /* Ordered by system */
+	    cmp = xml_cmp(x1, xc, 0);
     }
     if (cmp == 0)
 	return xc;
     else if (cmp < 0)
-	return xml_search1(x0, name, yangi, keyword,
+	return xml_search1(xp, x1, userorder, yangi, low, mid-1);
-			   keynr, keyvec, keyval, keycvec, low, mid-1);
     else 
-	return xml_search1(x0, name, yangi, keyword,
+	return xml_search1(xp, x1, userorder, yangi, mid+1, upper);
-			   keynr, keyvec, keyval, keycvec,  mid+1, upper);
     return NULL;
 }
 
-/*! Find XML children using binary search
+/*! Find XML child under xp matching x1 using binary search
- * @param[in] yangi  yang child order
+ * @param[in] xp     Parent xml node. 
+ * @param[in] yangi  Yang child order
  * @param[in] keynr  Length of keyvec/keyval vector when applicable
  * @param[in] keyvec Array of of yang key identifiers
  * @param[in] keyval Array of of yang key values
  */
 static cxobj *
-xml_search(cxobj        *x0,
+xml_search(cxobj        *xp,
-	   char         *name,
+	   cxobj        *x1,
-	   int           yangi,
+	   yang_stmt    *yc)
-	   enum rfc_6020 keyword,   
-	   int           keynr,
-	   char        **keyvec,
-	   char        **keyval,
-    	   cg_var      **keycvec)
 {
-    cxobj *xa;
+    cxobj     *xa;
-    int    low = 0;
+    int        low = 0;
-    int    high = xml_child_nr(x0);
+    int        upper = xml_child_nr(xp);
+    int        userorder=0;
+    cxobj     *xret = NULL;
+    int        yangi;
     
     /* Assume if there are any attributes, they are first in the list, mask
        them by raising low to skip them */
-    for (low=0; low<high; low++)
+    for (low=0; low<upper; low++)
-	if ((xa = xml_child_i(x0, low)) == NULL || xml_type(xa)!=CX_ATTR)
+	if ((xa = xml_child_i(xp, low)) == NULL || xml_type(xa)!=CX_ATTR)
 	    break;
-    return xml_search1(x0, name, yangi, keyword, keynr, keyvec, keyval, keycvec,
+    /* Find if non-config and if ordered-by-user */
-		       low, high);
+    if (yang_config(yc)==0)
+	userorder = 1;
+    else if (yang_keyword_get(yc) == Y_LIST || yang_keyword_get(yc) == Y_LEAF_LIST)
+	userorder = (yang_find(yc, Y_ORDERED_BY, "user") != NULL);
+    yangi = yang_order(yc);
+    xret = xml_search1(xp, x1, userorder, yangi, low, upper);
+    return xret;
 }
 
-#ifdef NOTUSED
+/*! Insert xn in xp:s sorted child list
-/*! Position where to insert xml object into a list of children nodes
+ * Find a point in xp childvec with two adjacent nodes xi,xi+1 such that
- * @note EXPERIMENTAL
+ * xi<=xn<=xi+1 or xn<=x0 or xmax<=xn
- * Insert after position returned
+ */
- * @param[in]  x0       XML parent node.
+static int
- * @param[in]  low       Lower bound
+xml_insert2(cxobj     *xp,
- * @param[in]  upper     Upper bound (+1)
+	    cxobj     *xn,
- * @retval     position 
+	    yang_stmt *yn,
- * XXX: Problem with this is that evrything must be known before insertion
+	    int        yni,
+	    int        userorder,
+	    int        low, 
+	    int        upper)
+{
+    int        retval = -1;
+    int        mid;
+    int        cmp;
+    cxobj     *xc;
+    yang_stmt *yc;
+    int        i;
+    
+    if (low > upper){  /* beyond range */
+	clicon_err(OE_XML, 0, "low>upper %d %d", low, upper);	
+	goto done;
+    }
+    if (low == upper){
+	retval = low;
+	goto done;
+    }
+    mid = (low + upper) / 2;
+    if (mid >= xml_child_nr(xp)){  /* beyond range */
+	clicon_err(OE_XML, 0, "Beyond range %d %d %d", low, mid, upper);	
+	goto done;
+    }
+    xc = xml_child_i(xp, mid);
+    if ((yc = xml_spec(xc)) == NULL){
+	clicon_err(OE_XML, 0, "No spec found %s", xml_name(xc));	
+	goto done;
+    }
+    if (yc == yn){ /* Same yang */
+	if (userorder){ /* append: increment linearly until no longer equal */
+	    for (i=mid+1; i<xml_child_nr(xp); i++){ /* First increment */
+		xc = xml_child_i(xp, i);
+		yc = xml_spec(xc);
+		if (yc != yn){
+		    retval = i;
+		    goto done;
+		}
+	    }
+	    retval = i;
+	    goto done;
+	}
+	else /* Ordered by system */
+	    cmp = xml_cmp(xn, xc, 0);
+    }
+    else{ /* Not equal yang - compute diff */
+	cmp = yni - yang_order(yc);
+	/* One case is a choice where 
+	 * xc = <tcp/>, xn = <udp/>
+	 * same order but different yang spec
+	 */
+    }
+    if (low +1 == upper){ /* termination criterium */
+	if (cmp<0) {
+	    retval = mid;
+	    goto done;
+	}
+	retval = mid+1;
+	goto done;
+    }
+    if (cmp == 0){
+	retval = mid;
+	goto done;
+    }
+    else if (cmp < 0)
+	return xml_insert2(xp, xn, yn, yni, userorder, low, mid);
+    else 
+	return xml_insert2(xp, xn, yn, yni, userorder, mid+1, upper);
+ done:
+    return retval;
+}
+
+/*! Insert xc as child to xp in sorted place. Remove xc from previous parent.
+ * @param[in] xp  Parent xml node. If NULL just remove from old parent.
+ * @param[in] x   Child xml node to insert under xp
+ * @retval    0   OK
+ * @retval   -1   Error
+ * @see xml_addsub where xc is appended. xml_insert is xml_addsub();xml_sort()
  */
 int
-xml_insert_pos(cxobj        *x0,
+xml_insert(cxobj        *xp,
-	       char         *name,
+	   cxobj        *xi)
-	       int           yangi,
-	       enum rfc_6020 keyword,   
-	       int           keynr,
-	       char        **keyvec,
-	       char        **keyval,
-	       int           low, 
-	       int           upper)
 {
-    int        mid;
+    int        retval = -1;
-    cxobj     *xc;
+    cxobj     *xa;
+    int        low = 0;
+    int        upper;
     yang_stmt *y;
-    int        cmp;
-    int        i;
     int        userorder= 0;
+    int        yi; /* Global yang-stmt order */
+    int        i;
 
-    if (upper < low)
+    /* Ensure the intermediate state that xp is parent of x but has not yet been
-	return low; /* not found */
+     * added as a child
-    mid = (low + upper) / 2;
+     */
-    if (mid >= xml_child_nr(x0))
+    if (xml_parent(xi) != NULL){
-	return xml_child_nr(x0); /* upper range */
+	clicon_err(OE_XML, 0, "XML node %s should not have parent", xml_name(xi));
-    xc = xml_child_i(x0, mid); 
+	goto done;
-    y = xml_spec(xc);
-    cmp = yangi-yang_order(y);
-    if (cmp == 0){
-	cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, &userorder);
-	if (userorder){	    /* Look inside this yangi order */
-	    /* Special case: append last of equals if ordered by user */
-	    for (i=mid+1;i<xml_child_nr(x0);i++){
-		xc = xml_child_i(x0, i);
-		if (strcmp(xml_name(xc), name))
-		    break;
-		mid=i; /* still ok */
-	    }
-	    return mid;
-	}
     }
-    if (cmp == 0)
+    if ((y = xml_spec(xi)) == NULL){
-	return mid;
+	clicon_err(OE_XML, 0, "No spec found %s", xml_name(xi));
-    else if (cmp < 0)
+	goto done;
-	return xml_insert_pos(x0, name, yangi, keyword,
+    }
-			      keynr, keyvec, keyval, keycvec, low, mid-1);
+    upper = xml_child_nr(xp);
-    else
+    /* Assume if there are any attributes, they are first in the list, mask
-	return xml_insert_pos(x0, name, yangi, keyword,
+       them by raising low to skip them */
-			      keynr, keyvec, keyval, keycvec, mid+1, upper);
+    for (low=0; low<upper; low++)
+	if ((xa = xml_child_i(xp, low)) == NULL || xml_type(xa)!=CX_ATTR)
+	    break;
+    /* Find if non-config and if ordered-by-user */
+    if (yang_config(y)==0)
+	userorder = 1;
+    else if (yang_keyword_get(y) == Y_LIST || yang_keyword_get(y) == Y_LEAF_LIST)
+	userorder = (yang_find(y, Y_ORDERED_BY, "user") != NULL);
+    yi = yang_order(y);
+    if ((i = xml_insert2(xp, xi, y, yi, userorder, low, upper)) < 0)
+	goto done;
+    if (xml_child_insert_pos(xp, xi, i) < 0)
+	goto done;
+    xml_parent_set(xi, xp);
+    retval = 0;
+ done:
+    return retval;
 }
-#endif /* NOTUSED */
 
 /*! Verify all children of XML node are sorted according to xml_sort()
  * @param[in]   x       XML node. Check its children
@@ -625,7 +623,7 @@ xml_sort_verify(cxobj *x0,
     xml_enumerate_children(x0);
     while ((x = xml_child_each(x0, x, -1)) != NULL) {
 	if (xprev != NULL){ /* Check xprev <= x */
-	    if (xml_cmp(xprev, x) > 0)
+	    if (xml_cmp(xprev, x, 1) > 0)
 		goto done;
 	}
 	xprev = x;
@@ -652,15 +650,8 @@ match_base_child(cxobj      *x0,
     int        retval = -1;
     cvec      *cvk = NULL; /* vector of index keys */
     cg_var    *cvi;
-    char      *b;
     cxobj     *xb;
     char      *keyname;
-    char       keynr = 0;
-    char     **keyval = NULL;
-    char     **keyvec = NULL;
-    cg_var   **keycvec = NULL;
-    int        i;
-    int        yorder;
     cxobj     *x0c = NULL;
     yang_stmt *y0c;
     yang_stmt *y0p;
@@ -686,19 +677,10 @@ match_base_child(cxobj      *x0,
     case Y_LEAF: 	/* Equal regardless */
 	break;
     case Y_LEAF_LIST: /* Match with name and value */
-	keynr = 1;
+	if (xml_body(x1c) == NULL){ /* Treat as empty string */
-	if ((keyval = calloc(keynr+1, sizeof(char*))) == NULL){
+	    //	    assert(0);
-	    clicon_err(OE_UNIX, errno, "calloc");
-	    goto done;
-	}
-	if ((keyval[0] = xml_body(x1c)) == NULL)
 	    goto ok;
-	if ((keycvec = calloc(keynr+1, sizeof(cg_var*))) == NULL){
-	    clicon_err(OE_UNIX, errno, "calloc");
-	    goto done;
 	}
-	if (xml_cv_cache(x1c, &keycvec[0]) < 0) /* error case */
-	    goto done;
 	break;
     case Y_LIST: /* Match with key values */
 	cvk = yang_cvec_get(yc); /* Use Y_LIST cache, see ys_populate_list() */
@@ -706,51 +688,22 @@ match_base_child(cxobj      *x0,
 	 * Then create two vectors one with names and one with values of x1c,
 	 * ec: keyvec: [a,b,c]  keyval: [1,2,3]
 	 */
-	cvi = NULL; keynr = 0;
+	cvi = NULL; 
-	while ((cvi = cvec_each(cvk, cvi)) != NULL) 
-	    keynr++;
-	if ((keyval = calloc(keynr+1, sizeof(char*))) == NULL){
-	    clicon_err(OE_UNIX, errno, "calloc");
-	    goto done;
-	}
-	if ((keyvec = calloc(keynr+1, sizeof(char*))) == NULL){
-	    clicon_err(OE_UNIX, errno, "calloc");
-	    goto done;
-	}
-	if ((keycvec = calloc(keynr+1, sizeof(char*))) == NULL){
-	    clicon_err(OE_UNIX, errno, "calloc");
-	    goto done;
-	}
-	cvi = NULL; i = 0;
 	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
 	    keyname = cv_string_get(cvi);
-	    keyvec[i] = keyname;
+	    //	    keyvec[i] = keyname;
-	    if ((xb = xml_find(x1c, keyname)) == NULL)
+	    if ((xb = xml_find(x1c, keyname)) == NULL){
 		goto ok;
-	    if ((b = xml_body(xb)) == NULL)
+	    }
-		goto ok;
-	    keyval[i] = b;
-	    if (xml_cv_cache(xb, &keycvec[i]) < 0) /* error case */
-		goto done;
-	    i++;
 	}
-	break;
     default:
 	break;
     }
     /* Get match. */
-    yorder = yang_order(yc);
+    x0c = xml_search(x0, x1c, yc);
-    x0c = xml_search(x0, xml_name(x1c), yorder, yang_keyword_get(yc), keynr, keyvec, keyval, keycvec);
  ok:
     *x0cp = x0c;
     retval = 0;
- done:
-    if (keyval)
-	free(keyval);
-    if (keyvec)
-	free(keyvec);
-    if (keycvec)
-	free(keycvec);
     return retval;
 }
 	   

lib/src/clixon_yang.c

@@ -69,6 +69,7 @@
 #include <sys/param.h>
 #include <netinet/in.h>
 #include <libgen.h>
+#include <regex.h>
 
 /* cligen */
 #include <cligen/cligen.h>
@@ -223,6 +224,46 @@ yang_cvec_get(yang_stmt *ys)
     return ys->ys_cvec;
 }
 
+/*! Set yang statement CLIgen variable vector
+ * @param[in] ys   Yang statement node
+ * @param[in] cvec CLIgen vector
+ * @retval    0    OK
+ * @retval   -1    Error
+ */
+int
+yang_cvec_set(yang_stmt *ys,
+	      cvec      *cvv)
+{
+    if (ys->ys_cvec)
+	cvec_free(ys->ys_cvec);
+    ys->ys_cvec = cvv;
+    return 0;
+}
+
+/*! Get regular expression cache - the compiled regex
+ * @param[in] ys  Yang statement node
+ * @retval    re  Compiled regex
+ * @see regcomp
+ */
+void*
+yang_regex_cache_get(yang_stmt *ys)
+{
+    return ys->ys_regex_cache;
+}
+
+/*! Set regular expression cache - the compiled regex
+ * @param[in] ys  Yang statement node
+ * @param[in] re  Compiled regex
+ * @see regcomp
+ */
+int
+yang_regex_cache_set(yang_stmt *ys,
+		     void      *regex)
+{
+    ys->ys_regex_cache = regex;
+    return 0;
+}
+
 /* End access functions */
 
 /*! Create new yang specification
@@ -267,6 +308,17 @@ ys_new(enum rfc_6020 keyw)
     return ys;
 }
 
+
+static int
+yang_regex_cache_free(yang_stmt *ys)
+{
+    if (ys->ys_regex_cache){
+	regfree(ys->ys_regex_cache);
+	free(ys->ys_regex_cache);
+    }
+    return 0;
+}
+
 /*! Free a single yang statement */
 static int 
 ys_free1(yang_stmt *ys)
@@ -281,6 +333,7 @@ ys_free1(yang_stmt *ys)
 	cvec_free(ys->ys_cvec);
     if (ys->ys_typecache)
 	yang_type_cache_free(ys->ys_typecache);
+    yang_regex_cache_free(ys);
     free(ys);
     return 0;
 }
@@ -720,6 +773,30 @@ yang_choice(yang_stmt *y)
     return NULL;
 }
 
+static int
+order1_choice(yang_stmt *yp,
+	      yang_stmt *y)
+{
+    yang_stmt  *ys;
+    yang_stmt  *yc;
+    int         i;
+    int         j;
+
+    for (i=0; i<yp->ys_len; i++){
+	ys = yp->ys_stmt[i];
+	if (ys->ys_keyword == Y_CASE){
+	    for (j=0; j<ys->ys_len; j++){
+		yc = ys->ys_stmt[j];
+		if (yang_datanode(yc) && yc == y)
+		    return 1;
+	    }
+	}
+	else if (yang_datanode(ys) && ys == y)
+	    return 1;
+    }
+    return 0;
+}
+
 /*! Find matching y in yp:s children, return 0 and index or -1 if not found.
  * @param[in]  yp     Parent
  * @param[in]  y      Yang datanode to find
@@ -737,10 +814,16 @@ order1(yang_stmt *yp,
     
     for (i=0; i<yp->ys_len; i++){
 	ys = yp->ys_stmt[i];
-	if (!yang_datanode(ys))
+	if (ys->ys_keyword == Y_CHOICE){
-	    continue;
+	    if (order1_choice(ys, y) == 1) /* If one of the choices is "y" */
-	if (ys==y)
+		return 1;
-	    return 1;
+	}
+	else {
+	    if (!yang_datanode(ys))
+		continue;
+	    if (ys==y)
+		return 1;
+	}
 	(*index)++;
     }
     return 0;
@@ -763,7 +846,14 @@ yang_order(yang_stmt *y)
     int         j=0;
     int         tot = 0;
     
+    /* Some special handling if yp is choice (or case) and maybe union?
+     * if so, the real parent (from an xml point of view) is the parents
+     * parent. 
+     */
     yp = y->ys_parent;
+    while (yp->ys_keyword == Y_CASE || yp->ys_keyword == Y_CHOICE)
+	yp = yp->ys_parent;
+
     /* XML nodes with yang specs that are children of modules are special - 
      * In clixon, they are seen as an "implicit" container where the XML can come from different
      * modules. The order must therefore be global among yang top-symbols to be unique.
@@ -1950,6 +2040,7 @@ yang_parse_str(char         *str,
  * @param[in] ysp    Yang specification. Should have been created by caller using yspec_new
  * @retval ymod      Top-level yang (sub)module
  * @retval NULL      Error 
+ * @note this function simply parse a yang spec, no dependencies or checks
  */
 yang_stmt *
 yang_parse_file(int         fd,

lib/src/clixon_yang_internal.h

@@ -95,6 +95,7 @@ struct yang_stmt{
 					Y_TYPE & identity: store all derived types
 				     */
     yang_type_cache   *ys_typecache; /* If ys_keyword==Y_TYPE, cache all typedef data except unions */
+    void             *ys_regex_cache; /* regex cache */
     int               _ys_vector_i;   /* internal use: yn_each */
 };
 

lib/src/clixon_yang_module.c

@@ -160,7 +160,8 @@ yang_modules_revision(clicon_handle h)
 }
 
 /*! Actually build the yang modules state XML tree
-*/
+ * @see RFC7895
+ */
 static int
 yms_build(clicon_handle    h,
 	  yang_stmt       *yspec,
@@ -198,8 +199,11 @@ yms_build(clicon_handle    h,
 	cprintf(cb,"<name>%s</name>", ymod->ys_argument);
 	if ((ys = yang_find(ymod, Y_REVISION, NULL)) != NULL)
 	    cprintf(cb,"<revision>%s</revision>", ys->ys_argument);
-	else
+	else{
+	    /* RFC7895 1 If no (such) revision statement exists, the module's or 
+	       submodule's revision is the zero-length string. */
 	    cprintf(cb,"<revision></revision>");
+	}
 	if ((ys = yang_find(ymod, Y_NAMESPACE, NULL)) != NULL)
 	    cprintf(cb,"<namespace>%s</namespace>", ys->ys_argument);
 	else

lib/src/clixon_yang_type.c

@@ -49,6 +49,7 @@
 #include <regex.h>
 #include <syslog.h>
 #include <assert.h>
+#include <regex.h>
 #include <netinet/in.h>
 
 /* cligen */
@@ -100,11 +101,106 @@ static const map_str2int ytmap[] = {
     {NULL,         -1}
 };
 
+/*! Mapping from yang string types --> cligen types
+ * @note not 100% same as map_str2int since it has significant order AND
+ *       string->CGV_REST entry removed
+ */
+static const map_str2int ytmap2[] = {
+    {"binary",      CGV_STRING},
+    {"bits",        CGV_STRING},
+    {"boolean",     CGV_BOOL},
+    {"decimal64",   CGV_DEC64},
+    {"empty",       CGV_VOID},  /* May not include any content */
+    {"enumeration", CGV_STRING},
+    {"identityref", CGV_STRING},  /* XXX */
+    {"instance-identifier", CGV_STRING}, /* XXX */
+    {"int16",       CGV_INT16},
+    {"int32",       CGV_INT32},
+    {"int64",       CGV_INT64},
+    {"int8",        CGV_INT8},
+    {"leafref",     CGV_STRING},  /* XXX */
+    {"string",      CGV_STRING},
+    {"uint16",      CGV_UINT16},
+    {"uint32",      CGV_UINT32},
+    {"uint64",      CGV_UINT64},
+    {"uint8",       CGV_UINT8},
+    {"union",       CGV_REST},  /* Is replaced by actual type */
+    {NULL,         -1}
+};
+
+/*! Regular expression compiling
+ * @retval -1 Error
+ * @retval  0 regex problem (no match?)
+ * @retval  1 OK Match
+ * @see match_regexp the CLIgen original composite function
+ */
+static int
+regex_compile(char    *pattern0,
+	      regex_t *re)
+{
+    int  retval = -1;
+    char pattern[1024];
+    //    char errbuf[1024];
+    int  len0;
+    int  status;
+
+    len0 = strlen(pattern0);
+    if (len0 > sizeof(pattern)-5){
+	clicon_err(OE_XML, EINVAL, "pattern too long");
+	goto done;
+    }
+    strncpy(pattern, "^(", 2);
+    strncpy(pattern+2, pattern0, sizeof(pattern)-2);
+    strncat(pattern, ")$",  sizeof(pattern)-len0-1);
+    if ((status = regcomp(re, pattern, REG_NOSUB|REG_EXTENDED)) != 0) {
+#if 0 /* ignore error msg for now */
+	regerror(status, re, errbuf, sizeof(errbuf));
+#endif
+	goto fail;
+    }
+    retval = 1;
+ done:
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
+/*! Regular expression execution
+ * @retval -1 Error
+ * @retval  0 regex problem (no match?)
+ * @retval  1 OK Match
+ * @see match_regexp the CLIgen original composite function
+ */
+static int
+regex_exec(regex_t *re,
+	   char    *string)
+{
+    int retval = -1;
+    int status;
+    //    char errbuf[1024];
+    
+    status = regexec(re, string, (size_t) 0, NULL, 0);
+    if (status != 0) {
+#if 0 /* ignore error msg for now */
+	regerror(status, re, errbuf, sizeof(errbuf));
+#endif
+	goto fail;
+    }
+    retval = 1;
+ done:
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
+
 /* return 1 if built-in, 0 if not */
 static int
 yang_builtin(char *type)
 {
-    if (clicon_str2int(ytmap, type) != -1)
+    if (clicon_str2int_search(ytmap2, type, (sizeof(ytmap)/sizeof(map_str2int))-2) != -1)
 	return 1;
     return 0;
 }
@@ -250,7 +346,7 @@ yang2cv_type(char         *ytype,
 
     *cv_type = CGV_ERR;
     /* built-in types */
-    if ((ret = clicon_str2int(ytmap, ytype)) != -1){
+    if ((ret = clicon_str2int_search(ytmap2, ytype, (sizeof(ytmap)/sizeof(map_str2int))-2)) != -1){
 	*cv_type = ret;
 	return 0;
     }
@@ -526,14 +622,41 @@ cv_validate1(cg_var      *cv,
 	}
 	if ((options & YANG_OPTIONS_PATTERN) != 0){
 	    char *posix = NULL;
-	    if (regexp_xsd2posix(pattern, &posix) < 0)
+	    regex_t *re = NULL;
-		goto done;
+
-	    if ((retval2 = match_regexp(str?str:"", posix)) < 0){
+	    if ((re = yang_regex_cache_get(yrestype)) == NULL){
-		clicon_err(OE_DB, 0, "match_regexp: %s", pattern);
+		/* Transform to posix regex */
-		return -1;
+		if (regexp_xsd2posix(pattern, &posix) < 0)
+		    goto done;
+		/* Create regex cache */
+		if ((re = malloc(sizeof(*re))) == NULL){
+		    clicon_err(OE_UNIX, errno, "malloc");		    
+		    goto done;
+		}
+		memset(re, 0, sizeof(*re));
+		/* Compute regex pattern for use in patterns */
+		if ((retval2 = regex_compile(posix, re)) < 0)
+		    goto done;
+		if (retval2 == 0){
+		    if (reason)
+			*reason = cligen_reason("regexp match fail: \"%s\" does not match %s",
+						str, pattern);
+		    goto fail;
+		    break;
+		}
+		yang_regex_cache_set(yrestype, re);
+		if (posix)
+		    free(posix);
+	    }
+	    if ((retval2 = regex_exec(re, str?str:"")) < 0)
+		goto done;
+	    if (retval2 == 0){
+		if (reason)
+		    *reason = cligen_reason("regexp match fail: \"%s\" does not match %s",
+					    str, pattern);
+		goto fail;
+		break;
 	    }
-	    if (posix)
-		free(posix);
 	    if (retval2 == 0){
 		if (reason)
 		    *reason = cligen_reason("regexp match fail: \"%s\" does not match %s",
@@ -668,7 +791,7 @@ ys_cv_validate_union(yang_stmt *ys,
 /*! Validate cligen variable cv using yang statement as spec
  *
  * @param[in]  cv      A cligen variable to validate. This is a correctly parsed cv.
- * @param[in]  ys      A yang statement, must be leaf of leaf-list.
+ * @param[in]  ys      A yang statement, must be leaf or leaf-list.
  * @param[out] reason  If given, and if return value is 0, contains a malloced string
  *                      describing the reason why the validation failed. Must be freed.
  * @retval -1  Error (fatal), with errno set to indicate error

test/README.md

@@ -60,6 +60,10 @@ The `mem.sh` runs memory checks using valgrind. Start it with no arguments to te
   mem.sh restconf backend # Only backend and cli
 ```
 
+## Performance plots
+
+The script `plot_perf.sh` produces gnuplots for some testcases.
+
 ## Site.sh
 You may add your site-specific modifications in a `site.sh` file. Example:
 ```

test/lib.sh

@@ -201,6 +201,7 @@ new(){
 # - expected command return value (0 if OK)
 # - expected stdout outcome,
 # - expected2 stdout outcome,
+# Example: expectfn "$clixon_cli -1 -f $cfg show conf cli" 0 "^$"
 expectfn(){
     cmd=$1
     retval=$2
@@ -221,7 +222,7 @@ expectfn(){
     if [ $r != $retval ]; then
 	echo -e "\e[31m\nError ($r != $retval) in Test$testnr [$testname]:"
 	echo -e "\e[0m:"
-	return
+	exit -1
     fi
 #    if [ $r != 0 ]; then
 #	return

test/plot_perf.sh

@@ -1,13 +1,48 @@
 #!/bin/bash
-# Transactions per second for large lists read/write plotter using gnuplot
+# Performance of large lists. See large-lists.md
-# WORK IN PROGRESS
+# The parameters are shown below (under Default values)
-. ./lib.sh
+# Examples
-max=2000 # Nr of db entries
+# 1. run all measurements up to 10000 entris collect all results in /tmp/plots
-step=200
+#    run=true plot=false to=10000 resdir=/tmp/plots ./plot_perf.sh 
-reqs=500
+# 2. Use existing data plot and show on X11
+#    run=false plot=true resdir=/tmp/plots term=x11 ./plot_perf.sh
+# 3. Use existing data plot i686 and armv7l data as png
+#    archs="i686 armv7l" run=false plot=true resdir=/tmp/plots term=png ./plot_perf.sh 
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+arch=$(arch)
+# Default values
+: ${to:=5000}  # Max N
+: ${step=1000} # Iterate in steps (also starting point)
+: ${reqs=100} # Number of requests in each burst
+: ${run:=true} # run tests (or skip them). If false just plot
+: ${term:=x11} # x11 interactive, alt: png
+: ${resdir=$dir} # Result dir (both data and gnuplot)
+: ${plot=false} # Result dir (both data and gnuplot)
+: ${archs=$arch} # Plotting can be made for many architectures (not run)
+
+# 0 prefix to protect against shell dynamic binding)
+to0=$to
+step0=$step
+reqs0=$reqs
+
+ext=$term # gnuplot output file extenstion
+
+# Global variables
+APPNAME=example
 cfg=$dir/plot-conf.xml
 fyang=$dir/plot.yang
-fconfig=$dir/config
+fxml=$dir/data.xml
+fjson=$dir/data.json
+
+# Resultdir - if different from $dir that gets erased
+#resdir=$dir
+
+if [ ! -d $resdir ]; then
+    mkdir $resdir
+fi
 
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
@@ -15,149 +50,479 @@ fconfig=$dir/config
 clixon_netconf=clixon_netconf
 
 cat <<EOF > $fyang
-module ietf-ip{
+module scaling{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix sc;
    container x {
-    list y {
+      description "top-level container";
-      key "a";
+      list y {
-      leaf a {
+         description "List with potential large number of elements";
-        type string;
+         key "a";
+         leaf a {
+            description "key in list";
+            type int32;
+         }
+         leaf b {
+            description "payload data";
+           type string;
+         }
       }
-      leaf b {
+   }
-        type string;
-      }
-    }
-    leaf-list c {
-       type string;
-    }
-  }
 }
 EOF
 
 cat <<EOF > $cfg
-<config>
+<clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
-  <CLICON_YANG_DIR>$fyang</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
-  <CLICON_YANG_MODULE_MAIN>ietf-ip</CLICON_YANG_MODULE_MAIN>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
-  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_YANG_MODULE_MAIN>scaling</CLICON_YANG_MODULE_MAIN>
-  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_SOCK>/usr/local/var/example/example.sock</CLICON_SOCK>
-<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/example/example.pidfile</CLICON_BACKEND_PIDFILE>
-  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
-</config>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
+</clixon-config>
 EOF
 
-run(){
+# Generate file with n entries
-    nr=$1 # Number of entries in DB
+# argument: <n> <proto>
-    reqs=$2
+genfile(){
-    mode=$3
+    if [ $2 = netconf ]; then
+	echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fxml
+	for (( i=0; i<$1; i++ )); do  
+	    echo -n "<y><a>$i</a><b>$i</b></y>" >> $fxml
+	done
+	echo "</x></config></edit-config></rpc>]]>]]>" >> $fxml    
+    else # restconf
+	echo -n '{"scaling:x":{"y":[' > $fjson
+	for (( i=0; i<$1; i++ )); do  
+	    if [ $i -ne 0 ]; then
+	    	echo -n ',' >> $fjson
+	    fi
+	    echo -n "{\"a\":$i,\"b\":\"$i\"}" >> $fjson
+	done
+	echo ']}}' >> $fjson
+    fi
+}
 
-    echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x>" > $fconfig
+# Run netconffunction
-    for (( i=0; i<$nr; i++ )); do  
+# args: <op> <proto> <load> <n> <reqs>
-	case $mode in
+# where proto is one of:
-	    readlist|writelist|restreadlist|restwritelist)
+#   netconf, restconf
-		echo -n "<y><a>$i</a><b>$i</b></y>" >> $fconfig
+# where op is one of:
-		;;
+#   get put delete commit
-	    writeleaflist)
+runnet(){
-		echo -n "<c>$i</c>" >> $fconfig
+    op=$1
-		;;
+    nr=$2 # Number of entries in DB (keep diff from n due to shell dynamic binding)
-	    esac
+    reqs=$3
-    done
 
-    echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig    
+    file=$resdir/$op-netconf-$reqs-$arch
-
+    echo -n "$nr " >>  $file
-    expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+    case $op in
-
+	put)
-    case $mode in
+	    if [ $reqs = 0 ]; then # Write all in one go
-	readlist)
+		genfile $nr netconf;
-	time -p for (( i=0; i<$reqs; i++ )); do
+		{ time -p cat $fxml | $clixon_netconf -qf $cfg -y $fyang ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
-    rnd=$(( ( RANDOM % $nr ) ))
+	    else # reqs != 0
-    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
+		{ time -p for (( i=0; i<$reqs; i++ )); do
-done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
+	rnd=$(( ( RANDOM % $nr ) ));
-            ;;
+	echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>";
-	writelist)
+    done | $clixon_netconf -qf $cfg -y $fyang > /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
-	    time -p for (( i=0; i<$reqs; i++ )); do
+	    fi
-	rnd=$(( ( RANDOM % $nr ) ))
-	echo "<rpc><edit-config><target><candidate/></target><config><x><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
-done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
-    ;;
-    	restreadlist)
-	time -p for (( i=0; i<$reqs; i++ )); do
-    rnd=$(( ( RANDOM % $nr ) ))
-    curl -sSG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null
-done 
-            ;;
-    writeleaflist)
-	time -p for (( i=0; i<$reqs; i++ )); do
-	rnd=$(( ( RANDOM % $nr ) ))
-	echo "<rpc><edit-config><target><candidate/></target><config><x><c>$rnd</c></x></config></edit-config></rpc>]]>]]>"
-done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
 	    ;;
+	get)
+	    if [ $reqs = 0 ]; then # Read all in one go
+		{ time -p  echo "<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>" | $clixon_netconf -qf $cfg -y $fyang > /dev/null ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
+	    else # reqs != 0
+		{ time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ))
+	echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
+		done | $clixon_netconf -qf $cfg -y $fyang > /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
+	    fi
+	    ;;
+	delete)
+	    { time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ))
+	echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg -y $fyang; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
+	    ;;
+	commit)
+	    { time -p  echo "<rpc><commit/></rpc>]]>]]>" | $clixon_netconf -qf $cfg -y $fyang > /dev/null ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
+	    ;;
+	    *)
+	err "Operation not supported" "$op"
+	exit
+	;;
     esac
-    expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
-
 }
 
-step(){
+# Run restconf function
-    i=$1
+# args: <op> <proto> <load> <n> <reqs>
-    mode=$2
+# where proto is one of:
-    echo -n "" > $fconfig
+#   netconf, restconf
-    t=$(TEST=%e run $i $reqs $mode 2>&1 | awk '/real/ {print $2}')
+# where op is one of:
-    #TEST=%e run $i $reqs $mode 2>&1 
+#   get put delete 
-    #  t is time in secs of $reqs -> transactions per second. $reqs
+runrest(){
-    p=$(echo "$reqs/$t" | bc -lq)
+    op=$1
-    # p is transactions per second. 
+    nr=$2 # Number of entries in DB
-    echo "$i $p" >> $dir/$mode
+    reqs=$3
-#    echo "m:$mode i:$i t=$t p=$p" 
+    
+    file=$resdir/$op-restconf-$reqs-$arch
+    echo -n "$nr " >>  $file
+    case $op in
+	put)
+	    if [ $reqs = 0 ]; then # Write all in one go
+		genfile $nr restconf
+		# restconf @- means from stdin
+		{ time -p curl -sS -X PUT -d @$fjson http://localhost/restconf/data/scaling:x ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
+	    else # Small requests
+		{ time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ));
+	curl -sS -X PUT http://localhost/restconf/data/scaling:x/y=$rnd -d "{\"scaling:y\":{\"a\":$rnd,\"b\":\"$rnd\"}}" 
+    done ; } 2>&1 | awk '/real/ {print $2}' | tr , .>> $file
+		# 
+	    fi
+	    ;;
+	get)
+	    if [ $reqs = 0 ]; then # Read all in one go
+		{ time -p curl -sS -X GET http://localhost/restconf/data/scaling:x > /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file
+	    else # Small requests
+		{ time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ));
+	curl -sS -X GET http://localhost/restconf/data/scaling:x/y=$rnd  
+    done ; } 2>&1 | awk '/real/ {print $2}' | tr , .>> $file
+	    fi
+	    ;;
+	delete)
+		{ time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ));
+	curl -sS -X GET http://localhost/restconf/data/scaling:x/y=$rnd  
+    done ; } 2>&1 | awk '/real/ {print $2}' | tr , .>> $file
+		;;
+	    *)
+	err "Operation not supported" "$op"
+	exit
+	;;
+    esac
 }
 
-once(){
+
-    # kill old backend (if any)
+commit(){
-    sudo clixon_backend -zf $cfg -y $fyang
+    # commit to running
-    if [ $? -ne 0 ]; then
+    expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
-	err
+}
+
+reset(){
+    # delete all in candidate
+    expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><edit-config><target><candidate/></target><default-operation>none</default-operation><config operation='delete'/></edit-config></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]>$'
+    # commit to running
+    commit
+}
+
+# Load n entries into candidate
+# Args: <n>
+load(){
+    # Generate file ($fxml)
+    genfile $1 netconf
+    # Write it to backend in one chunk
+    expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fxml" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+}
+
+# Run an operation, iterate from <from> to <to> in increment of <step> 
+# Each operation do <reqs> times
+# args: <op> <protocol> <from> <step> <to> <reqs> <cand> <run>
+# <reqs>=0 means all in one go
+# <cand> <run> means a priori loaded into datastore
+plot(){
+    op=$1
+    proto=$2
+    from=$3
+    step=$4
+    to=$5
+    reqs=$6
+    can=$7
+    run=$8
+
+    if [ $# -ne 8 ]; then
+	exit "plot should be called with 8 arguments, got $#"
     fi
     
-    # start new backend
+    # reset file
-    sudo clixon_backend -s init -f $cfg -y $fyang
+    new "Create file $resdir/$op-$proto-$reqs-$arch"
-    if [ $? -ne 0 ]; then
+    echo -n "" > $resdir/$op-$proto-$reqs-$arch
-	err
+    for (( n=$from; n<=$to; n=$n+$step )); do  
-    fi
+	reset
-
+	if [ $can = n ]; then 
-    # Always as a start
+	    load $n
-    for (( i=10; i<=$step; i=i+10 )); do  
+	    if [ $run = n ]; then 
-	step $i readlist
+		commit
-	step $i writelist
+	    fi
-	step $i restreadlist
+	fi
-	step $i writeleaflist
+	new "$op-$proto-$reqs-$arch $n"
+	if [ $proto = netconf ]; then
+	    runnet $op $n $reqs
+	else
+	    runrest $op $n $reqs
+	fi
     done
-    # Actual steps
+    echo # newline
-    for (( i=$step; i<=$max; i=i+$step )); do  
+}
-	step $i readlist
+
-	step $i writelist
+# Run an operation, iterate from <from> to <to> in increment of <step> 
-	step $i restreadlist
+# Each operation do <reqs> times
-	step $i writeleaflist
+# args: <op> <protocol> <from> <step> <to> <reqs> <cand> <run>
+# <reqs>=0 means all in one go
+startup(){
+    from=$1
+    step=$2
+    to=$3
+    mode=startup
+
+    if [ $# -ne 3 ]; then
+	exit "plot should be called with 3 arguments, got $#"
+    fi
+
+    # gnuplot file
+    gfile=$resdir/startup-$arch    
+    new "Create file $gfile"
+    echo -n "" > $gfile
+
+    # Startup db: load with n entries
+    dbfile=$dir/${mode}_db
+    sudo touch $dbfile
+    sudo chmod 666 $dbfile
+    for (( n=$from; n<=$to; n=$n+$step )); do  
+	new "startup-$arch $n"
+	new "Generate $n entries to $dbfile"
+	echo -n "<config><x xmlns=\"urn:example:clixon\">" > $dbfile
+	for (( i=0; i<$n; i++ )); do  
+	    echo -n "<y><a>$i</a><b>$i</b></y>" >> $dbfile
+	done
+	echo "</x></config>" >> $dbfile
+
+	new "Startup backend once -s $mode -f $cfg -y $fyang"
+	echo -n "$n " >>  $gfile
+	{ time -p sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang 2> /dev/null; } 2>&1 |  awk '/real/ {print $2}' | tr , . >> $gfile
+
+    done
+    echo # newline
+}
+
+if $run; then
+
+    # Startup test before regular backend/restconf start since we only start
+    # backend a single time
+    startup $step $step $to
+
+    new "test params: -f $cfg -y $fyang"
+    if [ $BE -ne 0 ]; then
+	new "kill old backend"
+	sudo clixon_backend -zf $cfg -y $fyang
+	if [ $? -ne 0 ]; then
+	    err
+	fi
+	new "start backend -s init -f $cfg -y $fyang"
+	start_backend -s init -f $cfg -y $fyang
+    fi
+
+    new "kill old restconf daemon"
+    sudo pkill -u www-data -f "/www-data/clixon_restconf"
+
+    new "start restconf daemon"
+    start_restconf -f $cfg -y $fyang
+
+    new "waiting"
+    sleep $RCWAIT
+
+
+    to=$to0
+    step=$step0
+    reqs=$reqs0
+
+
+    # Put all tests
+    for proto in netconf restconf; do
+	new "$proto put all entries to candidate (restconf:running)"
+	plot put  $proto  $step $step $to 0 0 0 # all candidate 0 running 0
     done
 
-    # Check if still alive
+    # Get all tests
-    pid=`pgrep clixon_backend`
+    for proto in netconf restconf; do
-    if [ -z "$pid" ]; then
+	new "$proto get all entries from running"
-	err "backend already dead"
+	plot get $proto  $step $step $to 0 n n # start w full datastore
-    fi
+    done
-    # kill backend
-    sudo clixon_backend -zf $cfg
-    if [ $? -ne 0 ]; then
-	err "kill backend"
-    fi
-}
 
-once
+    # Netconf commit all
+    new "Netconf commit all entries from candidate to running"
+    plot commit netconf $step $step $to 0 n 0 # candidate full running empty
+
+    # Transactions get/put/delete
+    reqs=$reqs0
+    for proto in netconf restconf; do
+	new "$proto get $reqs from full database"
+	plot get $proto $step $step $to $reqs n n 
+
+	new "$proto put $reqs to full database(replace / alter values)"
+	plot put $proto $step $step $to $reqs n n
+
+	new "$proto delete $reqs from full database(replace / alter values)"
+	plot delete $proto $step $step $to $reqs n n
+    done
+
+    new "Kill restconf daemon"
+    stop_restconf
+
+    if [ $BE -ne 0 ]; then
+	new "Kill backend"
+	# Check if premature kill
+	pid=`pgrep -u root -f clixon_backend`
+	if [ -z "$pid" ]; then
+	    err "backend already dead"
+	fi
+	# kill backend
+	stop_backend -f $cfg
+    fi
+fi # if run
+
+if $plot; then
+
+# 0. Startup
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/startup-$a\" title \"startup-$a\","
+done
 
 gnuplot -persist <<EOF
-set title "Clixon transactions per second r/w large lists" font ",14" textcolor rgbcolor "royalblue"
+set title "Clixon startup"
-set xlabel "entries"
+set style data linespoint
-set ylabel "transactions per second"
+set xlabel "Entries"
-set terminal wxt  enhanced title "Clixon transactions " persist raise
+set ylabel "Time[s]"
-plot "$dir/readlist" with linespoints title "read list", "$dir/writelist" with linespoints title "write list", "$dir/writeleaflist" with linespoints title "write leaf-list" , "$dir/restreadlist" with linespoints title "rest get list" 
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-startup.$term"
+plot $gplot
 EOF
 
-rm -rf $dir
+# 1. Get config
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/get-restconf-0-$a\" title \"rc-$a\", \"$resdir/get-netconf-0-$a\" title \"nc-$a\","
+done
+
+gnuplot -persist <<EOF
+set title "Clixon get config"
+set style data linespoint
+set xlabel "Entries"
+set ylabel "Time[s]"
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-get-0.$term"
+plot $gplot
+EOF
+
+
+# 2. Put config
+
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/put-restconf-0-$a\" title \"rc-$a\", \"$resdir/put-netconf-0-$a\" title \"nc-$a\","
+done
+
+gnuplot -persist <<EOF
+set title "Clixon put config"
+set style data linespoint
+set xlabel "Entries"
+set ylabel "Time[s]"
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-put-0.$term"
+plot $gplot
+EOF
+
+# 3. Commit config
+
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/commit-netconf-0-$a\" title \"nc-$a\","
+done
+
+gnuplot -persist <<EOF
+set title "Clixon commit config"
+set style data linespoint
+set xlabel "Entries"
+set ylabel "Time[s]"
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-commit-0.$term"
+plot $gplot
+EOF
+
+# 4. Get single entry 
+
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/get-restconf-100-$a\" using 1:(\$2/$reqs0) title \"rc-$a\", \"$resdir/get-netconf-100-$a\" using 1:(\$2/$reqs0) title \"nc-$a\","
+done
+
+gnuplot -persist <<EOF
+set title "Clixon get single entry"
+set style data linespoint
+set xlabel "Entries"
+set ylabel "Time[s]"
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-get-100.$term"
+plot $gplot
+EOF
+
+# 5. Put single entry 
+
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/put-restconf-100-$a\" using 1:(\$2/$reqs0) title \"rc-$a\", \"$resdir/put-netconf-100-$a\" using 1:(\$2/$reqs0) title \"nc-$a\","
+done
+
+gnuplot -persist <<EOF
+set title "Clixon put single entry"
+set style data linespoint
+set xlabel "Entries"
+set ylabel "Time[s]"
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-put-100.$term"
+plot $gplot
+EOF
+
+# 6. Delete single entry 
+
+gplot=""
+for a in $archs; do
+    gplot="$gplot \"$resdir/delete-restconf-100-$a\" using 1:(\$2/$reqs0) title \"rc-$a\", \"$resdir/delete-netconf-100-$a\" using 1:(\$2/$reqs0) title \"nc-$a\","
+
+done
+
+
+gnuplot -persist <<EOF
+set title "Clixon delete single entry"
+set style data linespoint
+set xlabel "Entries"
+set ylabel "Time[s]"
+set grid
+set terminal $term
+set yrange [*:*]
+set output "$resdir/clixon-delete-100.$term"
+plot $gplot
+EOF
+
+fi # if plot
+
+#rm -rf $dir
 

test/test_insert.sh

@@ -0,0 +1,215 @@
+#!/bin/bash
+# XML Insert unit test
+# First a list with 0-5 base elements, insert in different places
+# Second varying yangs: container, leaf, list, leaf-list, choice, user-order list
+
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+: ${clixon_util_insert:=clixon_util_insert}
+
+OPTS="-D $DBG -s"
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyang=$dir/example.yang
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_CACHE>true</CLICON_XMLDB_CACHE>
+</clixon-config>
+EOF
+
+cat <<EOF > $fyang
+module example {
+    yang-version 1.1;
+    namespace "urn:example:example";
+    prefix ex;
+    revision 2019-01-13;
+    container c{
+      list a{
+	key x;
+	leaf x{
+	    type int32;
+	}
+      }
+    }
+}
+EOF
+
+testrun(){
+    x0=$1
+    xi="<c xmlns=\"urn:example:example\">$2</c>"
+    xp=c
+
+    new "random list add leaf-list"
+    # First run sorted (assume this is the refernce == correct)
+    rs=$($clixon_util_insert -y $fyang -x "$xi" -b "$x0" -p $xp $OPTS -s)
+    # Then run actual insert
+    r0=$($clixon_util_insert -y $fyang -x "$xi" -b "$x0" -p $xp $OPTS)
+    # If both are null something is amiss
+    if [ -z "$r0" -a -z "$rs" ]; then
+	err "length of retval is zero"
+    fi
+ #   echo "rs:$rs"
+#    echo "r0:$r0"
+    # Check they are equal
+    if [[ "$r0" != "$rs" ]]; then
+	err "$rs" "$r0"
+    fi
+}
+
+new "test params: -y $fyang $OPTS"
+
+# Empty element base list
+x0='<c xmlns="urn:example:example"></c>'
+new "empty list"
+testrun "$x0" "<a><x>1</x></a>"
+
+# One element base list
+x0='<c xmlns="urn:example:example"><a><x>99</x></a></c>'
+new "one element list first"
+testrun "$x0" "<a><x>1</x></a>"
+
+new "one element list last"
+testrun "$x0" "<a><x>100</x></a>"
+
+# Two element base list
+x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a></c>'
+new "two element list first"
+testrun "$x0" "<a><x>1</x></a>"
+
+new "two element list mid"
+testrun "$x0" "<a><x>12</x></a>"
+
+new "two element list last"
+testrun "$x0" "<a><x>3000</x></a>"
+
+# Three element base list
+x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a><a><x>101</x></a></c>'
+new "three element list first"
+testrun "$x0" "<a><x>1</x></a>"
+
+new "three element list second"
+testrun "$x0" "<a><x>10</x></a>"
+
+new "three element list third"
+testrun "$x0" "<a><x>100</x></a>"
+
+new "three element list last"
+testrun "$x0" "<a><x>1000</x></a>"
+
+# Four element base list
+x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a><a><x>101</x></a><a><x>200</x></a></c>'
+
+new "four element list first"
+testrun "$x0" "<a><x>1</x></a>"
+
+new "four element list second"
+testrun "$x0" "<a><x>10</x></a>"
+
+new "four element list third"
+testrun "$x0" "<a><x>100</x></a>"
+
+new "four element list fourth"
+testrun "$x0" "<a><x>102</x></a>"
+
+new "four element list last"
+testrun "$x0" "<a><x>1000</x></a>"
+
+# Five element base list
+x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a><a><x>101</x></a><a><x>200</x></a><a><x>300</x></a></c>'
+
+new "five element list first"
+testrun "$x0" "<a><x>1</x></a>"
+
+new "five element list mid"
+testrun "$x0" "<a><x>100</x></a>"
+
+new "five element list last"
+testrun "$x0" "<a><x>1000</x></a>"
+
+cat <<EOF > $fyang
+module example {
+    yang-version 1.1;
+    namespace "urn:example:example";
+    prefix ex;
+    revision 2019-01-13;
+    container c{
+      leaf a{
+        type string;
+      }
+      container b{
+        leaf a {
+          type string;
+        }
+      }
+      choice c1{ 
+        case a{
+          leaf x{
+            type string;
+          }
+        }
+        case b{
+          leaf y{
+            type int32;
+          }
+        }
+      }
+      choice c2{ 
+        leaf z{
+            type string;
+        }
+        leaf t{
+            type int32;
+          }
+      }
+      list d{
+	key x;
+	leaf x{
+	    type int32;
+	}
+        ordered-by user;
+      }
+      leaf-list e{
+        type int32;
+      }
+    }
+}
+EOF
+
+# Advanced list
+# Empty base list
+x0='<c xmlns="urn:example:example"></c>'
+xp=c
+new "adv empty list add leaf"
+testrun "$x0" "<a>leaf</a>"
+
+new "adv empty list add choice c1"
+testrun "$x0" "<x>choice1</x>"
+
+xi='<c xmlns="urn:example:example"><e>33</e></c>'
+new "adv empty list add leaf-list"
+testrun "$x0" "<e>33</e>"
+
+# base list
+x0='<c xmlns="urn:example:example"><a>leaf</a><x>choice</x><d><x>1</x></d><e>33</e></c>'
+
+new "adv list add leaf-list"
+testrun "$x0" "<e>32</e>"
+
+new "adv list add leaf-list"
+testrun "$x0" "<e>32</e>"
+
+rm -rf $dir

test/test_perf.sh

@@ -5,7 +5,7 @@
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
 # Number of list/leaf-list entries in file
-: ${perfnr:=1000}
+: ${perfnr:=20000}
 
 # Number of requests made get/put
 : ${perfreq:=100}
@@ -15,6 +15,7 @@ APPNAME=example
 cfg=$dir/scaling-conf.xml
 fyang=$dir/scaling.yang
 fconfig=$dir/large.xml
+fconfig2=$dir/large2.xml
 
 cat <<EOF > $fyang
 module scaling{
@@ -43,16 +44,49 @@ cat <<EOF > $cfg
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
-  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>scaling</CLICON_YANG_MODULE_MAIN>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
-  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/example/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
-  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
+  <CLICON_CLI_MODE>example</CLICON_CLI_MODE>
+  <CLICON_CLI_DIR>/usr/local/lib/example/cli</CLICON_CLI_DIR>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
+  <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
 </clixon-config>
 EOF
 
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg -y $fyang
+    if [ $? -ne 0 ]; then
+	err
+    fi
+fi
+# Try startup mode w startup
+for mode in startup running; do
+    file=$dir/${mode}_db
+    sudo touch $file
+    sudo chmod 666 $file
+    new "generate large startup config ($file) with $perfnr list entries in mode $mode"
+    echo -n "<config><x xmlns=\"urn:example:clixon\">" > $file
+    for (( i=0; i<$perfnr; i++ )); do  
+	echo -n "<y><a>$i</a><b>$i</b></y>" >> $file
+    done
+    echo "</x></config>" >> $file
+
+    new "Startup backend once -s $mode -f $cfg -y $fyang"
+    # Cannot use start_backend here due to expected error case
+    time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
+done
+
+new "Startup backend once -s $mode -f $cfg -y $fyang"
+# Cannot use start_backend here due to expected error case
+time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
+
 new "test params: -f $cfg -y $fyang"
 if [ $BE -ne 0 ]; then
     new "kill old backend"
@@ -86,13 +120,9 @@ new "netconf write large config"
 expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 # Here, there are $perfnr entries in candidate
-
 new "netconf write large config again"
 expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
-# Remove the file, its used for different purposes further down
-rm $fconfig
-
 # Now commit it from candidate to running 
 new "netconf commit large config"
 expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
@@ -119,7 +149,7 @@ done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 new "restconf get $perfreq small config"
 time -p for (( i=0; i<$perfreq; i++ )); do
     rnd=$(( ( RANDOM % $perfnr ) ))
-    curl -sG http://localhost/restconf/data/scaling:x/y=$rnd,$rnd > /dev/null
+    curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null
 done
 
 new "restconf add $perfreq small config"
@@ -135,19 +165,23 @@ expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg  -y $fyang" 0 "<rpc><get
 new "restconf get large config"
 expecteof "/usr/bin/time -f %e curl -sG http://localhost/restconf/data" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" '^{"data": {"scaling:x": {"y": \[{"a": 0,"b": 0},{ "a": 1,"b": 1},{ "a": 2,"b": 2},{ "a": 3,"b": 3},'
 
+new "restconf delete $perfreq small config"
+time -p for (( i=0; i<$perfreq; i++ )); do
+    rnd=$(( ( RANDOM % $perfnr ) ))
+    curl -s -X DELETE http://localhost/restconf/data/scaling:x/y=$rnd
+done 
+
 # Now do leaf-lists istead of leafs
 
 new "generate large leaf-list config"
-echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fconfig
+echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fconfig2
 for (( i=0; i<$perfnr; i++ )); do  
-    echo -n "<c>$i</c>" >> $fconfig
+    echo -n "<c>$i</c>" >> $fconfig2
 done
-echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
+echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig2
 
 new "netconf replace large list-leaf config"
-expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig2" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
-
-rm $fconfig
 
 new "netconf commit large leaf-list config"
 expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
@@ -183,4 +217,5 @@ fi
 # kill backend
 stop_backend -f $cfg
 
+
 rm -rf $dir

test/test_restconf.sh

@@ -58,7 +58,7 @@ sleep $RCWAIT
 new "restconf tests"
 
 new "restconf root discovery. RFC 8040 3.1 (xml+xrd)"
-expecteq "$(curl  -s -X GET http://localhost/.well-known/host-meta)" 0 "<XRD xmlns='http://docs.oasis-open.org/ns/xri/xrd-1.0'>
+expecteq "$(curl -s -X GET http://localhost/.well-known/host-meta)" 0 "<XRD xmlns='http://docs.oasis-open.org/ns/xri/xrd-1.0'>
    <Link rel='restconf' href='/restconf'/>
 </XRD>
"
 

test/test_restconf2.sh

@@ -95,7 +95,7 @@ expectfn 'curl -s -X POST -d {"example:cont1":{"interface":{"type":"regular"}}}
 new "restconf POST initial tree"
 expectfn 'curl -s -X POST -d {"example:cont1":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' 0 ""
 
-new "restconf GET datastore intial"
+new "restconf GET datastore initial"
 expectfn "curl -s -X GET http://localhost/restconf/data/example:cont1" 0 '{"example:cont1": {"interface": \[{"name": "local0","type": "regular"}\]}}'
 
 new "restconf GET interface subtree"

test/test_startup.sh

@@ -6,6 +6,9 @@
 # - An extra xml configuration file starts with an "extra" interface
 # - running db starts with a "run" interface
 # - startup db starts with a "start" interface
+# There is also an "invalid" XML and a "broken" XML
+# There are two steps, first run through everything OK
+# Then try with invalid and borken XML and ensure the backend quits and all is untouched
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@@ -36,27 +39,37 @@ cat <<EOF > $cfg
 
 EOF
 
-# Create running-db containin the interface "run"
+# Create running-db containin the interface "run" OK
 runvar='<interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces>'
 
-# Create startup-db containing the interface "startup"
+# Create startup-db containing the interface "startup" OK
 startvar='<interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces>'
 
-# extra
+# extra OK
 extravar='<interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces>'
 
+# invalid (contains <not-defined/>), but OK XML syntax
+invalidvar='<interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><not-defined/><name>invalid</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces>'
+
+# Broken XML (contains </nmae>)
+brokenvar='<interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>broken</nmae><type>ex:eth</type><enabled>true</enabled></interface></interfaces>'
+
 # Create a pre-set running, startup and (extra) config.
 # The configs are identified by an interface called run, startup, extra.
 # Depending on startup mode (init, none, running, or startup)
 # expect different output of an initial get-config of running
 testrun(){
     mode=$1
-    exprun=$2 # expected running_db after startup
+    rdb=$2    # running db at start
+    sdb=$3    # startup db at start
+    edb=$4    # extradb at start
+    exprun=$5 # expected running_db after startup
+
 
     sudo rm -f  $dir/*_db
-    echo "<config>$runvar</config>" > $dir/running_db
+    echo "<config>$rdb</config>" > $dir/running_db
-    echo "<config>$startvar</config>" > $dir/startup_db
+    echo "<config>$sdb</config>" > $dir/startup_db
-    echo "<config>$extravar</config>" > $dir/extra_db
+    echo "<config>$edb</config>" > $dir/extra_db
 
     if [ $BE -ne 0 ]; then     # Bring your own backend
 	# kill old backend (if any)
@@ -81,7 +94,7 @@ testrun(){
     expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$exprun</rpc-reply>]]>]]>$"
 
     new "Startup test for $mode mode, check startup is untouched"
-    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>' "^<rpc-reply><data>$startvar</data></rpc-reply>]]>]]>$"
+    expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>' "^<rpc-reply><data>$sdb</data></rpc-reply>]]>]]>$"
     
     new "Kill backend"
     # Check if premature kill
@@ -93,17 +106,79 @@ testrun(){
     stop_backend -f $cfg
 } # testrun
 
+
+# The backend should fail with 255 and all db:s should be unaffected
+testfail(){
+    mode=$1
+    rdb=$2    # running db at start
+    sdb=$3    # startup db at start
+    edb=$4    # extradb at start
+
+    sudo rm -f  $dir/*_db
+
+    echo "<config>$rdb</config>" > $dir/running_db
+    echo "<config>$sdb</config>" > $dir/startup_db
+    echo "<config>$edb</config>" > $dir/extra_db
+
+    # kill old backend (if any)
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+    new "start backend -f $cfg -s $mode -c $dir/extra_db"
+    ret=$(start_backend -1 -s $mode -f $cfg -c $dir/extra_db 2> /dev/null)
+    r=$?
+    if [ $r -ne 255 ]; then
+	err "Unexpected retval" $r
+    fi
+    # permission kludges
+    sudo chmod 666 $dir/running_db
+    sudo chmod 666 $dir/startup_db
+    new "Checking running unchanged"
+    ret=$(diff $dir/running_db <(echo "<config>$rdb</config>"))
+    if [ $? -ne 0 ]; then
+	err "<config>$rdb</config>" "$ret"
+    fi	
+    new "Checking startup unchanged"
+    ret=$(diff $dir/startup_db <(echo "<config>$sdb</config>"))
+    if [ $? -ne 0 ]; then
+	err "<config>$sdb</config>" "$ret"
+    fi
+
+    new "Checking extra unchanged"
+    ret=$(diff $dir/extra_db <(echo "<config>$edb</config>"))
+    if [ $? -ne 0 ]; then
+	err "<config>$edb</config>" "$ret"
+    fi
+}
+
+# 1. Try different modes on OK running/startup/extra
 # Init mode: delete running and reload from scratch (just extra)
-testrun init "<data>$extravar</data>"
+testrun init "$runvar" "$startvar" "$extravar" "<data>$extravar</data>" 
 
 # None mode: do nothing, running remains
-testrun none  "<data>$runvar</data>"
+testrun none  "$runvar" "$startvar" "$extravar" "<data>$runvar</data>"
 
 # Running mode: keep running but load also extra
-testrun running '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
+testrun running "$runvar" "$startvar" "$extravar" '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>run</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
 
 # Startup mode: scratch running, load startup with extra on top
-testrun startup '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>'
+testrun startup "$runvar" "$startvar" "$extravar" '<data><interfaces xmlns="urn:ietf:params:xml:ns:yang:ietf-interfaces"><interface><name>extra</name><type>ex:eth</type><enabled>true</enabled></interface><interface><name>startup</name><type>ex:eth</type><enabled>true</enabled></interface></interfaces></data>' 
 
-echo $dir
+# 2. Try different modes on Invalid running/startup/extra WITHOUT failsafe
-#rm -rf $dir
+# ensure all db:s are unchanged after failure.
+
+new "Test invalid running in running mode"
+testfail running "$invalidvar" "$startvar" "$extravar"
+
+new "Run invalid startup in startup mode"
+testfail startup "$runvar" "$invalidvar" "$extravar"
+
+new "Test broken running in running mode"
+testfail running "$brokenvar" "$startvar" "$extravar"
+
+new "Run broken startup in startup mode"
+testfail startup "$runvar" "$brokenvar" "$extravar"
+
+rm -rf $dir

util/Makefile.in

@@ -73,6 +73,7 @@ APPSRC   += clixon_util_json.c
 APPSRC   += clixon_util_yang.c
 APPSRC   += clixon_util_xpath.c
 APPSRC   += clixon_util_datastore.c
+APPSRC   += clixon_util_insert.c
 ifeq ($(with_restconf),yes)
 APPSRC   += clixon_util_stream.c # Needs curl
 endif
@@ -107,6 +108,9 @@ clixon_util_stream: clixon_util_stream.c $(LIBDEPS)
 clixon_util_datastore: clixon_util_datastore.c $(LIBDEPS)
 	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@
 
+clixon_util_insert: clixon_util_insert.c $(LIBDEPS)
+	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@
+
 distclean: clean
 	rm -f Makefile *~ .depend
 

util/clixon_util_insert.c

@@ -0,0 +1,215 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+See https://www.w3.org/TR/xpath/
+
+ * Turn this on to get an xpath test program 
+ * Usage: xpath [<xpath>] 
+ * read xpath on first line and xml on rest of lines from input
+ * Example compile:
+ gcc -g -o xpath -I. -I../clixon ./clixon_xsl.c -lclixon -lcligen
+ * Example run:
+echo "a\n<a><b/></a>" | xpath
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <limits.h>
+#include <fnmatch.h>
+#include <stdint.h>
+#include <assert.h>
+#include <syslog.h>
+#include <fcntl.h>
+
+/* cligen */
+#include <cligen/cligen.h>
+
+/* clixon */
+#include "clixon/clixon.h"
+
+static int
+usage(char *argv0)
+{
+    fprintf(stderr, "usage:%s [options]\n"
+	    "where options are\n"
+            "\t-h \t\tHelp\n"
+    	    "\t-D <level>\tDebug\n"
+	    "\t-y <file> \tYANG spec file (or stdin)\n"
+	    "\t-b <base> \tXML base expression\n"
+	    "\t-x <xml>  \tXML to insert\n"
+	    "\t-p <xpath>\tXpath to where in base and XML\n"
+	    "\t-s        \tSort output after insert\n"
+	    "Assume insert xml is first child of xpath. Ie if xml=<a><x>23 and xpath=a, then inserted element is <x>23\n",
+	    argv0
+	    );
+    exit(0);
+}
+
+int
+main(int argc, char **argv)
+{
+    int         retval = -1;
+    char       *argv0 = argv[0];
+    int         c;
+    char       *filename = NULL;
+    int         fd = 0; /* unless overriden by argv[1] */
+    char       *x0str = NULL;
+    char       *xistr = NULL;
+    char       *xpath = NULL;
+    yang_stmt  *yspec;
+    cxobj      *x0 = NULL;
+    cxobj      *xb;
+    cxobj      *xi = NULL;
+    int         sort = 0;
+    clicon_handle h;
+
+    clicon_log_init("clixon_insert", LOG_DEBUG, CLICON_LOG_STDERR); 
+    if ((h = clicon_handle_init()) == NULL)
+	goto done;
+    optind = 1;
+    opterr = 0;
+    while ((c = getopt(argc, argv, "hD:y:b:x:p:s")) != -1)
+	switch (c) {
+	case 'h':
+	    usage(argv0);
+	    break;
+    	case 'D':
+	    if (sscanf(optarg, "%d", &debug) != 1)
+		usage(argv0);
+	    break;
+	case 'y': /* YANG spec file */
+	    filename = optarg;
+	    if (0 && (fd = open(filename, O_RDONLY)) < 0){
+		clicon_err(OE_UNIX, errno, "open(%s)", argv[1]);
+		goto done;
+	    }
+	    break;
+	case 'b': /* Base XML expression */
+	    x0str = optarg;
+	    break;
+	case 'x': /* XML to insert */
+	    xistr = optarg;
+	    break;
+	case 'p': /* XPATH base */
+	    xpath = optarg;
+	    break;
+	case 's': /* sort output after insert */
+	    sort++;
+	    break;
+	default:
+	    usage(argv[0]);
+	    break;
+	}
+    if (xistr == NULL || x0str == NULL)
+	usage(argv0);
+    if (xpath == NULL)
+	usage(argv0);
+    if (filename == NULL)
+	usage(argv0);
+    clicon_debug(1, "xistr:%s", xistr);
+    clicon_debug(1, "x0str:%s", x0str);
+    clicon_debug(1, "xpath:%s", xpath);
+    if ((yspec = yspec_new()) == NULL)
+	goto done;
+#if 1
+    if (yang_spec_parse_file(h, filename, yspec) < 0)
+	goto done;
+#else
+    if (yang_parse_file(fd, "yang test", yspec) == NULL)
+	goto done;
+#endif
+    /* Parse base XML */
+    if (xml_parse_string(x0str, yspec, &x0) < 0){
+	clicon_err(OE_XML, 0, "Parsing base xml: %s", x0str);
+	goto done;
+    }
+    if (xml_apply(x0, CX_ELMNT, xml_spec_populate, yspec) < 0)
+	goto done;
+    if ((xb = xpath_first(x0, "%s", xpath)) == NULL){
+	clicon_err(OE_XML, 0, "xpath: %s not found in x0", xpath);
+	goto done;
+    }
+    if (debug){
+	clicon_debug(1, "xb:");
+	xml_print(stderr, xb);
+    }
+    /* Parse insert XML */
+    if (xml_parse_string(xistr, yspec, &xi) < 0){
+	clicon_err(OE_XML, 0, "Parsing insert xml: %s", xistr);
+	goto done;
+    }
+    if (xml_apply(xi, CX_ELMNT, xml_spec_populate, yspec) < 0)
+	goto done;
+    if ((xi = xpath_first(xi, "%s", xpath)) == NULL){
+	clicon_err(OE_XML, 0, "xpath: %s not found in xi", xpath);
+	goto done;
+    }
+    /* Find first element child */
+    if ((xi = xml_child_i_type(xi, 0, CX_ELMNT)) == NULL){
+	clicon_err(OE_XML, 0, "xi has no element child");
+	goto done;
+    }
+    /* Remove it from parent */
+    if (xml_rm(xi) < 0)
+    	goto done;
+    if (debug){
+	clicon_debug(1, "xi:");
+	xml_print(stderr, xi);
+    }
+    if (xml_insert(xb, xi) < 0) 
+	goto done;
+    if (debug){
+	clicon_debug(1, "x0:");
+	xml_print(stderr, x0);
+    }
+    if (sort)
+	xml_sort(xb, NULL);
+    clicon_xml2file(stdout, xb, 0, 0);
+
+    retval = 0;
+ done:
+    if (x0)
+	xml_free(x0);
+    if (yspec)
+	yspec_free(yspec);
+    if (fd > 0)
+	close(fd);
+    return retval;
+}

util/clixon_util_xpath.c

@@ -33,13 +33,7 @@
 
 See https://www.w3.org/TR/xpath/
 
- * Turn this on to get an xpath test program 
+*
- * Usage: xpath [<xpath>] 
- * read xpath on first line and xml on rest of lines from input
- * Example compile:
- gcc -g -o xpath -I. -I../clixon ./clixon_xsl.c -lclixon -lcligen
- * Example run:
-echo "a\n<a><b/></a>" | xpath
 */
 
 #ifdef HAVE_CONFIG_H