Added CLICON_XMLDB_PRETTY option. If set to false, XML database files will be more compact.

Added CLICON_XMLDB_FORMAT option. Default is "xml". If set to "json", XML database files uses JSON format.
Escape " in JSON names and strings and values
Optimized search performance for large lists by sorting and binary search.

CHANGELOG.md

@@ -1,35 +1,20 @@
 # Clixon Changelog
 
 ## 3.4.0 (Upcoming)
+### Known issues
+* Please use text datastore, key-value datastore no up-to-date
 
+### Major changes:
 * Optimized search performance for large lists by sorting and binary search.
   * New CLICON_XML_SORT configuration option. Default is 1. Disable by setting to 0.
   * New yang config file: clixon-config@2017-12-27.yang
   * Added yang ordered-by user. The default (ordered-by system) will now sort lists and leaf-lists alphabetically to increase search performance.
   * This replaces XML hash experimental code, ie xml_child_hash variables and all xml_hash_ functions have been removed.
-  
+  * Cached keys in yang Y_LIST node as cligen vector, see ys_populate_list()  
-
-* Cached keys in yang Y_LIST node as cligen vector, see ys_populate_list()
-
-* Clixon_backend now returns -1/255 on error instead of 0. Useful for systemd restarts, for example.
-
-* Fixed bug that deletes running on startup if backup started with -m running.
-  When clixon starts again, running is lost.
-  The error was that the running (or startup) configuration may fail when
-  clixon backend starts. 
-  The fix now makes a copy of running and copies it back on failure.
-  
-* experimental netconf yang rpc 
-
-* datastore/keyvalue/Makefile is left behind on make distclean. Fixed by conditional configure. Thanks renato@netgate.com.
-
-* Better semantic versioning, eg MAJOR/MINOR/PATCH, where increment in PATCH does not change API.
 
 * Datastore cache introduced: cache XML tree in memory for faster get access. Use CLICON_XMLDB_CACHE configuration option. Default is 1.
 
-* Moved XML_CHILD_HASH to variable instead of constant.
+* Changed C functional API for XML creation and parsing for more coherency and closer YANG/XML integration. A new yang spec parameter has been added (default NULL) and functions have been removed and renamed. You may need to change the XML calls as follows.
-	
-* Changed XML creation and parse API for more coherency and closer YANG/XML integration. A new yang spec parameter has been added (default NULL) and functions have been removed and renamed. You may need to change the XML calls as follows.
   * xml_new(name, parent) --> xml_new(name, xn_parent, yspec)
   * xml_new_spec(name, parent, spec) --> xml_new(name, parent, spec)
   * clicon_xml_parse(&xt, format, ...) --> xml_parse_va(&xt, yspec, format, ...)
@@ -42,6 +27,24 @@
       configure --with-xml-compat
   ```
   
+### Minor changes:
+* Better semantic versioning, eg MAJOR/MINOR/PATCH, where increment in PATCH does not change API.
+* Added CLICON_XMLDB_PRETTY option. If set to false, XML database files will be more compact.
+* Added CLICON_XMLDB_FORMAT option. Default is "xml". If set to "json", XML database files uses JSON format.
+* Clixon_backend now returns -1/255 on error instead of 0. Useful for systemd restarts, for example.
+* Experimental: netconf yang rpc. That is, using ietf-netconf@2011-06-01.yang
+  formal specification instead of hardcoded C-code.
+
+### Corrected Bugs
+
+* Fixed bug that deletes running on startup if backup started with -m running.
+  When clixon starts again, running is lost.
+  The error was that the running (or startup) configuration may fail when
+  clixon backend starts. 
+  The fix now makes a copy of running and copies it back on failure.
+* datastore/keyvalue/Makefile was left behind on make distclean. Fixed by conditional configure. Thanks renato@netgate.com.
+* Escape " in JSON names and strings and values
+  
 ## 3.3.3 (25 November 2017)
 
 Thanks to Matthew Smith, Joe Loeliger at Netgate; Fredrik Pettai at

apps/backend/backend_main.c

@@ -666,6 +666,8 @@ main(int argc, char **argv)
     int           sockfamily;
     char         *xmldb_plugin;
     int           xml_cache;
+    int           xml_pretty;
+    char         *xml_format;
 
     /* In the startup, logs to stderr & syslog and debug flag set later */
     clicon_log_init(__PROGRAM__, LOG_INFO, CLICON_LOG_STDERR|CLICON_LOG_SYSLOG);
@@ -879,6 +881,12 @@ main(int argc, char **argv)
     if ((xml_cache = clicon_option_bool(h, "CLICON_XMLDB_CACHE")) >= 0)
 	if (xmldb_setopt(h, "xml_cache", (void*)(intptr_t)xml_cache) < 0)
 	    goto done;
+    if ((xml_format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) >= 0)
+	if (xmldb_setopt(h, "format", (void*)xml_format) < 0)
+	    goto done;
+    if ((xml_pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) >= 0)
+	if (xmldb_setopt(h, "pretty", (void*)(intptr_t)xml_pretty) < 0)
+	    goto done;
     /* If startup mode is not defined, eg via OPTION or -s, assume old method */
     startup_mode = clicon_startup_mode(h);
     if (startup_mode == -1){ 	/* Old style, fragmented mode, phase out */

apps/cli/cli_common.c

@@ -724,8 +724,7 @@ load_config_file(clicon_handle h,
     opstr  = cv_string_get(cvec_i(argv, 1));
     if (strcmp(opstr, "merge") == 0) 
 	replace = 0;
-    else
+    else if (strcmp(opstr, "replace") == 0) 
-    if (strcmp(opstr, "replace") == 0) 
 	replace = 1;
     else{
 	clicon_err(OE_PLUGIN, 0, "No such op: %s, expected merge or replace", opstr);	
@@ -738,7 +737,7 @@ load_config_file(clicon_handle h,
     filename = cv_string_get(cv);
     if (stat(filename, &st) < 0){
  	clicon_err(OE_UNIX, 0, "load_config: stat(%s): %s", 
- 		filename, strerror(errno));
+		   filename, strerror(errno));
 	goto done;
     }
     /* Open and parse local file into xml */
@@ -750,26 +749,23 @@ load_config_file(clicon_handle h,
 	goto done;
     if (xt == NULL)
 	goto done;
-
+    if ((cbxml = cbuf_new()) == NULL)
-    //    if ((xn = xml_child_i(xt, 0)) != NULL){
+	goto done;
-    
+    x = NULL;
-	if ((cbxml = cbuf_new()) == NULL)
+    while ((x = xml_child_each(xt, x, -1)) != NULL) {
+	/* Ensure top-level is "config", maybe this is too rough? */
+	xml_name_set(x, "config");
+	if (clicon_xml2cbuf(cbxml, x, 0, 0) < 0)
 	    goto done;
-	x = NULL;
+    }
-	while ((x = xml_child_each(xt, x, -1)) != NULL) {
+    if (clicon_rpc_edit_config(h, "candidate",
-	    /* Ensure top-level is "config", maybe this is too rough? */
+			       replace?OP_REPLACE:OP_MERGE, 
-	    xml_name_set(x, "config");
+			       cbuf_get(cbxml)) < 0)
-	    if (clicon_xml2cbuf(cbxml, x, 0, 0) < 0)
+	goto done;
-		goto done;
+    cbuf_free(cbxml);
-	}
+    //    }
-	if (clicon_rpc_edit_config(h, "candidate",
-				   replace?OP_REPLACE:OP_MERGE, 
-				   cbuf_get(cbxml)) < 0)
-	    goto done;
-	cbuf_free(cbxml);
-	//    }
     ret = 0;
-  done:
+ done:
     if (xt)
 	xml_free(xt);
     if (fd != -1)

datastore/text/clixon_xmldb_text.c

@@ -93,10 +93,16 @@ struct db_element{
 /* Keep datastore text in memory so that get operation need only read memory.
  * Write to file on modification or file change.
  * Assumes single backend
- * Experimental
+ * XXX MOVE TO HANDLE all three below
  */
 static int xmltree_cache = 1;
 
+/* Format */
+static char *xml_format = "xml";
+
+/* Store xml/json pretty-printed. Or not. */
+static int xml_pretty = 1;
+
 /*! Check struct magic number for sanity checks
  * return 0 if OK, -1 if fail.
  */
@@ -232,6 +238,12 @@ text_getopt(xmldb_handle xh,
 	*value = th->th_yangspec;
     else if (strcmp(optname, "dbdir") == 0)
 	*value = th->th_dbdir;
+    else if (strcmp(optname, "xml_cache") == 0)
+	*value = &xmltree_cache;
+    else if (strcmp(optname, "format") == 0)
+	*value = xml_format;
+    else if (strcmp(optname, "pretty") == 0)
+	*value = &xml_pretty;
     else{
 	clicon_err(OE_PLUGIN, 0, "Option %s not implemented by plugin", optname);
 	goto done;
@@ -243,7 +255,7 @@ text_getopt(xmldb_handle xh,
 
 /*! Set value of generic plugin option. Type of value is given by context
  * @param[in]  xh      XMLDB handle
- * @param[in]  optname Option name
+ * @param[in]  optname Option name: yangspec, xml_cache, format, prettyprint
  * @param[in]  value   Value of option
  * @retval     0       OK
  * @retval    -1       Error
@@ -267,6 +279,19 @@ text_setopt(xmldb_handle xh,
     else if (strcmp(optname, "xml_cache") == 0){
 	xmltree_cache = (intptr_t)value;
     }
+    else if (strcmp(optname, "format") == 0){
+	if (strcmp(value,"xml")==0)
+	    xml_format = "xml";
+	else if (strcmp(value,"json")==0)
+	    xml_format = "json";
+	else{
+	    clicon_err(OE_PLUGIN, 0, "Option %s unrecognized format: %s", optname, value);
+	    goto done;
+	}
+    }
+    else if (strcmp(optname, "pretty") == 0){
+	xml_pretty = (intptr_t)value;
+    }
     else{
 	clicon_err(OE_PLUGIN, 0, "Option %s not implemented by plugin", optname);
 	goto done;
@@ -430,7 +455,11 @@ text_get(xmldb_handle xh,
 	    goto done;
 	}    
 	/* Parse file into XML tree */
-	if ((xml_parse_file(fd, "</config>", yspec, &xt)) < 0)
+	if (strcmp(xml_format,"json")==0){
+	    if ((json_parse_file(fd, yspec, &xt)) < 0)
+		goto done;
+	}
+	else if ((xml_parse_file(fd, "</config>", yspec, &xt)) < 0)
 	    goto done;
 	/* Always assert a top-level called "config". 
 	   To ensure that, deal with two cases:
@@ -445,7 +474,7 @@ text_get(xmldb_handle xh,
 	    if (singleconfigroot(xt, &xt) < 0)
 		goto done;
 	}
-	/* Sort XML children according to YANG and ordered-by XXX */
+	/* Sort XML children according to YANG */
 	if (xml_child_sort && xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0)
 	    goto done;
     } /* xt == NULL */
@@ -511,9 +540,11 @@ text_get(xmldb_handle xh,
     /* Order XML children according to YANG */
     if (!xml_child_sort && xml_apply(xt, CX_ELMNT, xml_order, NULL) < 0)
 	goto done;
-    /* XXX again just so default values are placed correctly */
+    /* Again just so default values are placed correctly */
     if (xml_child_sort && xml_apply0(xt, CX_ELMNT, xml_sort, NULL) < 0)
 	goto done;
+    if (xml_child_sort && xml_apply0(xt, -1, xml_sort_verify, NULL) < 0)
+	clicon_log(LOG_NOTICE, "%s: verify failed #2", __FUNCTION__);
     if (debug>1)
     	clicon_xml2file(stderr, xt, 0, 1);
     *xtop = xt;
@@ -854,7 +885,11 @@ text_put(xmldb_handle        xh,
 	    goto done;
 	}    
 	/* Parse file into XML tree */
-	if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
+	if (strcmp(xml_format,"json")==0){
+	    if ((json_parse_file(fd, yspec, &x0)) < 0)
+		goto done;
+	}
+	else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
 	    goto done;
 	/* Always assert a top-level called "config". 
 	   To ensure that, deal with two cases:
@@ -931,7 +966,11 @@ text_put(xmldb_handle        xh,
 	clicon_err(OE_CFG, errno, "Creating file %s", dbfile);
 	goto done;
     } 
-    if (xml_print(f, x0) < 0)
+    if (strcmp(xml_format,"json")==0){
+	if (xml2json(f, x0, xml_pretty) < 0)
+	    goto done;
+    }
+    else if (clicon_xml2file(f, x0, 0, xml_pretty) < 0)
 	goto done;
     retval = 0;
  done:
@@ -1315,7 +1354,14 @@ main(int    argc,
 	xpath = argc>5?argv[5]:NULL;
 	if (xmldb_get(h, db, xpath, &xt, NULL, 1, NULL) < 0)
 	    goto done;
-	clicon_xml2file(stdout, xt, 0, 1);	
+	if (strcmp(xml_format,"json")==0){
+	    if (xml2json(stdout, xt, xml_pretty) < 0)
+		goto done;
+	}
+	else{
+	    if (clicon_xml2file(stdout, xt, 0, xml_pretty) < 0)
+		goto done;
+	}
     }
     else
     if (strcmp(cmd, "put")==0){

lib/clixon/clixon_json.h

@@ -39,10 +39,11 @@
 /*
  * Prototypes
  */
-int json_parse_str(char *str, cxobj **xt);
 int xml2json_cbuf(cbuf *cb, cxobj *x, int pretty);
 int xml2json_cbuf_vec(cbuf *cb, cxobj **vec, size_t veclen, int pretty);
 int xml2json(FILE *f, cxobj *x, int pretty);
 int xml2json_vec(FILE *f, cxobj **vec, size_t veclen, int pretty);
+int json_parse_str(char *str, cxobj **xt);
+int json_parse_file(int fd, yang_spec *yspec, cxobj **xt);
 
 #endif /* _CLIXON_JSON_H */

lib/clixon/clixon_xml.h

@@ -160,7 +160,6 @@ int       xml_body_int32(cxobj *xb, int32_t *val);
 int       xml_body_uint32(cxobj *xb, uint32_t *val);
 int       xml_operation(char *opstr, enum operation_type *op);
 char     *xml_operation2str(enum operation_type op);
-int       xml_sort(cxobj *x0, void *arg);
 
 #ifdef XML_COMPAT /* See CHANGELOG */
 /* MANUAL CHANGE: xml_new(name, parent) --> xml_new(name, parent, NULL) */

lib/clixon/clixon_xml_sort.h

@@ -45,8 +45,12 @@ extern int xml_child_sort;
  * Prototypes
  */
 int    xml_child_spec(char  *name, cxobj  *xp, yang_spec *yspec, yang_stmt **yp);
+int    xml_cmp(const void* arg1, const void* arg2);
+int    xml_sort(cxobj *x0, void *arg);
 cxobj *xml_search(cxobj *x, char *name, int yangi, enum rfc_6020 keyword, int keynr, char **keyvec, char **keyval);
+cxobj *xml_sort_insert(cxobj *x0, cxobj *x);
 cxobj *xml_match(cxobj *x0, char *name, enum rfc_6020 keyword, int keynr, char **keyvec, char **keyval);
-int match_base_child(cxobj *x0, cxobj *x1c, cxobj **x0cp, yang_stmt *yc);
+int    xml_sort_verify(cxobj *x, void *arg);
+int    match_base_child(cxobj *x0, cxobj *x1c, cxobj **x0cp, yang_stmt *yc);
 
 #endif /* _CLIXON_XML_SORT_H */

lib/src/clixon_json.c

@@ -69,6 +69,12 @@
 */
 #define VEC_ARRAY 1
 
+/* Size of json read buffer when reading from file*/
+#define BUFLEN 1024
+
+/* Name of xml top object created by xml parse functions */
+#define JSON_TOP_SYMBOL "top"
+
 enum array_element_type{
     NO_ARRAY=0,
     FIRST_ARRAY,
@@ -185,32 +191,40 @@ json_escape(char *str)
 
     j = 0;
     for (i=0;i<strlen(str);i++)
-	if (str[i]=='\n')
+	switch (str[i]){
+	case '\n':
+	case '\"':
+	case '\\':
 	    j++;
+	    break;
+	}
     if ((snew = malloc(strlen(str)+1+j))==NULL){
 	clicon_err(OE_XML, errno, "malloc");
 	return NULL;
     }
     j = 0;
     for (i=0;i<strlen(str);i++)
-	if (str[i]=='\n'){
+	switch (str[i]){
+	case '\n':
+	case '\"':
+	case '\\':
 	    snew[j++]='\\';
-	    snew[j++]='n';
+	default: /* fall thru */
-	}
-	else
 	    snew[j++]=str[i];
+	}
     snew[j++]='\0';
     return snew;
 }
 
 /*! Do the actual work of translating XML to JSON 
- * @param[out]   cb       Cligen text buffer containing json on exit
+ * @param[out]   cb        Cligen text buffer containing json on exit
- * @param[in]    x        XML tree structure containing XML to translate
+ * @param[in]    x         XML tree structure containing XML to translate
  * @param[in]    arraytype Does x occur in a array (of its parent) and how?
  * @param[in]    level     Indentation level
  * @param[in]    pretty    Pretty-print output (2 means debug)
  * @param[in]    flat      Dont print NO_ARRAY object name (for _vec call)
  *
+ * @note Does not work with XML attributes
  * The following matrix explains how the mapping is done.
  * You need to understand what arraytype means (no/first/middle/last)
  * and what childtype is (null,body,any)
@@ -246,10 +260,11 @@ xml2json1_cbuf(cbuf                  *cb,
 	       int                    pretty,
 	       int                    flat)
 {
-    int             retval = -1;
+    int              retval = -1;
-    int             i;
+    int              i;
-    cxobj          *xc;
+    cxobj           *xc;
     enum childtype   childt;
+    enum array_element_type xc_arraytype;
 
     childt = childtype(x);
     if (pretty==2)
@@ -326,7 +341,6 @@ xml2json1_cbuf(cbuf                  *cb,
 	break;
     }
     for (i=0; i<xml_child_nr(x); i++){
-	enum array_element_type xc_arraytype;
 	xc = xml_child_i(x, i);
 	xc_arraytype = array_eval(i?xml_child_i(x,i-1):NULL, 
 				xc, 
@@ -628,6 +642,82 @@ json_parse_str(char   *str,
     return json_parse(str, "", *xt);
 }
 
+/*! Read a JSON definition from file and parse it into a parse-tree. 
+ *
+ * @param[in]  fd  A file descriptor containing the JSON file (as ASCII characters)
+ * @param[in]  yspec   Yang specification, or NULL
+ * @param[in,out] xt   Pointer to (XML) parse tree. If empty, create.
+ * @retval        0  OK
+ * @retval       -1  Error with clicon_err called
+ *
+ * @code
+ *  cxobj *xt = NULL;
+ *  if (json_parse_file(0, NULL, &xt) < 0)
+ *    err;
+ *  xml_free(xt);
+ * @endcode
+ * @note  you need to free the xml parse tree after use, using xml_free()
+ * @note, If xt empty, a top-level symbol will be added so that <tree../> will be:  <top><tree.../></tree></top>
+ * @note May block on file I/O
+ */
+int 
+json_parse_file(int        fd,
+		yang_spec *yspec,
+		cxobj    **xt)
+{
+    int   retval = -1;
+    int   ret;
+    char *jsonbuf = NULL;
+    int   jsonbuflen = BUFLEN; /* start size */
+    int   oldjsonbuflen;
+    char *ptr;
+    char  ch;
+    int   len = 0;
+    
+    if ((jsonbuf = malloc(jsonbuflen)) == NULL){
+	clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__);
+	goto done;
+    }
+    memset(jsonbuf, 0, jsonbuflen);
+    ptr = jsonbuf;
+    while (1){
+	if ((ret = read(fd, &ch, 1)) < 0){
+	    clicon_err(OE_XML, errno, "%s: read: [pid:%d]\n", 
+		    __FUNCTION__,
+		    (int)getpid());
+	    break;
+	}
+	if (ret != 0)
+	    jsonbuf[len++] = ch;
+	if (ret == 0){
+	    if (*xt == NULL)
+		if ((*xt = xml_new(JSON_TOP_SYMBOL, NULL, NULL)) == NULL)
+		    goto done;
+	    if (len && json_parse(ptr, "", *xt) < 0)
+		goto done;
+	    break;
+	}
+	if (len>=jsonbuflen-1){ /* Space: one for the null character */
+	    oldjsonbuflen = jsonbuflen;
+	    jsonbuflen *= 2;
+	    if ((jsonbuf = realloc(jsonbuf, jsonbuflen)) == NULL){
+		clicon_err(OE_XML, errno, "%s: realloc", __FUNCTION__);
+		goto done;
+	    }
+	    memset(jsonbuf+oldjsonbuflen, 0, jsonbuflen-oldjsonbuflen);
+	    ptr = jsonbuf;
+	}
+    }
+    retval = 0;
+ done:
+    if (retval < 0 && *xt){
+	free(*xt);
+	*xt = NULL;
+    }
+    if (jsonbuf)
+	free(jsonbuf);
+    return retval;    
+}
 
 /*
  * Turn this on to get a json parse and pretty print test program

lib/src/clixon_xml.c

@@ -63,6 +63,7 @@
 #include "clixon_handle.h"
 #include "clixon_yang.h"
 #include "clixon_xml.h"
+#include "clixon_xml_sort.h"
 #include "clixon_xml_parse.h"
 
 /*
@@ -528,6 +529,7 @@ xml_childvec_get(cxobj *x)
  * @endcode
  * @note yspec may be NULL either because it is not known or it is irrelevant, 
  *       eg for body or attribute
+ * @see xml_sort_insert
  */
 cxobj *
 xml_new(char      *name, 
@@ -1290,7 +1292,6 @@ xml_parse_file(int        fd,
 
     if (endtag != NULL)
 	endtaglen = strlen(endtag);
-    *xt = NULL;
     if ((xmlbuf = malloc(xmlbuflen)) == NULL){
 	clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__);
 	goto done;
@@ -1558,7 +1559,7 @@ cxvec_append(cxobj   *x,
  * The tree is traversed depth-first, which at least guarantees that a parent is
  * traversed before a child.
  * @param[in]  xn   XML node
- * @param[in]  type matching type or -1 for any
+ * @param[in]  type Matching type or -1 for any
  * @param[in]  fn   Callback
  * @param[in]  arg  Argument
  * @retval    -1    Error, aborted at first error encounter
@@ -1610,6 +1611,10 @@ xml_apply(cxobj          *xn,
 }
 
 /*! Apply a function call on top object and all xml node children recursively 
+ * @param[in]  xn   XML node
+ * @param[in]  type Matching type or -1 for any
+ * @param[in]  fn   Callback
+ * @param[in]  arg  Argument
  * @retval    -1    Error, aborted at first error encounter
  * @retval     0    OK, all nodes traversed (subparts may have been skipped)
  * @retval     1    OK, aborted on first fn returned 1
@@ -1836,93 +1841,6 @@ xml_operation2str(enum operation_type op)
     }
 }
 
-/*! Help function to qsort for sorting entries in xml child vector
- * @param[in]  arg1
- * @param[in]  arg2
- * @retval  0  If equal
- * @retval <0  if arg1 is less than arg2
- * @retval >0  if arg1 is greater than arg2
- * @note must be in clixon_xml.c since it uses internal (hidden) struct xml
- */
-static int
-xml_cmp(const void* arg1, 
-	const void* arg2)
-{
-    struct xml *x1 = *(struct xml**)arg1;
-    struct xml *x2 = *(struct xml**)arg2;
-    yang_stmt  *y1;
-    yang_stmt  *y2;
-    int         yi1;
-    int         yi2;
-    cvec       *cvk = NULL; /* vector of index keys */
-    cg_var     *cvi;
-    int         equal = 0;
-    char       *b1;
-    char       *b2;
-    char       *keyname;
-    
-    if (x1 == NULL){
-	if (x2 == NULL)
-	    return 0;
-	else
-	    return -1;
-    }
-    else if (x2 == NULL)
-	return 1;
-    y1 = xml_spec(x1);
-    y2 = xml_spec(x2);
-    if (y1==NULL || y2==NULL)
-	return 0; /* just ignore */
-    if (y1 != y2){
-	yi1 = yang_order(y1);
-	yi2 = yang_order(y2);
-	if ((equal = yi1-yi2) != 0)
-	    return equal;
-    }
-    /* Now y1=y2, same Yang spec, can only be list or leaf-list,
-     * sort according to key
-     */
-    if (yang_find((yang_node*)y1, Y_ORDERED_BY, "user") != NULL)
-	return 0; /* Ordered by user: maintain existing order */
-    switch (y1->ys_keyword){
-    case Y_LEAF_LIST: /* Match with name and value */
-	equal = strcmp(xml_body(x1), xml_body(x2));
-	break;
-    case Y_LIST: /* Match with key values 
-		  * Use Y_LIST cache (see struct yang_stmt)
-		  */
-	cvk = y1->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
-	cvi = NULL;
-	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
-	    keyname = cv_string_get(cvi);
-	    b1 = xml_find_body(x1, keyname);
-	    b2 = xml_find_body(x2, keyname);
-	    if ((equal = strcmp(b1,b2)) != 0)
-		goto done;
-	}
-	equal = 0;
-	break;
-    default:
-	break;
-    }
- done:
-    return equal;
-}
-
-/*! Sort children of an XML node
- * Assume populated by yang spec.
- * @param[in] x0   XML node
- * @param[in] arg  Dummy so it can be called by xml_apply()
- * @note must be in clixon_xml.c since it uses internal (hidden) struct xml
- */
-int
-xml_sort(cxobj *x,
-	 void  *arg)
-{
-    qsort(x->x_childvec, x->x_childvec_len, sizeof(struct xml*), xml_cmp);
-    return 0;
-}
-
 
 /*
  * Turn this on to get a xml parse and pretty print test program

lib/src/clixon_xml_map.c

@@ -229,7 +229,10 @@ xml2cli(FILE              *f,
 	    term = xml_name(x);
 	if (prepend0)
 	    fprintf(f, "%s ", prepend0);
-	fprintf(f, "%s\n", term);
+	if (index(term, ' '))
+	    fprintf(f, "\"%s\"\n", term);
+	else
+	    fprintf(f, "%s\n", term);
 	retval = 0;
 	goto done;
     }

lib/src/clixon_xml_sort.c

@@ -100,6 +100,80 @@ xml_child_spec(char       *name,
     return 0;
 }
 
+/*! Help function to qsort for sorting entries in xml child vector
+ * @param[in]  arg1 - actually cxobj**
+ * @param[in]  arg2 - actually cxobj**
+ * @retval  0  If equal
+ * @retval <0  if arg1 is less than arg2
+ * @retval >0  if arg1 is greater than arg2
+ * @note args are pointer ot pointers, to fit into qsort cmp function
+ * @see xml_cmp1   Similar, but for one object
+ */
+int
+xml_cmp(const void* arg1, 
+	const void* arg2)
+{
+    cxobj *x1 = *(struct xml**)arg1;
+    cxobj *x2 = *(struct xml**)arg2;
+    yang_stmt  *y1;
+    yang_stmt  *y2;
+    int         yi1;
+    int         yi2;
+    cvec       *cvk = NULL; /* vector of index keys */
+    cg_var     *cvi;
+    int         equal = 0;
+    char       *b1;
+    char       *b2;
+    char       *keyname;
+    
+    if (x1 == NULL){
+	if (x2 == NULL)
+	    return 0;
+	else
+	    return -1;
+    }
+    else if (x2 == NULL)
+	return 1;
+    y1 = xml_spec(x1);
+    y2 = xml_spec(x2);
+    if (y1==NULL || y2==NULL)
+	return 0; /* just ignore */
+    if (y1 != y2){
+	yi1 = yang_order(y1);
+	yi2 = yang_order(y2);
+	if ((equal = yi1-yi2) != 0)
+	    return equal;
+    }
+    /* Now y1=y2, same Yang spec, can only be list or leaf-list,
+     * sort according to key
+     */
+    if (yang_find((yang_node*)y1, Y_ORDERED_BY, "user") != NULL)
+	return 0; /* Ordered by user: maintain existing order */
+    switch (y1->ys_keyword){
+    case Y_LEAF_LIST: /* Match with name and value */
+	equal = strcmp(xml_body(x1), xml_body(x2));
+	break;
+    case Y_LIST: /* Match with key values 
+		  * Use Y_LIST cache (see struct yang_stmt)
+		  */
+	cvk = y1->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
+	cvi = NULL;
+	while ((cvi = cvec_each(cvk, cvi)) != NULL) {
+	    keyname = cv_string_get(cvi);
+	    b1 = xml_find_body(x1, keyname);
+	    b2 = xml_find_body(x2, keyname);
+	    if ((equal = strcmp(b1,b2)) != 0)
+		goto done;
+	}
+	equal = 0;
+	break;
+    default:
+	break;
+    }
+ done:
+    return equal;
+}
+
 /*!
  * @param[in] yangi  Yang order
  * @param[in]  keynr    Length of keyvec/keyval vector when applicable
@@ -109,6 +183,7 @@ xml_child_spec(char       *name,
  * @retval  0  If equal (or userorder set)
  * @retval <0  if arg1 is less than arg2
  * @retval >0  if arg1 is greater than arg2
+ * @see xml_cmp   Similar, but for two objects
  */
 static int
 xml_cmp1(cxobj        *x,
@@ -155,6 +230,21 @@ xml_cmp1(cxobj        *x,
     return 0; /* should not reach here */
 }
 
+/*! Sort children of an XML node
+ * Assume populated by yang spec.
+ * @param[in] x0   XML node
+ * @param[in] arg  Dummy so it can be called by xml_apply()
+ */
+int
+xml_sort(cxobj *x,
+	 void  *arg)
+{
+    qsort(xml_childvec_get(x), xml_child_nr(x), sizeof(cxobj *), xml_cmp);
+    return 0;
+}
+
+/*! Special case search for ordered-by user where linear sort is used
+ */
 static cxobj *
 xml_search_userorder(cxobj        *x0,
 		     yang_stmt    *y,
@@ -236,7 +326,7 @@ xml_search1(cxobj        *x0,
     return NULL;
 }
 
-/*!      
+/*! Find XML children using binary search
  * @param[in] yangi  yang child order
  * @param[in]  keynr    Length of keyvec/keyval vector when applicable
  * @param[in]  keyvec   Array of of yang key identifiers
@@ -255,6 +345,63 @@ xml_search(cxobj        *x0,
 		       0, xml_child_nr(x0));
 }
 
+static int
+xml_insert_pos(cxobj *x0,
+		    cxobj *x,
+		    int    low, 
+		    int    upper)
+{
+    int     mid;
+    cxobj  *xc;
+    int     cmp;
+    int     i;
+
+    if (upper < low)
+	return low; /* not found */
+    mid = (low + upper) / 2;
+    if (mid >= xml_child_nr(x0))
+	return xml_child_nr(x0);
+    xc = xml_child_i(x0, mid);
+    cmp = xml_cmp(&x, &xc); 
+    if (cmp == 0){
+	/* Special case: append last of equals if ordered by user */
+	for (i=mid+1;i<xml_child_nr(x0);i++){
+	    xc = xml_child_i(x0, i);
+	    if (xml_cmp(&x, &xc) != 0)
+		break;
+	    mid=i; /* still ok */
+	}
+	return mid;
+    }
+    else if (cmp < 0)
+	return xml_insert_pos(x0, x, low, mid-1);
+    else
+	return xml_insert_pos(x0, x, mid+1, upper);
+}
+		    
+/*! Add xml object in sorted position 
+ * @param[in]  x0       XML parent node.
+ * @param[in]  x        XML node (to insert)
+ * Assume already under x0
+ * XXX WORK IN PROGRESS 
+ */
+cxobj *
+xml_sort_insert(cxobj *x0,
+		cxobj *x)
+{
+    int pos;
+    /* find closest to x, insert after pos. */
+    xml_rm(x);
+    pos = xml_insert_pos(x0, x, 0, xml_child_nr(x0));
+    fprintf(stderr, "%d\n", pos);
+#if 0
+    if (pos < xml_child_nr(x0))
+	xml_child_i_set(x0, pos);
+    xml_parent_set(x, x0);
+#endif
+    return x;
+}
+
 /*! Find matching xml child given name and optional key values
  * container: x0, y->keyword, name
  * list:      x0, y->keyword, y->key, name
@@ -350,6 +497,34 @@ xml_match(cxobj        *x0,
  ok:
     return x;
 }
+
+/*! Verify all children of XML node are sorted according to xml_sort()
+ * @param[in]   x       XML node. Check its children
+ * @param[in]   arg     Dummy. Ensures xml_apply can be used with this fn
+ @ @retval      0       Sorted
+ @ @retval     -1       Not sorted
+ * @see xml_apply
+ */
+int
+xml_sort_verify(cxobj *x0,
+		void  *arg)
+{
+    int    retval = -1;
+    cxobj *x = NULL;
+    cxobj *xprev = NULL;
+
+    while ((x = xml_child_each(x0, x, -1)) != NULL) {
+	if (xprev != NULL){ /* Check xprev <= x */
+	    if (xml_cmp(&xprev, &x) > 0)
+		goto done;
+	}
+	xprev = x;
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Given child tree x1c, find matching child in base tree x0
  * param[in]  x0   Base tree node
  * param[in]  x1c  Modification tree child

test/test_cli.sh

@@ -21,8 +21,7 @@ sudo clixon_backend -z -f $cfg
 if [ $? -ne 0 ]; then
     err
 fi
-new "start backend"
+new "start backend -s init -f $cfg"
-# start new backend
 sudo clixon_backend -s init -f $cfg 
 if [ $? -ne 0 ]; then
     err

test/test_order.sh

@@ -171,7 +171,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-#sudo clixon_backend -zf $cfg
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

test/test_yang.sh

@@ -86,7 +86,7 @@ if [ $? -ne 0 ]; then
     err
 fi
 
-new "start backend"
+new "start backend -s init -f $cfg -y $fyang"
 # start new backend
 sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
@@ -147,7 +147,7 @@ if [ -z "$pid" ]; then
     err "backend already dead"
 fi
 # kill backend
-#sudo clixon_backend -zf $cfg
+sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi

yang/clixon-config@2017-12-27.yang

@@ -70,6 +70,18 @@ module clixon-config {
 	    }
 	}
     }
+    typedef xmldb_format{
+	description
+	    "Format of TEXT xml database format.";
+	type enumeration{
+	    enum xml{
+		description "Save and load xmldb as XML";
+	    }
+	    enum json{
+		description "Save and load xmldb as JSON";
+	    }
+	}
+    }
     container config {
 	leaf CLICON_CONFIGFILE{
 	    type string;
@@ -241,6 +253,19 @@ module clixon-config {
                  If set, XML candidate/running parsed tree is stored in memory
                  If not set, candidate/running is always accessed via disk.";
 	}
+	leaf CLICON_XMLDB_FORMAT {
+	    type xmldb_format;
+	    default xml;
+	    description	"XMLDB datastore format.";
+	}
+	leaf CLICON_XMLDB_PRETTY {
+	    type boolean;
+	    default true;
+	    description
+		"XMLDB datastore pretty print. 
+                 If set, insert spaces and line-feeds making the XML/JSON human
+                 readable. If not set, make the XML/JSON more compact.";
+	}
 	leaf CLICON_XML_SORT {
 	    type boolean;
 	    default true;