New XMLDB_FORMAT added: tree. An experimental record-based tree database for direct access of records.

CHANGELOG.md

@@ -111,6 +111,7 @@
 
 ### Minor changes
 
+* New XMLDB_FORMAT added: `tree`. An experimental record-based tree database for direct access of records. 
 * A new "hello world" example is added
 * Experimental customized error output strings, see [lib/clixon/clixon_err_string.h]
 * Empty leaf values, eg <a></a> are now checked at validation.

lib/clixon/clixon_queue.h

@@ -126,5 +126,6 @@ typedef struct _qelem_t {
  *  NEXTQ(struct a*, el);
  */
 #define NEXTQ(type, elem)	((type)((elem)?((qelem_t *)(elem))->q_next:NULL))
+#define PREVQ(type, elem)	((type)((elem)?((qelem_t *)(elem))->q_prev:NULL))
 
 #endif	/* _CLIXON_QUEUE_H_ */

lib/src/Makefile.in

@@ -75,6 +75,7 @@ SRC     = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \
 	  clixon_proto.c clixon_proto_client.c \
 	  clixon_xpath.c clixon_xpath_ctx.c clixon_sha1.c \
 	  clixon_datastore.c clixon_datastore_write.c clixon_datastore_read.c \
+	  clixon_datastore_tree.c \
 	  clixon_netconf_lib.c clixon_stream.c clixon_nacm.c
 
 YACCOBJS := lex.clixon_xml_parse.o clixon_xml_parse.tab.o \

lib/src/clixon_datastore_read.c

@@ -78,6 +78,7 @@
 
 #include "clixon_datastore.h"
 #include "clixon_datastore_read.h"
+#include "clixon_datastore_tree.h"
 
 #define handle(xh) (assert(text_handle_check(xh)==0),(struct text_handle *)(xh))
 
@@ -322,30 +323,40 @@ xmldb_readfile(clicon_handle      h,
 	clicon_err(OE_XML, 0, "dbfile NULL");
 	goto done;
     }
-    if ((fd = open(dbfile, O_RDONLY)) < 0) {
+    if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
-	clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
+	clicon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
 	goto done;
-    }    
+    }
-    /* Parse file into XML tree */
+    /* Parse file into internal XML tree from different formats */
-    format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
+    if (strcmp(format, "tree")==0){
-    if (format && strcmp(format, "json")==0){
+	if (datastore_tree_read(h, dbfile, &x0) < 0)
-	if ((json_parse_file(fd, yspec, &x0)) < 0)
 	    goto done;
     }
-    else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
+    else{
-	goto done;
+	if ((fd = open(dbfile, O_RDONLY)) < 0) {
-    /* Always assert a top-level called "config". 
+	    clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
-       To ensure that, deal with two cases:
-       1. File is empty <top/> -> rename top-level to "config" */
-    if (xml_child_nr(x0) == 0){ 
-	if (xml_name_set(x0, "config") < 0)
-	    goto done;     
-    }
-    /* 2. File is not empty <top><config>...</config></top> -> replace root */
-    else{ 
-	/* There should only be one element and called config */
-	if (singleconfigroot(x0, &x0) < 0)
 	    goto done;
+	}    
+	if (strcmp(format, "json")==0){
+	    if ((json_parse_file(fd, yspec, &x0)) < 0)
+		goto done;
+	}
+	else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
+	    goto done;
+
+	/* Always assert a top-level called "config". 
+	   To ensure that, deal with two cases:
+	   1. File is empty <top/> -> rename top-level to "config" */
+	if (xml_child_nr(x0) == 0){ 
+	    if (xml_name_set(x0, "config") < 0)
+		goto done;     
+	}
+	/* 2. File is not empty <top><config>...</config></top> -> replace root */
+	else{ 
+	    /* There should only be one element and called config */
+	    if (singleconfigroot(x0, &x0) < 0)
+		goto done;
+	}
     }
     /* From Clixon 3.10,datastore files may contain module-state defining
      * which modules are used in the file. 
@@ -396,43 +407,13 @@ xmldb_get_nocache(clicon_handle       h,
     cxobj         **xvec = NULL;
     size_t          xlen;
     int             i;
-    char           *format;
 
     if ((yspec = clicon_dbspec_yang(h)) == NULL){
 	clicon_err(OE_YANG, ENOENT, "No yang spec");
 	goto done;
     }
-    if (xmldb_db2file(h, db, &dbfile) < 0)
+    if (xmldb_readfile(h, db, yspec, &xt, msd) < 0)
 	goto done;
-    if (dbfile==NULL){
-	clicon_err(OE_XML, 0, "dbfile NULL");
-	goto done;
-    }
-    if ((fd = open(dbfile, O_RDONLY)) < 0){
-	clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
-	goto done;
-    }    
-    /* Parse file into XML tree */
-    format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
-    if (format && strcmp(format, "json")==0){
-	if ((json_parse_file(fd, yspec, &xt)) < 0)
-	    goto done;
-    }
-    else if ((xml_parse_file(fd, "</config>", yspec, &xt)) < 0)
-	goto done;
-    /* Always assert a top-level called "config". 
-       To ensure that, deal with two cases:
-       1. File is empty <top/> -> rename top-level to "config" */
-    if (xml_child_nr(xt) == 0){ 
-	if (xml_name_set(xt, "config") < 0)
-	    goto done;     
-    }
-    /* 2. File is not empty <top><config>...</config></top> -> replace root */
-    else{ 
-	/* There should only be one element and called config */
-	if (singleconfigroot(xt, &xt) < 0)
-	    goto done;
-    }
     /* Here xt looks like: <config>...</config> */
     /* Given the xpath, return a vector of matches in xvec */
     if (xpath_vec(xt, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
@@ -595,7 +576,7 @@ xmldb_get1_cache(clicon_handle       h,
 		 const char         *db, 
 		 char               *xpath,
 		 cxobj             **xtop,
-		 modstate_diff_t    *modst)
+		 modstate_diff_t    *msd)
 {
     int             retval = -1;
     yang_stmt      *yspec;
@@ -618,7 +599,7 @@ xmldb_get1_cache(clicon_handle       h,
     de = clicon_db_elmnt_get(h, db);
     if (de == NULL || de->de_xml == NULL){ /* Cache miss, read XML from file */
 	/* If there is no xml x0 tree (in cache), then read it from file */
-	if (xmldb_readfile(h, db, yspec, &x0t, modst) < 0)
+	if (xmldb_readfile(h, db, yspec, &x0t, msd) < 0)
 	    goto done;
 	/* XXX: should we validate file if read from disk? 
 	 * Argument against: we may want to have a semantically wrong file and wish

lib/src/clixon_datastore_tree.c

@@ -0,0 +1,451 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+ *
+ * XML file save
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ctype.h>
+#include <string.h>
+#include <syslog.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <arpa/inet.h>
+#include <sys/param.h>
+#include <netinet/in.h>
+
+/* cligen */
+#include <cligen/cligen.h>
+
+/* clicon */
+
+#include "clixon_string.h"
+#include "clixon_queue.h"
+#include "clixon_hash.h"
+#include "clixon_err.h"
+#include "clixon_log.h"
+#include "clixon_handle.h"
+#include "clixon_yang.h"
+#include "clixon_xml.h"
+
+#include "clixon_datastore_tree.h"
+
+#define align4(s) (((s)/4)*4 + 4)
+
+struct indexlist{
+    qelem_t  il_q; /* this must be there */
+    uint32_t il_i;
+    int      il_len; /* How many */
+};
+typedef struct indexlist indexlist;
+
+struct indexhead{
+    indexlist *ih_list;
+};
+typedef struct indexhead indexhead;
+
+int
+index_get(indexhead *ih,
+	  size_t     len, /* in bytes */
+	  uint32_t  *indexp)
+{
+    int               retval = -1;
+    int               nr;
+    uint32_t          index = 0;
+    struct indexlist *il;
+    indexlist        *new;
+
+    nr = len/DF_BLOCK + 1;
+    if ((new = malloc(sizeof(indexlist))) == NULL){
+	clicon_err(OE_XML, errno, "malloc");
+	goto done;
+    }
+    memset(new, 0, sizeof(indexlist));
+    //    new->il_i = i; Dont know i yet
+    new->il_len = nr;
+    if ((il = ih->ih_list) != NULL) {
+	il = PREVQ(indexlist*, il);
+	index = il->il_i + il->il_len;
+    }
+    ADDQ(new, ih->ih_list);
+    new->il_i = index;
+    *indexp = index;
+    retval = 0;
+ done:
+    return retval;
+}
+#ifdef NOTYET
+int
+index_get(indexhead *ih,
+	  size_t     len, /* in bytes */
+	  uint32_t  *indexp)
+{
+    int               retval = -1;
+    int               nr;
+    uint32_t          index = 0;
+    struct indexlist *il;
+    indexlist        *new;
+
+    nr = len/DF_BLOCK + 1;
+    if ((new = malloc(sizeof(indexlist))) == NULL){
+	clicon_err(OE_XML, errno, "malloc");
+	goto done;
+    }
+    memset(new, 0, sizeof(indexlist));
+    //    new->il_i = i; Dont know i yet
+    new->il_len = nr;
+
+    /* Find nr conseq blocks */
+    if ((il = ih->ih_list) != NULL) {
+	do {
+	    if (il->il_i - index >= nr){ /* Found */
+		INSQ(new, il);
+		break;
+	    }
+	    index = il->il_i + il->il_len;
+	    il = NEXTQ(indexlist*, il);
+	} while (il != ih->ih_list);
+	ADDQ(new, il); /* after? */
+    }
+    else
+	INSQ(new, ih->ih_list);
+    new->il_i = index;
+    *indexp = index;
+    retval = 0;
+ done:
+    return retval;
+}
+#endif
+
+int
+index_dump(FILE      *f,
+	   indexhead *ih)
+{
+    indexlist *il;
+
+    if ((il = ih->ih_list) != NULL) {
+	do {
+	    fprintf(f, "%u %d\n", il->il_i, il->il_len);
+	    il = NEXTQ(indexlist*, il);
+	}  while (il != ih->ih_list);
+    }
+    return 0;
+}
+
+/*
+ * @param[out] xp
+ * @param[out] vec If type is CX_ELMNT, points to vector of indexes,...
+ * @retval    -1   Error
+ * @retval     0   Sanity check failed
+ * @retval     1   OK  xp set (or NULL if empty)
+ */
+static int
+buf2xml(FILE     *f,
+	uint32_t  index,
+	cxobj    **xp)
+{
+    int      retval = -1;
+    uint8_t  ver;
+    uint8_t  type;
+    int      ptr;
+    uint32_t len;
+    cxobj   *x = NULL;
+    char    *name;
+    char    *prefix = NULL;
+    char     hdr[8];
+    char    *buf = NULL;
+    int       vlen;
+    int       i;
+    uint32_t  ind;
+    cxobj    *xc;
+
+    /* Read hdr
+     * +-----+-----+-----+-----+-----+-----+-----+-----+
+     * | ver | type| de  | ad  | len                   |
+     * +-----+-----+-----+-----+-----+-----+-----+-----+
+     */
+    if (fseek(f, index*DF_BLOCK, SEEK_SET) < 0){
+	clicon_err(OE_UNIX, errno, "fseek");
+	goto done;
+    }
+    if (fread(hdr, sizeof(char), sizeof(hdr), f) < 0){
+	clicon_err(OE_XML, errno, "fread");
+	goto done;
+    }
+    if ((ver = hdr[0]&0xff) == 0) /* Means empty */
+	goto ok;
+    if (ver != 1){
+	clicon_debug(1, "%s Wrong version: %d", __FUNCTION__, ver);
+	goto fail;
+    }
+    type =  hdr[1]&0x0f;
+    if (type != CX_BODY && type != CX_ELMNT && type != CX_ATTR){
+	clicon_debug(1, "%s Wrong type: %d", __FUNCTION__, type);
+	goto fail;
+    }
+    if ((hdr[2]&0xff) != 0xde){
+	clicon_debug(1, "%s Expected 0xde: %x", __FUNCTION__, hdr[2]&0xff);
+	goto fail;
+    }
+    if ((hdr[3]&0xff) != 0xad){
+	clicon_debug(1, "%s Expected 0xad: %x", __FUNCTION__, hdr[3]&0xff);
+	goto fail;
+    }
+    /* Copy and byte-swap length field */
+    memcpy(&len, &hdr[4], 4);
+    len = ntohl(len);
+
+    /* Read rest
+     */
+    ptr = 0;
+    if ((buf = malloc(len - sizeof(hdr))) == NULL){
+	clicon_err(OE_UNIX, errno, "malloc");
+	goto done;
+    }
+    if (fread(buf, sizeof(char), len-sizeof(hdr), f) < 0){
+	clicon_err(OE_XML, errno, "fread");
+	goto done;
+    }
+    if ((name = strchr(buf, ':')) != NULL){
+	*name = '\0';
+	name++;
+	prefix = buf;
+	ptr += strlen(prefix)+1;
+    }
+    else
+	name = buf;
+    ptr += strlen(name) + 1;
+    ptr = align4(ptr);
+    if ((x = xml_new(name, NULL, NULL)) == NULL)
+	goto done;
+    if (prefix)
+	if (xml_prefix_set(x, prefix) < 0)
+	    goto done;
+    xml_type_set(x, type);
+    if (xml_type(x) != CX_ELMNT){
+	if (xml_value_set(x, &buf[ptr]) < 0)
+	    goto done;
+	ptr += strlen(xml_value(x)) +1;
+    }
+    else{
+	vlen = (len - sizeof(hdr) - ptr)/ sizeof(uint32_t);
+	if (vlen < 0){
+	    clicon_debug(1, "%s Nr of elements should not be negative: %d", __FUNCTION__, vlen);
+	    goto fail;
+	}
+	for (i=0; i<vlen; i++){
+	    memcpy(&ind, &buf[ptr], sizeof(uint32_t));
+	    ptr += sizeof(uint32_t);
+	    if (buf2xml(f, ntohl(ind), &xc) < 0)
+		goto done;
+	    if (xml_addsub(x, xc) < 0)
+		goto done;
+	}
+    }
+    assert(len == sizeof(hdr)+ptr);
+ ok:
+    if (xp)
+	*xp = x;
+    retval = 1;
+ done:
+    if (buf)
+	free(buf);
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
+/*! Dump xml object to a datastore block 
+ */
+static int
+xml2buf(FILE      *f,
+	cxobj     *x,
+	indexhead *ih,
+	uint32_t  *indexp)
+{
+    int   retval = -1;
+    char  hdr[8] = {0,};
+    int   len; 
+    int   len1;
+    int   ptr;
+    int   i;
+    char *buf0;
+    char *buf;
+    uint32_t  myindex;
+    
+    hdr[0] = 1;
+    hdr[1] = 0x80 | xml_type(x);
+    hdr[2] = 0xde;
+    hdr[3] = 0xad;
+    len = sizeof(hdr);
+    if (xml_prefix(x))
+	len += strlen(xml_prefix(x)) + 1; /* colon */
+    len += strlen(xml_name(x)) + 1; /* end-of-string */
+    len = align4(len);
+    switch(xml_type(x)){
+    case CX_ELMNT:
+	len += xml_child_nr(x)*sizeof(uint32_t);
+	break;
+    case CX_BODY:
+    case CX_ATTR:
+	len += strlen(xml_value(x))+1;
+	break;
+    default:
+	break;
+    }
+    if (index_get(ih, len, &myindex) < 0)
+	goto done;
+    
+    len1 = htonl(len); /* swap */
+    memcpy(&hdr[4], &len1, 4);
+    /* Write rest
+     */
+    if ((buf0 = malloc(len)) == NULL){
+	clicon_err(OE_UNIX, errno, "malloc");
+	goto done;
+    }
+    buf = buf0+sizeof(hdr);
+    ptr = 0;
+    if (xml_prefix(x)){
+	strcpy(buf, xml_prefix(x));
+	ptr += strlen(xml_prefix(x));
+	buf[ptr++] = ':';
+    }
+    strcpy(&buf[ptr], xml_name(x));
+    ptr += strlen(xml_name(x)) + 1;
+    for (i=ptr; i< align4(ptr); i++)
+	buf[i] = 0;
+    ptr = align4(ptr);
+    if (xml_type(x) != CX_ELMNT){
+	strcpy(&buf[ptr], xml_value(x));
+	ptr += strlen(xml_value(x)) +1;
+    }
+    else{
+	uint32_t index;
+	for (i=0; i< xml_child_nr(x); i++){
+	    if (xml2buf(f, xml_child_i(x, i), ih, &index) < 0)
+		goto done;
+	    index = htonl(index);
+	    memcpy(&buf[ptr], &index, sizeof(uint32_t));
+	    ptr += sizeof(uint32_t);
+	}
+    }
+    assert(len == sizeof(hdr)+ptr);
+    /* Note write both here so we dont get an intermediate seek,... */
+    if (fseek(f, myindex*DF_BLOCK, SEEK_SET) < 0){
+	clicon_err(OE_UNIX, errno, "fseek");
+	goto done;
+    }
+    memcpy(buf0, hdr, sizeof(hdr));
+    if (0 && fwrite(hdr, sizeof(char), sizeof(hdr), f) < 0){
+	clicon_err(OE_UNIX, errno, "fwrite");
+	goto done;
+    }
+    if (fwrite(buf0, sizeof(char), len, f) < 0){
+	clicon_err(OE_XML, errno, "fwrite");
+	goto done;
+    }	
+    if (indexp)
+	*indexp = myindex;
+    retval = 0;
+ done:
+    if (buf0)
+	free(buf0);
+    return retval;
+}
+
+int
+datastore_tree_write(clicon_handle h,
+		     char         *filename,
+		     cxobj        *xt)
+{
+    int              retval = -1;
+    FILE            *f = NULL;
+    indexhead       ih = {0,};
+    indexlist      *il;
+    
+    if ((f = fopen(filename, "w+b")) == NULL){
+	clicon_err(OE_XML, errno, "Opening file %s", filename);
+	goto done;
+    }
+    if (xml2buf(f, xt, &ih, NULL) < 0)
+	goto done;
+    if (debug)
+	index_dump(stderr, &ih);
+    retval = 0;
+ done:
+    while ((il = ih.ih_list) != NULL)
+	DELQ(il, ih.ih_list, indexlist*);
+    if (f != NULL)
+	fclose(f);
+    return retval;
+}
+
+/*
+ * @retval    -1   Error
+ * @retval     0   Sanity check failed
+ * @retval     1   OK  xp set (or NULL if empty)
+ */
+int
+datastore_tree_read(clicon_handle h,
+		    char         *filename,
+		    cxobj       **xt)
+{
+    int              retval = -1;
+    FILE            *f = NULL;
+    
+    if ((f = fopen(filename, "r")) == NULL){
+	clicon_err(OE_XML, errno, "Opening file %s", filename);
+	goto done;
+    }
+    if ((retval = buf2xml(f, 0, xt)) < 0)
+	goto done;
+    if (retval == 0){ /* fail */
+	if ((*xt = xml_new("config", NULL, NULL)) == NULL)
+	    goto done;
+	xml_type_set(*xt, CX_ELMNT);
+    }
+ done:
+    if (f != NULL)
+	fclose(f);
+    return retval;
+}

lib/src/clixon_datastore_tree.h

@@ -0,0 +1,90 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+ *
+ * Datastore block tree file
+ */
+
+#ifndef _CLIXON_DATASTORE_TREE_H_
+#define _CLIXON_DATASTORE_TREE_H_
+
+/*
+ * Constants and macros
+ */
+/* Max of https://github.com/YangModels/yang
+ * name:      receive-recovery-scccp-avp-bad-value-challenge-response
+ * namespace: urn:ieee:std:802.3:yang:ieee802-ethernet-interface-half-duplex
+ */
+#define DF_BLOCK 32
+
+/*
+ * Types
+ */
+/* On file                      value (dep of type)
+ * off: Offset in bytes to where value starts (name ends +1)
+ * len: Total length in bytes (network byte order)
+ * childi: Block index to child (network byte order)
+ * ELMNT:                       
+ * +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+ * | 1   | 128 | de  | ad  | len                   | name ...  |  0  |
+ * +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+ * +-----+-----+-----+-----+-----+-----+-----+------+--
+ * | child0 (index in file)|  child1 (index in file)|
+ * +-----+-----+-----+-----+-----+-----+-----+-----+--
+ * ATTR:
+ * +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+ * | 1   | 129 | de  | ad  | len                   | name  ... |  0  |
+ * +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
+ * +-----+-----+-----+
+ * | value ... | 0   |
+ * +-----+-----+-----+
+ * BODY: (off = 8)
+ * +-----+-----+-----+-----+-----+-----+-----+-----+-----+
+ * | 1   | 130 | de  | ad  |   len                 | 0   |
+ * +-----+-----+-----+-----+-----+-----+-----+-----+-----+
+ * +-----+-----+-----+
+ * | value ... | 0   |
+ * +-----+-----+-----+
+ * FREE
+ * +------+
+ * | 0    |
+ * +------+
+*/
+
+/*
+ * Prototypes
+ */
+int datastore_tree_write(clicon_handle h, char *filename, cxobj *xt);
+int datastore_tree_read(clicon_handle h, char *filename, cxobj **xt);
+
+#endif  /* _CLIXON_DATASTORE_TREE_H_ */

lib/src/clixon_datastore_write.c

@@ -80,7 +80,7 @@
 #include "clixon_datastore.h"
 #include "clixon_datastore_write.h"
 #include "clixon_datastore_read.h"
-
+#include "clixon_datastore_tree.h"
 
 /*! Modify a base tree x0 with x1 with yang spec y according to operation op
  * @param[in]  th       Datastore text handle
@@ -730,17 +730,26 @@ xmldb_put(clicon_handle       h,
 	if (xml_addsub(x0, xmodst) < 0)
 	    goto done;
     }
-    if ((f = fopen(dbfile, "w")) == NULL){
+    if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
-	clicon_err(OE_CFG, errno, "Creating file %s", dbfile);
+	clicon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
 	goto done;
-    } 
-    format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
-    if (format && strcmp(format,"json")==0){
-	if (xml2json(f, x0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
-	    goto done;
     }
-    else if (clicon_xml2file(f, x0, 0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
+   if (strcmp(format, "tree") == 0){
-	goto done;
+       	if (datastore_tree_write(h, dbfile, x0) < 0)
+	    goto done;
+   }
+   else{
+       if ((f = fopen(dbfile, "w")) == NULL){
+	   clicon_err(OE_CFG, errno, "Creating file %s", dbfile);
+	   goto done;
+       } 
+       if (strcmp(format,"json")==0){
+	   if (xml2json(f, x0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
+	       goto done;
+       }
+       else if (clicon_xml2file(f, x0, 0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
+	   goto done;
+   }
     /* Remove modules state after writing to file
      */
     if (xmodst && xml_purge(xmodst) < 0)

test/test_perf.sh

@@ -5,7 +5,7 @@
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
 # Number of list/leaf-list entries in file
-: ${perfnr:=20000}
+: ${perfnr:=10000}
 
 # Number of requests made get/put
 : ${perfreq:=100}
@@ -17,6 +17,8 @@ fyang=$dir/scaling.yang
 fconfig=$dir/large.xml
 fconfig2=$dir/large2.xml
 
+format=xml
+
 cat <<EOF > $fyang
 module scaling{
    yang-version 1.1;
@@ -50,6 +52,7 @@ cat <<EOF > $cfg
   <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
+  <CLICON_XMLDB_FORMAT>$format</CLICON_XMLDB_FORMAT>
   <CLICON_CLI_MODE>example</CLICON_CLI_MODE>
   <CLICON_CLI_DIR>/usr/local/lib/example/cli</CLICON_CLI_DIR>
   <CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
@@ -66,27 +69,61 @@ if [ $BE -ne 0 ]; then
 	err
     fi
 fi
-# Try startup mode w startup
+
+# First generate large XML file
+# Use it latter to generate startup-db in xml, tree formats
+tmpx=$dir/tmp.xml
+new "generate large startup config ($tmpx) with $perfnr entries"
+echo -n "<config><x xmlns=\"urn:example:clixon\">" > $tmpx
+for (( i=0; i<$perfnr; i++ )); do  
+    echo -n "<y><a>$i</a><b>$i</b></y>" >> $tmpx
+done
+echo "</x></config>" >> $tmpx
+
+if false; then
+# Then generate large JSON file (cant translate namespace - long story)
+tmpj=$dir/tmp.json
+new "generate large startup config ($tmpj) with $perfnr entries"
+echo -n '{"config": {"scaling:x":{"y":[' > $tmpj
+for (( i=0; i<$perfnr; i++ )); do  
+    if [ $i -ne 0 ]; then
+	echo -n ",{\"a\":$i,\"b\":$i}" >> $tmpj
+    else
+	echo -n "{\"a\":$i,\"b\":$i}" >> $tmpj
+    fi
+
+done
+echo "]}}}" >> $tmpj
+fi
+
+# Loop over mode and format
 for mode in startup running; do
     file=$dir/${mode}_db
-    sudo touch $file
+    for format in tree xml; do # json - something w namespaces
-    sudo chmod 666 $file
+	sudo rm -f $file
-    new "generate large startup config ($file) with $perfnr list entries in mode $mode"
+	sudo touch $file
-    echo -n "<config><x xmlns=\"urn:example:clixon\">" > $file
+	sudo chmod 666 $file
-    for (( i=0; i<$perfnr; i++ )); do  
+	case $format in
-	echo -n "<y><a>$i</a><b>$i</b></y>" >> $file
+	    xml)
+		echo "cp $tmpx $file"
+		cp $tmpx $file
+	    ;;
+	    json)
+		cp $tmpj $file
+	    ;;
+	    tree)
+		echo "clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create"
+
+		clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create
+	    ;;
+	esac
+	new "Startup backend $format -s $mode -f $cfg -y $fyang"
+	echo "time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format"
+	# Cannot use start_backend here due to expected error case
+	time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format # 2> /dev/null	
     done
-    echo "</x></config>" >> $file
-
-    new "Startup backend once -s $mode -f $cfg -y $fyang"
-    # Cannot use start_backend here due to expected error case
-    time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
 done
 
-new "Startup backend once -s $mode -f $cfg -y $fyang"
-# Cannot use start_backend here due to expected error case
-time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
-
 new "test params: -f $cfg -y $fyang"
 if [ $BE -ne 0 ]; then
     new "kill old backend"
@@ -152,6 +189,7 @@ time -p for (( i=0; i<$perfreq; i++ )); do
     curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null
 done
 
+# Reference: i686 perfnr=10000 time: 27s 20190425 34s WITH startup copying
 new "restconf add $perfreq small config"
 time -p for (( i=0; i<$perfreq; i++ )); do
     rnd=$(( ( RANDOM % $perfnr ) ))

util/clixon_util_datastore.c

@@ -63,7 +63,7 @@
 #include <clixon/clixon.h>
 
 /* Command line options to be passed to getopt(3) */
-#define DATASTORE_OPTS "hDd:b:y:"
+#define DATASTORE_OPTS "hDd:b:f:x:y:"
 
 /*! usage
  */
@@ -76,11 +76,13 @@ usage(char *argv0)
 		"\t-D\t\tDebug\n"
 		"\t-d <db>\t\tDatabase name. Default: running. Alt: candidate,startup\n"
 		"\t-b <dir>\tDatabase directory. Mandatory\n"
+	        "\t-f <fmt>\tDatabase format: xml, json, tree\n"
+		"\t-x <xml>\tXML file. Alternative to put <xml> argument\n"
 		"\t-y <file>\tYang file. Mandatory\n"
 		"and command is either:\n"
 		"\tget [<xpath>]\n"
  	        "\tmget <nr> [<xpath>]\n"
-		"\tput (merge|replace|create|delete|remove) <xml>\n"
+		"\tput (merge|replace|create|delete|remove) [<xml>]\n"
 		"\tcopy <todb>\n"
 		"\tlock <pid>\n"
 		"\tunlock\n"
@@ -105,6 +107,7 @@ main(int argc, char **argv)
     char               *cmd = NULL;
     yang_stmt          *yspec = NULL;
     char               *yangfilename = NULL;
+    char               *xmlfilename = NULL;
     char               *dbdir = NULL;
     int                 ret;
     int                 pid;
@@ -122,6 +125,7 @@ main(int argc, char **argv)
     if ((h = clicon_handle_init()) == NULL)
 	goto done;
     /* getopt in two steps, first find config-file before over-riding options. */
+    clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", "xml"); /* default */
     while ((c = getopt(argc, argv, DATASTORE_OPTS)) != -1)
 	switch (c) {
 	case '?' :
@@ -141,6 +145,16 @@ main(int argc, char **argv)
 	        usage(argv0);
 	    dbdir = optarg;
 	    break;
+	case 'f': /* db format */
+	    if (!optarg)
+	        usage(argv0);
+	    clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", optarg);
+	    break;
+	case 'x': /* XML file */
+	    if (!optarg)
+	        usage(argv0);
+	    xmlfilename = optarg;
+	    break;
 	case 'y': /* Yang file */
 	    if (!optarg)
 	        usage(argv0);
@@ -213,7 +227,13 @@ main(int argc, char **argv)
 	fprintf(stdout, "\n");
     }
     else if (strcmp(cmd, "put")==0){
-	if (argc != 3){
+	if (argc == 2){
+	    if (xmlfilename == NULL){
+		clicon_err(OE_DB, 0, "XML filename expected");
+		usage(argv0);
+	    }
+	}
+	else if (argc != 3){
 	    clicon_err(OE_DB, 0, "Unexpected nr of args: %d", argc);
 	    usage(argv0);
 	}
@@ -221,8 +241,19 @@ main(int argc, char **argv)
 	    clicon_err(OE_DB, 0, "Unrecognized operation: %s", argv[1]);
 	    usage(argv0);
 	}
-	if (xml_parse_string(argv[2], NULL, &xt) < 0)
+	if (argc == 2){
-	    goto done;
+	    int fd;
+	    if ((fd = open(xmlfilename, O_RDONLY)) < 0){
+		clicon_err(OE_UNIX, errno, "open(%s)", xmlfilename);
+		goto done;
+	    }
+	    if (xml_parse_file(fd, "</config>", yspec, &xt) < 0)
+		goto done;
+	    close(fd);
+	}
+	else
+	    if (xml_parse_string(argv[2], yspec, &xt) < 0)
+		goto done;
 	if (xml_rootchild(xt, 0, &xt) < 0)
 	    goto done;
 	if ((cbret = cbuf_new()) == NULL){
@@ -231,7 +262,6 @@ main(int argc, char **argv)
 	}
 	if (xmldb_put(h, db, op, xt, NULL, cbret) < 1)
 	    goto done;
-	    
     }
     else if (strcmp(cmd, "copy")==0){
 	if (argc != 2)

yang/clixon/clixon-config@2019-03-05.yang

@@ -90,6 +90,10 @@ module clixon-config {
 	    enum json{
 		description "Save and load xmldb as JSON";
 	    }
+	    enum tree{
+		description "Save and load xmldb as Clixon record-based tree
+                             file format";
+	    }
 	}
     }
     typedef cli_genmodel_type{