diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5231da56..5f7e5094 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -111,6 +111,7 @@
### Minor changes
+* New XMLDB_FORMAT added: `tree`. An experimental record-based tree database for direct access of records.
* A new "hello world" example is added
* Experimental customized error output strings, see [lib/clixon/clixon_err_string.h]
* Empty leaf values, eg are now checked at validation.
diff --git a/lib/clixon/clixon_queue.h b/lib/clixon/clixon_queue.h
index ee277134..fafcc825 100644
--- a/lib/clixon/clixon_queue.h
+++ b/lib/clixon/clixon_queue.h
@@ -126,5 +126,6 @@ typedef struct _qelem_t {
* NEXTQ(struct a*, el);
*/
#define NEXTQ(type, elem) ((type)((elem)?((qelem_t *)(elem))->q_next:NULL))
+#define PREVQ(type, elem) ((type)((elem)?((qelem_t *)(elem))->q_prev:NULL))
#endif /* _CLIXON_QUEUE_H_ */
diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in
index 9ea7fa4c..23723854 100644
--- a/lib/src/Makefile.in
+++ b/lib/src/Makefile.in
@@ -75,6 +75,7 @@ SRC = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \
clixon_proto.c clixon_proto_client.c \
clixon_xpath.c clixon_xpath_ctx.c clixon_sha1.c \
clixon_datastore.c clixon_datastore_write.c clixon_datastore_read.c \
+ clixon_datastore_tree.c \
clixon_netconf_lib.c clixon_stream.c clixon_nacm.c
YACCOBJS := lex.clixon_xml_parse.o clixon_xml_parse.tab.o \
diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c
index 80d3e3c6..41753692 100644
--- a/lib/src/clixon_datastore_read.c
+++ b/lib/src/clixon_datastore_read.c
@@ -78,6 +78,7 @@
#include "clixon_datastore.h"
#include "clixon_datastore_read.h"
+#include "clixon_datastore_tree.h"
#define handle(xh) (assert(text_handle_check(xh)==0),(struct text_handle *)(xh))
@@ -322,30 +323,40 @@ xmldb_readfile(clicon_handle h,
clicon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
- if ((fd = open(dbfile, O_RDONLY)) < 0) {
- clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
+ if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
+ clicon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
- }
- /* Parse file into XML tree */
- format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
- if (format && strcmp(format, "json")==0){
- if ((json_parse_file(fd, yspec, &x0)) < 0)
+ }
+ /* Parse file into internal XML tree from different formats */
+ if (strcmp(format, "tree")==0){
+ if (datastore_tree_read(h, dbfile, &x0) < 0)
goto done;
}
- else if ((xml_parse_file(fd, "", yspec, &x0)) < 0)
- goto done;
- /* Always assert a top-level called "config".
- To ensure that, deal with two cases:
- 1. File is empty -> rename top-level to "config" */
- if (xml_child_nr(x0) == 0){
- if (xml_name_set(x0, "config") < 0)
- goto done;
- }
- /* 2. File is not empty ... -> replace root */
- else{
- /* There should only be one element and called config */
- if (singleconfigroot(x0, &x0) < 0)
+ else{
+ if ((fd = open(dbfile, O_RDONLY)) < 0) {
+ clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
goto done;
+ }
+ if (strcmp(format, "json")==0){
+ if ((json_parse_file(fd, yspec, &x0)) < 0)
+ goto done;
+ }
+ else if ((xml_parse_file(fd, "", yspec, &x0)) < 0)
+ goto done;
+
+ /* Always assert a top-level called "config".
+ To ensure that, deal with two cases:
+ 1. File is empty -> rename top-level to "config" */
+ if (xml_child_nr(x0) == 0){
+ if (xml_name_set(x0, "config") < 0)
+ goto done;
+ }
+ /* 2. File is not empty ... -> replace root */
+ else{
+ /* There should only be one element and called config */
+ if (singleconfigroot(x0, &x0) < 0)
+ goto done;
+ }
}
/* From Clixon 3.10,datastore files may contain module-state defining
* which modules are used in the file.
@@ -396,43 +407,13 @@ xmldb_get_nocache(clicon_handle h,
cxobj **xvec = NULL;
size_t xlen;
int i;
- char *format;
if ((yspec = clicon_dbspec_yang(h)) == NULL){
clicon_err(OE_YANG, ENOENT, "No yang spec");
goto done;
}
- if (xmldb_db2file(h, db, &dbfile) < 0)
+ if (xmldb_readfile(h, db, yspec, &xt, msd) < 0)
goto done;
- if (dbfile==NULL){
- clicon_err(OE_XML, 0, "dbfile NULL");
- goto done;
- }
- if ((fd = open(dbfile, O_RDONLY)) < 0){
- clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
- goto done;
- }
- /* Parse file into XML tree */
- format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
- if (format && strcmp(format, "json")==0){
- if ((json_parse_file(fd, yspec, &xt)) < 0)
- goto done;
- }
- else if ((xml_parse_file(fd, "", yspec, &xt)) < 0)
- goto done;
- /* Always assert a top-level called "config".
- To ensure that, deal with two cases:
- 1. File is empty -> rename top-level to "config" */
- if (xml_child_nr(xt) == 0){
- if (xml_name_set(xt, "config") < 0)
- goto done;
- }
- /* 2. File is not empty ... -> replace root */
- else{
- /* There should only be one element and called config */
- if (singleconfigroot(xt, &xt) < 0)
- goto done;
- }
/* Here xt looks like: ... */
/* Given the xpath, return a vector of matches in xvec */
if (xpath_vec(xt, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
@@ -595,7 +576,7 @@ xmldb_get1_cache(clicon_handle h,
const char *db,
char *xpath,
cxobj **xtop,
- modstate_diff_t *modst)
+ modstate_diff_t *msd)
{
int retval = -1;
yang_stmt *yspec;
@@ -618,7 +599,7 @@ xmldb_get1_cache(clicon_handle h,
de = clicon_db_elmnt_get(h, db);
if (de == NULL || de->de_xml == NULL){ /* Cache miss, read XML from file */
/* If there is no xml x0 tree (in cache), then read it from file */
- if (xmldb_readfile(h, db, yspec, &x0t, modst) < 0)
+ if (xmldb_readfile(h, db, yspec, &x0t, msd) < 0)
goto done;
/* XXX: should we validate file if read from disk?
* Argument against: we may want to have a semantically wrong file and wish
diff --git a/lib/src/clixon_datastore_tree.c b/lib/src/clixon_datastore_tree.c
new file mode 100644
index 00000000..91c34fdb
--- /dev/null
+++ b/lib/src/clixon_datastore_tree.c
@@ -0,0 +1,451 @@
+/*
+ *
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
+
+ This file is part of CLIXON.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Alternatively, the contents of this file may be used under the terms of
+ the GNU General Public License Version 3 or later (the "GPL"),
+ in which case the provisions of the GPL are applicable instead
+ of those above. If you wish to allow use of your version of this file only
+ under the terms of the GPL, and not to allow others to
+ use your version of this file under the terms of Apache License version 2,
+ indicate your decision by deleting the provisions above and replace them with
+ the notice and other provisions required by the GPL. If you do not delete
+ the provisions above, a recipient may use your version of this file under
+ the terms of any one of the Apache License version 2 or the GPL.
+
+ ***** END LICENSE BLOCK *****
+
+ *
+ * XML file save
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+/* cligen */
+#include
+
+/* clicon */
+
+#include "clixon_string.h"
+#include "clixon_queue.h"
+#include "clixon_hash.h"
+#include "clixon_err.h"
+#include "clixon_log.h"
+#include "clixon_handle.h"
+#include "clixon_yang.h"
+#include "clixon_xml.h"
+
+#include "clixon_datastore_tree.h"
+
+#define align4(s) (((s)/4)*4 + 4)
+
+struct indexlist{
+ qelem_t il_q; /* this must be there */
+ uint32_t il_i;
+ int il_len; /* How many */
+};
+typedef struct indexlist indexlist;
+
+struct indexhead{
+ indexlist *ih_list;
+};
+typedef struct indexhead indexhead;
+
+int
+index_get(indexhead *ih,
+ size_t len, /* in bytes */
+ uint32_t *indexp)
+{
+ int retval = -1;
+ int nr;
+ uint32_t index = 0;
+ struct indexlist *il;
+ indexlist *new;
+
+ nr = len/DF_BLOCK + 1;
+ if ((new = malloc(sizeof(indexlist))) == NULL){
+ clicon_err(OE_XML, errno, "malloc");
+ goto done;
+ }
+ memset(new, 0, sizeof(indexlist));
+ // new->il_i = i; Dont know i yet
+ new->il_len = nr;
+ if ((il = ih->ih_list) != NULL) {
+ il = PREVQ(indexlist*, il);
+ index = il->il_i + il->il_len;
+ }
+ ADDQ(new, ih->ih_list);
+ new->il_i = index;
+ *indexp = index;
+ retval = 0;
+ done:
+ return retval;
+}
+#ifdef NOTYET
+int
+index_get(indexhead *ih,
+ size_t len, /* in bytes */
+ uint32_t *indexp)
+{
+ int retval = -1;
+ int nr;
+ uint32_t index = 0;
+ struct indexlist *il;
+ indexlist *new;
+
+ nr = len/DF_BLOCK + 1;
+ if ((new = malloc(sizeof(indexlist))) == NULL){
+ clicon_err(OE_XML, errno, "malloc");
+ goto done;
+ }
+ memset(new, 0, sizeof(indexlist));
+ // new->il_i = i; Dont know i yet
+ new->il_len = nr;
+
+ /* Find nr conseq blocks */
+ if ((il = ih->ih_list) != NULL) {
+ do {
+ if (il->il_i - index >= nr){ /* Found */
+ INSQ(new, il);
+ break;
+ }
+ index = il->il_i + il->il_len;
+ il = NEXTQ(indexlist*, il);
+ } while (il != ih->ih_list);
+ ADDQ(new, il); /* after? */
+ }
+ else
+ INSQ(new, ih->ih_list);
+ new->il_i = index;
+ *indexp = index;
+ retval = 0;
+ done:
+ return retval;
+}
+#endif
+
+int
+index_dump(FILE *f,
+ indexhead *ih)
+{
+ indexlist *il;
+
+ if ((il = ih->ih_list) != NULL) {
+ do {
+ fprintf(f, "%u %d\n", il->il_i, il->il_len);
+ il = NEXTQ(indexlist*, il);
+ } while (il != ih->ih_list);
+ }
+ return 0;
+}
+
+/*
+ * @param[out] xp
+ * @param[out] vec If type is CX_ELMNT, points to vector of indexes,...
+ * @retval -1 Error
+ * @retval 0 Sanity check failed
+ * @retval 1 OK xp set (or NULL if empty)
+ */
+static int
+buf2xml(FILE *f,
+ uint32_t index,
+ cxobj **xp)
+{
+ int retval = -1;
+ uint8_t ver;
+ uint8_t type;
+ int ptr;
+ uint32_t len;
+ cxobj *x = NULL;
+ char *name;
+ char *prefix = NULL;
+ char hdr[8];
+ char *buf = NULL;
+ int vlen;
+ int i;
+ uint32_t ind;
+ cxobj *xc;
+
+ /* Read hdr
+ * +-----+-----+-----+-----+-----+-----+-----+-----+
+ * | ver | type| de | ad | len |
+ * +-----+-----+-----+-----+-----+-----+-----+-----+
+ */
+ if (fseek(f, index*DF_BLOCK, SEEK_SET) < 0){
+ clicon_err(OE_UNIX, errno, "fseek");
+ goto done;
+ }
+ if (fread(hdr, sizeof(char), sizeof(hdr), f) < 0){
+ clicon_err(OE_XML, errno, "fread");
+ goto done;
+ }
+ if ((ver = hdr[0]&0xff) == 0) /* Means empty */
+ goto ok;
+ if (ver != 1){
+ clicon_debug(1, "%s Wrong version: %d", __FUNCTION__, ver);
+ goto fail;
+ }
+ type = hdr[1]&0x0f;
+ if (type != CX_BODY && type != CX_ELMNT && type != CX_ATTR){
+ clicon_debug(1, "%s Wrong type: %d", __FUNCTION__, type);
+ goto fail;
+ }
+ if ((hdr[2]&0xff) != 0xde){
+ clicon_debug(1, "%s Expected 0xde: %x", __FUNCTION__, hdr[2]&0xff);
+ goto fail;
+ }
+ if ((hdr[3]&0xff) != 0xad){
+ clicon_debug(1, "%s Expected 0xad: %x", __FUNCTION__, hdr[3]&0xff);
+ goto fail;
+ }
+ /* Copy and byte-swap length field */
+ memcpy(&len, &hdr[4], 4);
+ len = ntohl(len);
+
+ /* Read rest
+ */
+ ptr = 0;
+ if ((buf = malloc(len - sizeof(hdr))) == NULL){
+ clicon_err(OE_UNIX, errno, "malloc");
+ goto done;
+ }
+ if (fread(buf, sizeof(char), len-sizeof(hdr), f) < 0){
+ clicon_err(OE_XML, errno, "fread");
+ goto done;
+ }
+ if ((name = strchr(buf, ':')) != NULL){
+ *name = '\0';
+ name++;
+ prefix = buf;
+ ptr += strlen(prefix)+1;
+ }
+ else
+ name = buf;
+ ptr += strlen(name) + 1;
+ ptr = align4(ptr);
+ if ((x = xml_new(name, NULL, NULL)) == NULL)
+ goto done;
+ if (prefix)
+ if (xml_prefix_set(x, prefix) < 0)
+ goto done;
+ xml_type_set(x, type);
+ if (xml_type(x) != CX_ELMNT){
+ if (xml_value_set(x, &buf[ptr]) < 0)
+ goto done;
+ ptr += strlen(xml_value(x)) +1;
+ }
+ else{
+ vlen = (len - sizeof(hdr) - ptr)/ sizeof(uint32_t);
+ if (vlen < 0){
+ clicon_debug(1, "%s Nr of elements should not be negative: %d", __FUNCTION__, vlen);
+ goto fail;
+ }
+ for (i=0; i $fyang
module scaling{
yang-version 1.1;
@@ -50,6 +52,7 @@ cat < $cfg
false
$dir
false
+ $format
example
/usr/local/lib/example/cli
/usr/local/lib/example/clispec
@@ -66,27 +69,61 @@ if [ $BE -ne 0 ]; then
err
fi
fi
-# Try startup mode w startup
+
+# First generate large XML file
+# Use it latter to generate startup-db in xml, tree formats
+tmpx=$dir/tmp.xml
+new "generate large startup config ($tmpx) with $perfnr entries"
+echo -n "" > $tmpx
+for (( i=0; i<$perfnr; i++ )); do
+ echo -n "$i$i" >> $tmpx
+done
+echo "" >> $tmpx
+
+if false; then
+# Then generate large JSON file (cant translate namespace - long story)
+tmpj=$dir/tmp.json
+new "generate large startup config ($tmpj) with $perfnr entries"
+echo -n '{"config": {"scaling:x":{"y":[' > $tmpj
+for (( i=0; i<$perfnr; i++ )); do
+ if [ $i -ne 0 ]; then
+ echo -n ",{\"a\":$i,\"b\":$i}" >> $tmpj
+ else
+ echo -n "{\"a\":$i,\"b\":$i}" >> $tmpj
+ fi
+
+done
+echo "]}}}" >> $tmpj
+fi
+
+# Loop over mode and format
for mode in startup running; do
file=$dir/${mode}_db
- sudo touch $file
- sudo chmod 666 $file
- new "generate large startup config ($file) with $perfnr list entries in mode $mode"
- echo -n "" > $file
- for (( i=0; i<$perfnr; i++ )); do
- echo -n "$i$i" >> $file
+ for format in tree xml; do # json - something w namespaces
+ sudo rm -f $file
+ sudo touch $file
+ sudo chmod 666 $file
+ case $format in
+ xml)
+ echo "cp $tmpx $file"
+ cp $tmpx $file
+ ;;
+ json)
+ cp $tmpj $file
+ ;;
+ tree)
+ echo "clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create"
+
+ clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create
+ ;;
+ esac
+ new "Startup backend $format -s $mode -f $cfg -y $fyang"
+ echo "time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format"
+ # Cannot use start_backend here due to expected error case
+ time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format # 2> /dev/null
done
- echo "" >> $file
-
- new "Startup backend once -s $mode -f $cfg -y $fyang"
- # Cannot use start_backend here due to expected error case
- time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
done
-new "Startup backend once -s $mode -f $cfg -y $fyang"
-# Cannot use start_backend here due to expected error case
-time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
-
new "test params: -f $cfg -y $fyang"
if [ $BE -ne 0 ]; then
new "kill old backend"
@@ -152,6 +189,7 @@ time -p for (( i=0; i<$perfreq; i++ )); do
curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null
done
+# Reference: i686 perfnr=10000 time: 27s 20190425 34s WITH startup copying
new "restconf add $perfreq small config"
time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) ))
diff --git a/util/clixon_util_datastore.c b/util/clixon_util_datastore.c
index 287351e8..3b0e5911 100644
--- a/util/clixon_util_datastore.c
+++ b/util/clixon_util_datastore.c
@@ -63,7 +63,7 @@
#include
/* Command line options to be passed to getopt(3) */
-#define DATASTORE_OPTS "hDd:b:y:"
+#define DATASTORE_OPTS "hDd:b:f:x:y:"
/*! usage
*/
@@ -76,11 +76,13 @@ usage(char *argv0)
"\t-D\t\tDebug\n"
"\t-d \t\tDatabase name. Default: running. Alt: candidate,startup\n"
"\t-b \tDatabase directory. Mandatory\n"
+ "\t-f \tDatabase format: xml, json, tree\n"
+ "\t-x \tXML file. Alternative to put argument\n"
"\t-y \tYang file. Mandatory\n"
"and command is either:\n"
"\tget []\n"
"\tmget []\n"
- "\tput (merge|replace|create|delete|remove) \n"
+ "\tput (merge|replace|create|delete|remove) []\n"
"\tcopy \n"
"\tlock \n"
"\tunlock\n"
@@ -105,6 +107,7 @@ main(int argc, char **argv)
char *cmd = NULL;
yang_stmt *yspec = NULL;
char *yangfilename = NULL;
+ char *xmlfilename = NULL;
char *dbdir = NULL;
int ret;
int pid;
@@ -122,6 +125,7 @@ main(int argc, char **argv)
if ((h = clicon_handle_init()) == NULL)
goto done;
/* getopt in two steps, first find config-file before over-riding options. */
+ clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", "xml"); /* default */
while ((c = getopt(argc, argv, DATASTORE_OPTS)) != -1)
switch (c) {
case '?' :
@@ -141,6 +145,16 @@ main(int argc, char **argv)
usage(argv0);
dbdir = optarg;
break;
+ case 'f': /* db format */
+ if (!optarg)
+ usage(argv0);
+ clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", optarg);
+ break;
+ case 'x': /* XML file */
+ if (!optarg)
+ usage(argv0);
+ xmlfilename = optarg;
+ break;
case 'y': /* Yang file */
if (!optarg)
usage(argv0);
@@ -213,7 +227,13 @@ main(int argc, char **argv)
fprintf(stdout, "\n");
}
else if (strcmp(cmd, "put")==0){
- if (argc != 3){
+ if (argc == 2){
+ if (xmlfilename == NULL){
+ clicon_err(OE_DB, 0, "XML filename expected");
+ usage(argv0);
+ }
+ }
+ else if (argc != 3){
clicon_err(OE_DB, 0, "Unexpected nr of args: %d", argc);
usage(argv0);
}
@@ -221,8 +241,19 @@ main(int argc, char **argv)
clicon_err(OE_DB, 0, "Unrecognized operation: %s", argv[1]);
usage(argv0);
}
- if (xml_parse_string(argv[2], NULL, &xt) < 0)
- goto done;
+ if (argc == 2){
+ int fd;
+ if ((fd = open(xmlfilename, O_RDONLY)) < 0){
+ clicon_err(OE_UNIX, errno, "open(%s)", xmlfilename);
+ goto done;
+ }
+ if (xml_parse_file(fd, "", yspec, &xt) < 0)
+ goto done;
+ close(fd);
+ }
+ else
+ if (xml_parse_string(argv[2], yspec, &xt) < 0)
+ goto done;
if (xml_rootchild(xt, 0, &xt) < 0)
goto done;
if ((cbret = cbuf_new()) == NULL){
@@ -231,7 +262,6 @@ main(int argc, char **argv)
}
if (xmldb_put(h, db, op, xt, NULL, cbret) < 1)
goto done;
-
}
else if (strcmp(cmd, "copy")==0){
if (argc != 2)
diff --git a/yang/clixon/clixon-config@2019-03-05.yang b/yang/clixon/clixon-config@2019-03-05.yang
index d3449cee..d8a1a366 100644
--- a/yang/clixon/clixon-config@2019-03-05.yang
+++ b/yang/clixon/clixon-config@2019-03-05.yang
@@ -90,6 +90,10 @@ module clixon-config {
enum json{
description "Save and load xmldb as JSON";
}
+ enum tree{
+ description "Save and load xmldb as Clixon record-based tree
+ file format";
+ }
}
}
typedef cli_genmodel_type{