diff --git a/CHANGELOG.md b/CHANGELOG.md index 5231da56..5f7e5094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,7 @@ ### Minor changes +* New XMLDB_FORMAT added: `tree`. An experimental record-based tree database for direct access of records. * A new "hello world" example is added * Experimental customized error output strings, see [lib/clixon/clixon_err_string.h] * Empty leaf values, eg are now checked at validation. diff --git a/lib/clixon/clixon_queue.h b/lib/clixon/clixon_queue.h index ee277134..fafcc825 100644 --- a/lib/clixon/clixon_queue.h +++ b/lib/clixon/clixon_queue.h @@ -126,5 +126,6 @@ typedef struct _qelem_t { * NEXTQ(struct a*, el); */ #define NEXTQ(type, elem) ((type)((elem)?((qelem_t *)(elem))->q_next:NULL)) +#define PREVQ(type, elem) ((type)((elem)?((qelem_t *)(elem))->q_prev:NULL)) #endif /* _CLIXON_QUEUE_H_ */ diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index 9ea7fa4c..23723854 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -75,6 +75,7 @@ SRC = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \ clixon_proto.c clixon_proto_client.c \ clixon_xpath.c clixon_xpath_ctx.c clixon_sha1.c \ clixon_datastore.c clixon_datastore_write.c clixon_datastore_read.c \ + clixon_datastore_tree.c \ clixon_netconf_lib.c clixon_stream.c clixon_nacm.c YACCOBJS := lex.clixon_xml_parse.o clixon_xml_parse.tab.o \ diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index 80d3e3c6..41753692 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -78,6 +78,7 @@ #include "clixon_datastore.h" #include "clixon_datastore_read.h" +#include "clixon_datastore_tree.h" #define handle(xh) (assert(text_handle_check(xh)==0),(struct text_handle *)(xh)) @@ -322,30 +323,40 @@ xmldb_readfile(clicon_handle h, clicon_err(OE_XML, 0, "dbfile NULL"); goto done; } - if ((fd = open(dbfile, O_RDONLY)) < 0) { - clicon_err(OE_UNIX, errno, "open(%s)", dbfile); + if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){ + clicon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT"); goto done; - } - /* Parse file into XML tree */ - format = clicon_option_str(h, "CLICON_XMLDB_FORMAT"); - if (format && strcmp(format, "json")==0){ - if ((json_parse_file(fd, yspec, &x0)) < 0) + } + /* Parse file into internal XML tree from different formats */ + if (strcmp(format, "tree")==0){ + if (datastore_tree_read(h, dbfile, &x0) < 0) goto done; } - else if ((xml_parse_file(fd, "", yspec, &x0)) < 0) - goto done; - /* Always assert a top-level called "config". - To ensure that, deal with two cases: - 1. File is empty -> rename top-level to "config" */ - if (xml_child_nr(x0) == 0){ - if (xml_name_set(x0, "config") < 0) - goto done; - } - /* 2. File is not empty ... -> replace root */ - else{ - /* There should only be one element and called config */ - if (singleconfigroot(x0, &x0) < 0) + else{ + if ((fd = open(dbfile, O_RDONLY)) < 0) { + clicon_err(OE_UNIX, errno, "open(%s)", dbfile); goto done; + } + if (strcmp(format, "json")==0){ + if ((json_parse_file(fd, yspec, &x0)) < 0) + goto done; + } + else if ((xml_parse_file(fd, "", yspec, &x0)) < 0) + goto done; + + /* Always assert a top-level called "config". + To ensure that, deal with two cases: + 1. File is empty -> rename top-level to "config" */ + if (xml_child_nr(x0) == 0){ + if (xml_name_set(x0, "config") < 0) + goto done; + } + /* 2. File is not empty ... -> replace root */ + else{ + /* There should only be one element and called config */ + if (singleconfigroot(x0, &x0) < 0) + goto done; + } } /* From Clixon 3.10,datastore files may contain module-state defining * which modules are used in the file. @@ -396,43 +407,13 @@ xmldb_get_nocache(clicon_handle h, cxobj **xvec = NULL; size_t xlen; int i; - char *format; if ((yspec = clicon_dbspec_yang(h)) == NULL){ clicon_err(OE_YANG, ENOENT, "No yang spec"); goto done; } - if (xmldb_db2file(h, db, &dbfile) < 0) + if (xmldb_readfile(h, db, yspec, &xt, msd) < 0) goto done; - if (dbfile==NULL){ - clicon_err(OE_XML, 0, "dbfile NULL"); - goto done; - } - if ((fd = open(dbfile, O_RDONLY)) < 0){ - clicon_err(OE_UNIX, errno, "open(%s)", dbfile); - goto done; - } - /* Parse file into XML tree */ - format = clicon_option_str(h, "CLICON_XMLDB_FORMAT"); - if (format && strcmp(format, "json")==0){ - if ((json_parse_file(fd, yspec, &xt)) < 0) - goto done; - } - else if ((xml_parse_file(fd, "", yspec, &xt)) < 0) - goto done; - /* Always assert a top-level called "config". - To ensure that, deal with two cases: - 1. File is empty -> rename top-level to "config" */ - if (xml_child_nr(xt) == 0){ - if (xml_name_set(xt, "config") < 0) - goto done; - } - /* 2. File is not empty ... -> replace root */ - else{ - /* There should only be one element and called config */ - if (singleconfigroot(xt, &xt) < 0) - goto done; - } /* Here xt looks like: ... */ /* Given the xpath, return a vector of matches in xvec */ if (xpath_vec(xt, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) @@ -595,7 +576,7 @@ xmldb_get1_cache(clicon_handle h, const char *db, char *xpath, cxobj **xtop, - modstate_diff_t *modst) + modstate_diff_t *msd) { int retval = -1; yang_stmt *yspec; @@ -618,7 +599,7 @@ xmldb_get1_cache(clicon_handle h, de = clicon_db_elmnt_get(h, db); if (de == NULL || de->de_xml == NULL){ /* Cache miss, read XML from file */ /* If there is no xml x0 tree (in cache), then read it from file */ - if (xmldb_readfile(h, db, yspec, &x0t, modst) < 0) + if (xmldb_readfile(h, db, yspec, &x0t, msd) < 0) goto done; /* XXX: should we validate file if read from disk? * Argument against: we may want to have a semantically wrong file and wish diff --git a/lib/src/clixon_datastore_tree.c b/lib/src/clixon_datastore_tree.c new file mode 100644 index 00000000..91c34fdb --- /dev/null +++ b/lib/src/clixon_datastore_tree.c @@ -0,0 +1,451 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * + * XML file save + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* cligen */ +#include + +/* clicon */ + +#include "clixon_string.h" +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_err.h" +#include "clixon_log.h" +#include "clixon_handle.h" +#include "clixon_yang.h" +#include "clixon_xml.h" + +#include "clixon_datastore_tree.h" + +#define align4(s) (((s)/4)*4 + 4) + +struct indexlist{ + qelem_t il_q; /* this must be there */ + uint32_t il_i; + int il_len; /* How many */ +}; +typedef struct indexlist indexlist; + +struct indexhead{ + indexlist *ih_list; +}; +typedef struct indexhead indexhead; + +int +index_get(indexhead *ih, + size_t len, /* in bytes */ + uint32_t *indexp) +{ + int retval = -1; + int nr; + uint32_t index = 0; + struct indexlist *il; + indexlist *new; + + nr = len/DF_BLOCK + 1; + if ((new = malloc(sizeof(indexlist))) == NULL){ + clicon_err(OE_XML, errno, "malloc"); + goto done; + } + memset(new, 0, sizeof(indexlist)); + // new->il_i = i; Dont know i yet + new->il_len = nr; + if ((il = ih->ih_list) != NULL) { + il = PREVQ(indexlist*, il); + index = il->il_i + il->il_len; + } + ADDQ(new, ih->ih_list); + new->il_i = index; + *indexp = index; + retval = 0; + done: + return retval; +} +#ifdef NOTYET +int +index_get(indexhead *ih, + size_t len, /* in bytes */ + uint32_t *indexp) +{ + int retval = -1; + int nr; + uint32_t index = 0; + struct indexlist *il; + indexlist *new; + + nr = len/DF_BLOCK + 1; + if ((new = malloc(sizeof(indexlist))) == NULL){ + clicon_err(OE_XML, errno, "malloc"); + goto done; + } + memset(new, 0, sizeof(indexlist)); + // new->il_i = i; Dont know i yet + new->il_len = nr; + + /* Find nr conseq blocks */ + if ((il = ih->ih_list) != NULL) { + do { + if (il->il_i - index >= nr){ /* Found */ + INSQ(new, il); + break; + } + index = il->il_i + il->il_len; + il = NEXTQ(indexlist*, il); + } while (il != ih->ih_list); + ADDQ(new, il); /* after? */ + } + else + INSQ(new, ih->ih_list); + new->il_i = index; + *indexp = index; + retval = 0; + done: + return retval; +} +#endif + +int +index_dump(FILE *f, + indexhead *ih) +{ + indexlist *il; + + if ((il = ih->ih_list) != NULL) { + do { + fprintf(f, "%u %d\n", il->il_i, il->il_len); + il = NEXTQ(indexlist*, il); + } while (il != ih->ih_list); + } + return 0; +} + +/* + * @param[out] xp + * @param[out] vec If type is CX_ELMNT, points to vector of indexes,... + * @retval -1 Error + * @retval 0 Sanity check failed + * @retval 1 OK xp set (or NULL if empty) + */ +static int +buf2xml(FILE *f, + uint32_t index, + cxobj **xp) +{ + int retval = -1; + uint8_t ver; + uint8_t type; + int ptr; + uint32_t len; + cxobj *x = NULL; + char *name; + char *prefix = NULL; + char hdr[8]; + char *buf = NULL; + int vlen; + int i; + uint32_t ind; + cxobj *xc; + + /* Read hdr + * +-----+-----+-----+-----+-----+-----+-----+-----+ + * | ver | type| de | ad | len | + * +-----+-----+-----+-----+-----+-----+-----+-----+ + */ + if (fseek(f, index*DF_BLOCK, SEEK_SET) < 0){ + clicon_err(OE_UNIX, errno, "fseek"); + goto done; + } + if (fread(hdr, sizeof(char), sizeof(hdr), f) < 0){ + clicon_err(OE_XML, errno, "fread"); + goto done; + } + if ((ver = hdr[0]&0xff) == 0) /* Means empty */ + goto ok; + if (ver != 1){ + clicon_debug(1, "%s Wrong version: %d", __FUNCTION__, ver); + goto fail; + } + type = hdr[1]&0x0f; + if (type != CX_BODY && type != CX_ELMNT && type != CX_ATTR){ + clicon_debug(1, "%s Wrong type: %d", __FUNCTION__, type); + goto fail; + } + if ((hdr[2]&0xff) != 0xde){ + clicon_debug(1, "%s Expected 0xde: %x", __FUNCTION__, hdr[2]&0xff); + goto fail; + } + if ((hdr[3]&0xff) != 0xad){ + clicon_debug(1, "%s Expected 0xad: %x", __FUNCTION__, hdr[3]&0xff); + goto fail; + } + /* Copy and byte-swap length field */ + memcpy(&len, &hdr[4], 4); + len = ntohl(len); + + /* Read rest + */ + ptr = 0; + if ((buf = malloc(len - sizeof(hdr))) == NULL){ + clicon_err(OE_UNIX, errno, "malloc"); + goto done; + } + if (fread(buf, sizeof(char), len-sizeof(hdr), f) < 0){ + clicon_err(OE_XML, errno, "fread"); + goto done; + } + if ((name = strchr(buf, ':')) != NULL){ + *name = '\0'; + name++; + prefix = buf; + ptr += strlen(prefix)+1; + } + else + name = buf; + ptr += strlen(name) + 1; + ptr = align4(ptr); + if ((x = xml_new(name, NULL, NULL)) == NULL) + goto done; + if (prefix) + if (xml_prefix_set(x, prefix) < 0) + goto done; + xml_type_set(x, type); + if (xml_type(x) != CX_ELMNT){ + if (xml_value_set(x, &buf[ptr]) < 0) + goto done; + ptr += strlen(xml_value(x)) +1; + } + else{ + vlen = (len - sizeof(hdr) - ptr)/ sizeof(uint32_t); + if (vlen < 0){ + clicon_debug(1, "%s Nr of elements should not be negative: %d", __FUNCTION__, vlen); + goto fail; + } + for (i=0; i $fyang module scaling{ yang-version 1.1; @@ -50,6 +52,7 @@ cat < $cfg false $dir false + $format example /usr/local/lib/example/cli /usr/local/lib/example/clispec @@ -66,27 +69,61 @@ if [ $BE -ne 0 ]; then err fi fi -# Try startup mode w startup + +# First generate large XML file +# Use it latter to generate startup-db in xml, tree formats +tmpx=$dir/tmp.xml +new "generate large startup config ($tmpx) with $perfnr entries" +echo -n "" > $tmpx +for (( i=0; i<$perfnr; i++ )); do + echo -n "$i$i" >> $tmpx +done +echo "" >> $tmpx + +if false; then +# Then generate large JSON file (cant translate namespace - long story) +tmpj=$dir/tmp.json +new "generate large startup config ($tmpj) with $perfnr entries" +echo -n '{"config": {"scaling:x":{"y":[' > $tmpj +for (( i=0; i<$perfnr; i++ )); do + if [ $i -ne 0 ]; then + echo -n ",{\"a\":$i,\"b\":$i}" >> $tmpj + else + echo -n "{\"a\":$i,\"b\":$i}" >> $tmpj + fi + +done +echo "]}}}" >> $tmpj +fi + +# Loop over mode and format for mode in startup running; do file=$dir/${mode}_db - sudo touch $file - sudo chmod 666 $file - new "generate large startup config ($file) with $perfnr list entries in mode $mode" - echo -n "" > $file - for (( i=0; i<$perfnr; i++ )); do - echo -n "$i$i" >> $file + for format in tree xml; do # json - something w namespaces + sudo rm -f $file + sudo touch $file + sudo chmod 666 $file + case $format in + xml) + echo "cp $tmpx $file" + cp $tmpx $file + ;; + json) + cp $tmpj $file + ;; + tree) + echo "clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create" + + clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create + ;; + esac + new "Startup backend $format -s $mode -f $cfg -y $fyang" + echo "time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format" + # Cannot use start_backend here due to expected error case + time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format # 2> /dev/null done - echo "" >> $file - - new "Startup backend once -s $mode -f $cfg -y $fyang" - # Cannot use start_backend here due to expected error case - time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null done -new "Startup backend once -s $mode -f $cfg -y $fyang" -# Cannot use start_backend here due to expected error case -time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null - new "test params: -f $cfg -y $fyang" if [ $BE -ne 0 ]; then new "kill old backend" @@ -152,6 +189,7 @@ time -p for (( i=0; i<$perfreq; i++ )); do curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null done +# Reference: i686 perfnr=10000 time: 27s 20190425 34s WITH startup copying new "restconf add $perfreq small config" time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) diff --git a/util/clixon_util_datastore.c b/util/clixon_util_datastore.c index 287351e8..3b0e5911 100644 --- a/util/clixon_util_datastore.c +++ b/util/clixon_util_datastore.c @@ -63,7 +63,7 @@ #include /* Command line options to be passed to getopt(3) */ -#define DATASTORE_OPTS "hDd:b:y:" +#define DATASTORE_OPTS "hDd:b:f:x:y:" /*! usage */ @@ -76,11 +76,13 @@ usage(char *argv0) "\t-D\t\tDebug\n" "\t-d \t\tDatabase name. Default: running. Alt: candidate,startup\n" "\t-b \tDatabase directory. Mandatory\n" + "\t-f \tDatabase format: xml, json, tree\n" + "\t-x \tXML file. Alternative to put argument\n" "\t-y \tYang file. Mandatory\n" "and command is either:\n" "\tget []\n" "\tmget []\n" - "\tput (merge|replace|create|delete|remove) \n" + "\tput (merge|replace|create|delete|remove) []\n" "\tcopy \n" "\tlock \n" "\tunlock\n" @@ -105,6 +107,7 @@ main(int argc, char **argv) char *cmd = NULL; yang_stmt *yspec = NULL; char *yangfilename = NULL; + char *xmlfilename = NULL; char *dbdir = NULL; int ret; int pid; @@ -122,6 +125,7 @@ main(int argc, char **argv) if ((h = clicon_handle_init()) == NULL) goto done; /* getopt in two steps, first find config-file before over-riding options. */ + clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", "xml"); /* default */ while ((c = getopt(argc, argv, DATASTORE_OPTS)) != -1) switch (c) { case '?' : @@ -141,6 +145,16 @@ main(int argc, char **argv) usage(argv0); dbdir = optarg; break; + case 'f': /* db format */ + if (!optarg) + usage(argv0); + clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", optarg); + break; + case 'x': /* XML file */ + if (!optarg) + usage(argv0); + xmlfilename = optarg; + break; case 'y': /* Yang file */ if (!optarg) usage(argv0); @@ -213,7 +227,13 @@ main(int argc, char **argv) fprintf(stdout, "\n"); } else if (strcmp(cmd, "put")==0){ - if (argc != 3){ + if (argc == 2){ + if (xmlfilename == NULL){ + clicon_err(OE_DB, 0, "XML filename expected"); + usage(argv0); + } + } + else if (argc != 3){ clicon_err(OE_DB, 0, "Unexpected nr of args: %d", argc); usage(argv0); } @@ -221,8 +241,19 @@ main(int argc, char **argv) clicon_err(OE_DB, 0, "Unrecognized operation: %s", argv[1]); usage(argv0); } - if (xml_parse_string(argv[2], NULL, &xt) < 0) - goto done; + if (argc == 2){ + int fd; + if ((fd = open(xmlfilename, O_RDONLY)) < 0){ + clicon_err(OE_UNIX, errno, "open(%s)", xmlfilename); + goto done; + } + if (xml_parse_file(fd, "", yspec, &xt) < 0) + goto done; + close(fd); + } + else + if (xml_parse_string(argv[2], yspec, &xt) < 0) + goto done; if (xml_rootchild(xt, 0, &xt) < 0) goto done; if ((cbret = cbuf_new()) == NULL){ @@ -231,7 +262,6 @@ main(int argc, char **argv) } if (xmldb_put(h, db, op, xt, NULL, cbret) < 1) goto done; - } else if (strcmp(cmd, "copy")==0){ if (argc != 2) diff --git a/yang/clixon/clixon-config@2019-03-05.yang b/yang/clixon/clixon-config@2019-03-05.yang index d3449cee..d8a1a366 100644 --- a/yang/clixon/clixon-config@2019-03-05.yang +++ b/yang/clixon/clixon-config@2019-03-05.yang @@ -90,6 +90,10 @@ module clixon-config { enum json{ description "Save and load xmldb as JSON"; } + enum tree{ + description "Save and load xmldb as Clixon record-based tree + file format"; + } } } typedef cli_genmodel_type{