experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

New XMLDB_FORMAT added: tree. An experimental record-based tree database for direct access of records.

Browse source
This commit is contained in:
Olof hagsand 2019-04-26 13:38:55 +02:00
parent 7847e74c5e
commit 50ca7b7845
10 changed files with 692 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -111,6 +111,7 @@
### Minor changes
* New XMLDB_FORMAT added: `tree`. An experimental record-based tree database for direct access of records.
* A new "hello world" example is added
* Experimental customized error output strings, see [lib/clixon/clixon_err_string.h]
* Empty leaf values, eg <a></a> are now checked at validation.

1
lib/clixon/clixon_queue.h
View file

@ -126,5 +126,6 @@ typedef struct _qelem_t {
* NEXTQ(struct a*, el);
*/
#define NEXTQ(type, elem) ((type)((elem)?((qelem_t *)(elem))->q_next:NULL))
#define PREVQ(type, elem) ((type)((elem)?((qelem_t *)(elem))->q_prev:NULL))
#endif /* _CLIXON_QUEUE_H_ */

1
lib/src/Makefile.in
View file

@ -75,6 +75,7 @@ SRC = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \
clixon_proto.c clixon_proto_client.c \
clixon_xpath.c clixon_xpath_ctx.c clixon_sha1.c \
clixon_datastore.c clixon_datastore_write.c clixon_datastore_read.c \
clixon_datastore_tree.c \
clixon_netconf_lib.c clixon_stream.c clixon_nacm.c
YACCOBJS := lex.clixon_xml_parse.o clixon_xml_parse.tab.o \

83
lib/src/clixon_datastore_read.c
View file

@ -78,6 +78,7 @@
#include "clixon_datastore.h"
#include "clixon_datastore_read.h"
#include "clixon_datastore_tree.h"
#define handle(xh) (assert(text_handle_check(xh)==0),(struct text_handle *)(xh))
@ -322,30 +323,40 @@ xmldb_readfile(clicon_handle h,
clicon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
if ((fd = open(dbfile, O_RDONLY)) < 0) {
clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
clicon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
}
/* Parse file into XML tree */
format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
if (format && strcmp(format, "json")==0){
if ((json_parse_file(fd, yspec, &x0)) < 0)
/* Parse file into internal XML tree from different formats */
if (strcmp(format, "tree")==0){
if (datastore_tree_read(h, dbfile, &x0) < 0)
goto done;
}
else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
goto done;
/* Always assert a top-level called "config".
To ensure that, deal with two cases:
1. File is empty <top/> -> rename top-level to "config" */
if (xml_child_nr(x0) == 0){
if (xml_name_set(x0, "config") < 0)
goto done;
}
/* 2. File is not empty <top><config>...</config></top> -> replace root */
else{
/* There should only be one element and called config */
if (singleconfigroot(x0, &x0) < 0)
if ((fd = open(dbfile, O_RDONLY)) < 0) {
clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
goto done;
}
if (strcmp(format, "json")==0){
if ((json_parse_file(fd, yspec, &x0)) < 0)
goto done;
}
else if ((xml_parse_file(fd, "</config>", yspec, &x0)) < 0)
goto done;
/* Always assert a top-level called "config".
To ensure that, deal with two cases:
1. File is empty <top/> -> rename top-level to "config" */
if (xml_child_nr(x0) == 0){
if (xml_name_set(x0, "config") < 0)
goto done;
}
/* 2. File is not empty <top><config>...</config></top> -> replace root */
else{
/* There should only be one element and called config */
if (singleconfigroot(x0, &x0) < 0)
goto done;
}
}
/* From Clixon 3.10,datastore files may contain module-state defining
* which modules are used in the file.
@ -396,43 +407,13 @@ xmldb_get_nocache(clicon_handle h,
cxobj **xvec = NULL;
size_t xlen;
int i;
char *format;
if ((yspec = clicon_dbspec_yang(h)) == NULL){
clicon_err(OE_YANG, ENOENT, "No yang spec");
goto done;
}
if (xmldb_db2file(h, db, &dbfile) < 0)
if (xmldb_readfile(h, db, yspec, &xt, msd) < 0)
goto done;
if (dbfile==NULL){
clicon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
if ((fd = open(dbfile, O_RDONLY)) < 0){
clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
goto done;
}
/* Parse file into XML tree */
format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
if (format && strcmp(format, "json")==0){
if ((json_parse_file(fd, yspec, &xt)) < 0)
goto done;
}
else if ((xml_parse_file(fd, "</config>", yspec, &xt)) < 0)
goto done;
/* Always assert a top-level called "config".
To ensure that, deal with two cases:
1. File is empty <top/> -> rename top-level to "config" */
if (xml_child_nr(xt) == 0){
if (xml_name_set(xt, "config") < 0)
goto done;
}
/* 2. File is not empty <top><config>...</config></top> -> replace root */
else{
/* There should only be one element and called config */
if (singleconfigroot(xt, &xt) < 0)
goto done;
}
/* Here xt looks like: <config>...</config> */
/* Given the xpath, return a vector of matches in xvec */
if (xpath_vec(xt, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
@ -595,7 +576,7 @@ xmldb_get1_cache(clicon_handle h,
const char *db,
char *xpath,
cxobj **xtop,
modstate_diff_t *modst)
modstate_diff_t *msd)
{
int retval = -1;
yang_stmt *yspec;
@ -618,7 +599,7 @@ xmldb_get1_cache(clicon_handle h,
de = clicon_db_elmnt_get(h, db);
if (de == NULL || de->de_xml == NULL){ /* Cache miss, read XML from file */
/* If there is no xml x0 tree (in cache), then read it from file */
if (xmldb_readfile(h, db, yspec, &x0t, modst) < 0)
if (xmldb_readfile(h, db, yspec, &x0t, msd) < 0)
goto done;
/* XXX: should we validate file if read from disk?
* Argument against: we may want to have a semantically wrong file and wish

451
lib/src/clixon_datastore_tree.c Normal file
View file

@ -0,0 +1,451 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*
* XML file save
*/
#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <ctype.h>
#include <string.h>
#include <syslog.h>
#include <fcntl.h>
#include <assert.h>
#include <arpa/inet.h>
#include <sys/param.h>
#include <netinet/in.h>
/* cligen */
#include <cligen/cligen.h>
/* clicon */
#include "clixon_string.h"
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_err.h"
#include "clixon_log.h"
#include "clixon_handle.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_datastore_tree.h"
#define align4(s) (((s)/4)*4 + 4)
struct indexlist{
qelem_t il_q; /* this must be there */
uint32_t il_i;
int il_len; /* How many */
};
typedef struct indexlist indexlist;
struct indexhead{
indexlist *ih_list;
};
typedef struct indexhead indexhead;
int
index_get(indexhead *ih,
size_t len, /* in bytes */
uint32_t *indexp)
{
int retval = -1;
int nr;
uint32_t index = 0;
struct indexlist *il;
indexlist *new;
nr = len/DF_BLOCK + 1;
if ((new = malloc(sizeof(indexlist))) == NULL){
clicon_err(OE_XML, errno, "malloc");
goto done;
}
memset(new, 0, sizeof(indexlist));
// new->il_i = i; Dont know i yet
new->il_len = nr;
if ((il = ih->ih_list) != NULL) {
il = PREVQ(indexlist*, il);
index = il->il_i + il->il_len;
}
ADDQ(new, ih->ih_list);
new->il_i = index;
*indexp = index;
retval = 0;
done:
return retval;
}
#ifdef NOTYET
int
index_get(indexhead *ih,
size_t len, /* in bytes */
uint32_t *indexp)
{
int retval = -1;
int nr;
uint32_t index = 0;
struct indexlist *il;
indexlist *new;
nr = len/DF_BLOCK + 1;
if ((new = malloc(sizeof(indexlist))) == NULL){
clicon_err(OE_XML, errno, "malloc");
goto done;
}
memset(new, 0, sizeof(indexlist));
// new->il_i = i; Dont know i yet
new->il_len = nr;
/* Find nr conseq blocks */
if ((il = ih->ih_list) != NULL) {
do {
if (il->il_i - index >= nr){ /* Found */
INSQ(new, il);
break;
}
index = il->il_i + il->il_len;
il = NEXTQ(indexlist*, il);
} while (il != ih->ih_list);
ADDQ(new, il); /* after? */
}
else
INSQ(new, ih->ih_list);
new->il_i = index;
*indexp = index;
retval = 0;
done:
return retval;
}
#endif
int
index_dump(FILE *f,
indexhead *ih)
{
indexlist *il;
if ((il = ih->ih_list) != NULL) {
do {
fprintf(f, "%u %d\n", il->il_i, il->il_len);
il = NEXTQ(indexlist*, il);
} while (il != ih->ih_list);
}
return 0;
}
/*
* @param[out] xp
* @param[out] vec If type is CX_ELMNT, points to vector of indexes,...
* @retval -1 Error
* @retval 0 Sanity check failed
* @retval 1 OK xp set (or NULL if empty)
*/
static int
buf2xml(FILE *f,
uint32_t index,
cxobj **xp)
{
int retval = -1;
uint8_t ver;
uint8_t type;
int ptr;
uint32_t len;
cxobj *x = NULL;
char *name;
char *prefix = NULL;
char hdr[8];
char *buf = NULL;
int vlen;
int i;
uint32_t ind;
cxobj *xc;
/* Read hdr
* +-----+-----+-----+-----+-----+-----+-----+-----+
* | ver | type| de | ad | len |
* +-----+-----+-----+-----+-----+-----+-----+-----+
*/
if (fseek(f, index*DF_BLOCK, SEEK_SET) < 0){
clicon_err(OE_UNIX, errno, "fseek");
goto done;
}
if (fread(hdr, sizeof(char), sizeof(hdr), f) < 0){
clicon_err(OE_XML, errno, "fread");
goto done;
}
if ((ver = hdr[0]&0xff) == 0) /* Means empty */
goto ok;
if (ver != 1){
clicon_debug(1, "%s Wrong version: %d", __FUNCTION__, ver);
goto fail;
}
type = hdr[1]&0x0f;
if (type != CX_BODY && type != CX_ELMNT && type != CX_ATTR){
clicon_debug(1, "%s Wrong type: %d", __FUNCTION__, type);
goto fail;
}
if ((hdr[2]&0xff) != 0xde){
clicon_debug(1, "%s Expected 0xde: %x", __FUNCTION__, hdr[2]&0xff);
goto fail;
}
if ((hdr[3]&0xff) != 0xad){
clicon_debug(1, "%s Expected 0xad: %x", __FUNCTION__, hdr[3]&0xff);
goto fail;
}
/* Copy and byte-swap length field */
memcpy(&len, &hdr[4], 4);
len = ntohl(len);
/* Read rest
*/
ptr = 0;
if ((buf = malloc(len - sizeof(hdr))) == NULL){
clicon_err(OE_UNIX, errno, "malloc");
goto done;
}
if (fread(buf, sizeof(char), len-sizeof(hdr), f) < 0){
clicon_err(OE_XML, errno, "fread");
goto done;
}
if ((name = strchr(buf, ':')) != NULL){
*name = '\0';
name++;
prefix = buf;
ptr += strlen(prefix)+1;
}
else
name = buf;
ptr += strlen(name) + 1;
ptr = align4(ptr);
if ((x = xml_new(name, NULL, NULL)) == NULL)
goto done;
if (prefix)
if (xml_prefix_set(x, prefix) < 0)
goto done;
xml_type_set(x, type);
if (xml_type(x) != CX_ELMNT){
if (xml_value_set(x, &buf[ptr]) < 0)
goto done;
ptr += strlen(xml_value(x)) +1;
}
else{
vlen = (len - sizeof(hdr) - ptr)/ sizeof(uint32_t);
if (vlen < 0){
clicon_debug(1, "%s Nr of elements should not be negative: %d", __FUNCTION__, vlen);
goto fail;
}
for (i=0; i<vlen; i++){
memcpy(&ind, &buf[ptr], sizeof(uint32_t));
ptr += sizeof(uint32_t);
if (buf2xml(f, ntohl(ind), &xc) < 0)
goto done;
if (xml_addsub(x, xc) < 0)
goto done;
}
}
assert(len == sizeof(hdr)+ptr);
ok:
if (xp)
*xp = x;
retval = 1;
done:
if (buf)
free(buf);
return retval;
fail:
retval = 0;
goto done;
}
/*! Dump xml object to a datastore block
*/
static int
xml2buf(FILE *f,
cxobj *x,
indexhead *ih,
uint32_t *indexp)
{
int retval = -1;
char hdr[8] = {0,};
int len;
int len1;
int ptr;
int i;
char *buf0;
char *buf;
uint32_t myindex;
hdr[0] = 1;
hdr[1] = 0x80 | xml_type(x);
hdr[2] = 0xde;
hdr[3] = 0xad;
len = sizeof(hdr);
if (xml_prefix(x))
len += strlen(xml_prefix(x)) + 1; /* colon */
len += strlen(xml_name(x)) + 1; /* end-of-string */
len = align4(len);
switch(xml_type(x)){
case CX_ELMNT:
len += xml_child_nr(x)*sizeof(uint32_t);
break;
case CX_BODY:
case CX_ATTR:
len += strlen(xml_value(x))+1;
break;
default:
break;
}
if (index_get(ih, len, &myindex) < 0)
goto done;
len1 = htonl(len); /* swap */
memcpy(&hdr[4], &len1, 4);
/* Write rest
*/
if ((buf0 = malloc(len)) == NULL){
clicon_err(OE_UNIX, errno, "malloc");
goto done;
}
buf = buf0+sizeof(hdr);
ptr = 0;
if (xml_prefix(x)){
strcpy(buf, xml_prefix(x));
ptr += strlen(xml_prefix(x));
buf[ptr++] = ':';
}
strcpy(&buf[ptr], xml_name(x));
ptr += strlen(xml_name(x)) + 1;
for (i=ptr; i< align4(ptr); i++)
buf[i] = 0;
ptr = align4(ptr);
if (xml_type(x) != CX_ELMNT){
strcpy(&buf[ptr], xml_value(x));
ptr += strlen(xml_value(x)) +1;
}
else{
uint32_t index;
for (i=0; i< xml_child_nr(x); i++){
if (xml2buf(f, xml_child_i(x, i), ih, &index) < 0)
goto done;
index = htonl(index);
memcpy(&buf[ptr], &index, sizeof(uint32_t));
ptr += sizeof(uint32_t);
}
}
assert(len == sizeof(hdr)+ptr);
/* Note write both here so we dont get an intermediate seek,... */
if (fseek(f, myindex*DF_BLOCK, SEEK_SET) < 0){
clicon_err(OE_UNIX, errno, "fseek");
goto done;
}
memcpy(buf0, hdr, sizeof(hdr));
if (0 && fwrite(hdr, sizeof(char), sizeof(hdr), f) < 0){
clicon_err(OE_UNIX, errno, "fwrite");
goto done;
}
if (fwrite(buf0, sizeof(char), len, f) < 0){
clicon_err(OE_XML, errno, "fwrite");
goto done;
}
if (indexp)
*indexp = myindex;
retval = 0;
done:
if (buf0)
free(buf0);
return retval;
}
int
datastore_tree_write(clicon_handle h,
char *filename,
cxobj *xt)
{
int retval = -1;
FILE *f = NULL;
indexhead ih = {0,};
indexlist *il;
if ((f = fopen(filename, "w+b")) == NULL){
clicon_err(OE_XML, errno, "Opening file %s", filename);
goto done;
}
if (xml2buf(f, xt, &ih, NULL) < 0)
goto done;
if (debug)
index_dump(stderr, &ih);
retval = 0;
done:
while ((il = ih.ih_list) != NULL)
DELQ(il, ih.ih_list, indexlist*);
if (f != NULL)
fclose(f);
return retval;
}
/*
* @retval -1 Error
* @retval 0 Sanity check failed
* @retval 1 OK xp set (or NULL if empty)
*/
int
datastore_tree_read(clicon_handle h,
char *filename,
cxobj **xt)
{
int retval = -1;
FILE *f = NULL;
if ((f = fopen(filename, "r")) == NULL){
clicon_err(OE_XML, errno, "Opening file %s", filename);
goto done;
}
if ((retval = buf2xml(f, 0, xt)) < 0)
goto done;
if (retval == 0){ /* fail */
if ((*xt = xml_new("config", NULL, NULL)) == NULL)
goto done;
xml_type_set(*xt, CX_ELMNT);
}
done:
if (f != NULL)
fclose(f);
return retval;
}

90
lib/src/clixon_datastore_tree.h Normal file
View file

@ -0,0 +1,90 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*
* Datastore block tree file
*/
#ifndef _CLIXON_DATASTORE_TREE_H_
#define _CLIXON_DATASTORE_TREE_H_
/*
* Constants and macros
*/
/* Max of https://github.com/YangModels/yang
* name: receive-recovery-scccp-avp-bad-value-challenge-response
* namespace: urn:ieee:std:802.3:yang:ieee802-ethernet-interface-half-duplex
*/
#define DF_BLOCK 32
/*
* Types
*/
/* On file value (dep of type)
* off: Offset in bytes to where value starts (name ends +1)
* len: Total length in bytes (network byte order)
* childi: Block index to child (network byte order)
* ELMNT:
* +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
* | 1 | 128 | de | ad | len | name ... | 0 |
* +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
* +-----+-----+-----+-----+-----+-----+-----+------+--
* | child0 (index in file)| child1 (index in file)|
* +-----+-----+-----+-----+-----+-----+-----+-----+--
* ATTR:
* +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
* | 1 | 129 | de | ad | len | name ... | 0 |
* +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+
* +-----+-----+-----+
* | value ... | 0 |
* +-----+-----+-----+
* BODY: (off = 8)
* +-----+-----+-----+-----+-----+-----+-----+-----+-----+
* | 1 | 130 | de | ad | len | 0 |
* +-----+-----+-----+-----+-----+-----+-----+-----+-----+
* +-----+-----+-----+
* | value ... | 0 |
* +-----+-----+-----+
* FREE
* +------+
* | 0 |
* +------+
*/
/*
* Prototypes
*/
int datastore_tree_write(clicon_handle h, char *filename, cxobj *xt);
int datastore_tree_read(clicon_handle h, char *filename, cxobj **xt);
#endif /* _CLIXON_DATASTORE_TREE_H_ */

27
lib/src/clixon_datastore_write.c
View file

@ -80,7 +80,7 @@
#include "clixon_datastore.h"
#include "clixon_datastore_write.h"
#include "clixon_datastore_read.h"
#include "clixon_datastore_tree.h"
/*! Modify a base tree x0 with x1 with yang spec y according to operation op
* @param[in] th Datastore text handle
@ -730,17 +730,26 @@ xmldb_put(clicon_handle h,
if (xml_addsub(x0, xmodst) < 0)
goto done;
}
if ((f = fopen(dbfile, "w")) == NULL){
clicon_err(OE_CFG, errno, "Creating file %s", dbfile);
if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
clicon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
}
format = clicon_option_str(h, "CLICON_XMLDB_FORMAT");
if (format && strcmp(format,"json")==0){
if (xml2json(f, x0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
if (strcmp(format, "tree") == 0){
if (datastore_tree_write(h, dbfile, x0) < 0)
goto done;
}
else if (clicon_xml2file(f, x0, 0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
goto done;
}
else{
if ((f = fopen(dbfile, "w")) == NULL){
clicon_err(OE_CFG, errno, "Creating file %s", dbfile);
goto done;
}
if (strcmp(format,"json")==0){
if (xml2json(f, x0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
goto done;
}
else if (clicon_xml2file(f, x0, 0, clicon_option_bool(h, "CLICON_XMLDB_PRETTY")) < 0)
goto done;
}
/* Remove modules state after writing to file
*/
if (xmodst && xml_purge(xmodst) < 0)

72
test/test_perf.sh
View file

@ -5,7 +5,7 @@
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
# Number of list/leaf-list entries in file
: ${perfnr:=20000}
: ${perfnr:=10000}
# Number of requests made get/put
: ${perfreq:=100}
@ -17,6 +17,8 @@ fyang=$dir/scaling.yang
fconfig=$dir/large.xml
fconfig2=$dir/large2.xml
format=xml
cat <<EOF > $fyang
module scaling{
yang-version 1.1;
@ -50,6 +52,7 @@ cat <<EOF > $cfg
<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
<CLICON_XMLDB_FORMAT>$format</CLICON_XMLDB_FORMAT>
<CLICON_CLI_MODE>example</CLICON_CLI_MODE>
<CLICON_CLI_DIR>/usr/local/lib/example/cli</CLICON_CLI_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
@ -66,27 +69,61 @@ if [ $BE -ne 0 ]; then
err
fi
fi
# Try startup mode w startup
# First generate large XML file
# Use it latter to generate startup-db in xml, tree formats
tmpx=$dir/tmp.xml
new "generate large startup config ($tmpx) with $perfnr entries"
echo -n "<config><x xmlns=\"urn:example:clixon\">" > $tmpx
for (( i=0; i<$perfnr; i++ )); do
echo -n "<y><a>$i</a><b>$i</b></y>" >> $tmpx
done
echo "</x></config>" >> $tmpx
if false; then
# Then generate large JSON file (cant translate namespace - long story)
tmpj=$dir/tmp.json
new "generate large startup config ($tmpj) with $perfnr entries"
echo -n '{"config": {"scaling:x":{"y":[' > $tmpj
for (( i=0; i<$perfnr; i++ )); do
if [ $i -ne 0 ]; then
echo -n ",{\"a\":$i,\"b\":$i}" >> $tmpj
else
echo -n "{\"a\":$i,\"b\":$i}" >> $tmpj
fi
done
echo "]}}}" >> $tmpj
fi
# Loop over mode and format
for mode in startup running; do
file=$dir/${mode}_db
sudo touch $file
sudo chmod 666 $file
new "generate large startup config ($file) with $perfnr list entries in mode $mode"
echo -n "<config><x xmlns=\"urn:example:clixon\">" > $file
for (( i=0; i<$perfnr; i++ )); do
echo -n "<y><a>$i</a><b>$i</b></y>" >> $file
for format in tree xml; do # json - something w namespaces
sudo rm -f $file
sudo touch $file
sudo chmod 666 $file
case $format in
xml)
echo "cp $tmpx $file"
cp $tmpx $file
;;
json)
cp $tmpj $file
;;
tree)
echo "clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create"
clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create
;;
esac
new "Startup backend $format -s $mode -f $cfg -y $fyang"
echo "time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format"
# Cannot use start_backend here due to expected error case
time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format # 2> /dev/null
done
echo "</x></config>" >> $file
new "Startup backend once -s $mode -f $cfg -y $fyang"
# Cannot use start_backend here due to expected error case
time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
done
new "Startup backend once -s $mode -f $cfg -y $fyang"
# Cannot use start_backend here due to expected error case
time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null
new "test params: -f $cfg -y $fyang"
if [ $BE -ne 0 ]; then
new "kill old backend"
@ -152,6 +189,7 @@ time -p for (( i=0; i<$perfreq; i++ )); do
curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null
done
# Reference: i686 perfnr=10000 time: 27s 20190425 34s WITH startup copying
new "restconf add $perfreq small config"
time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) ))

42
util/clixon_util_datastore.c
View file

@ -63,7 +63,7 @@
#include <clixon/clixon.h>
/* Command line options to be passed to getopt(3) */
#define DATASTORE_OPTS "hDd:b:y:"
#define DATASTORE_OPTS "hDd:b:f:x:y:"
/*! usage
*/
@ -76,11 +76,13 @@ usage(char *argv0)
"\t-D\t\tDebug\n"
"\t-d <db>\t\tDatabase name. Default: running. Alt: candidate,startup\n"
"\t-b <dir>\tDatabase directory. Mandatory\n"
"\t-f <fmt>\tDatabase format: xml, json, tree\n"
"\t-x <xml>\tXML file. Alternative to put <xml> argument\n"
"\t-y <file>\tYang file. Mandatory\n"
"and command is either:\n"
"\tget [<xpath>]\n"
"\tmget <nr> [<xpath>]\n"
"\tput (merge|replace|create|delete|remove) <xml>\n"
"\tput (merge|replace|create|delete|remove) [<xml>]\n"
"\tcopy <todb>\n"
"\tlock <pid>\n"
"\tunlock\n"
@ -105,6 +107,7 @@ main(int argc, char **argv)
char *cmd = NULL;
yang_stmt *yspec = NULL;
char *yangfilename = NULL;
char *xmlfilename = NULL;
char *dbdir = NULL;
int ret;
int pid;
@ -122,6 +125,7 @@ main(int argc, char **argv)
if ((h = clicon_handle_init()) == NULL)
goto done;
/* getopt in two steps, first find config-file before over-riding options. */
clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", "xml"); /* default */
while ((c = getopt(argc, argv, DATASTORE_OPTS)) != -1)
switch (c) {
case '?' :
@ -141,6 +145,16 @@ main(int argc, char **argv)
usage(argv0);
dbdir = optarg;
break;
case 'f': /* db format */
if (!optarg)
usage(argv0);
clicon_option_str_set(h, "CLICON_XMLDB_FORMAT", optarg);
break;
case 'x': /* XML file */
if (!optarg)
usage(argv0);
xmlfilename = optarg;
break;
case 'y': /* Yang file */
if (!optarg)
usage(argv0);
@ -213,7 +227,13 @@ main(int argc, char **argv)
fprintf(stdout, "\n");
}
else if (strcmp(cmd, "put")==0){
if (argc != 3){
if (argc == 2){
if (xmlfilename == NULL){
clicon_err(OE_DB, 0, "XML filename expected");
usage(argv0);
}
}
else if (argc != 3){
clicon_err(OE_DB, 0, "Unexpected nr of args: %d", argc);
usage(argv0);
}
@ -221,8 +241,19 @@ main(int argc, char **argv)
clicon_err(OE_DB, 0, "Unrecognized operation: %s", argv[1]);
usage(argv0);
}
if (xml_parse_string(argv[2], NULL, &xt) < 0)
goto done;
if (argc == 2){
int fd;
if ((fd = open(xmlfilename, O_RDONLY)) < 0){
clicon_err(OE_UNIX, errno, "open(%s)", xmlfilename);
goto done;
}
if (xml_parse_file(fd, "</config>", yspec, &xt) < 0)
goto done;
close(fd);
}
else
if (xml_parse_string(argv[2], yspec, &xt) < 0)
goto done;
if (xml_rootchild(xt, 0, &xt) < 0)
goto done;
if ((cbret = cbuf_new()) == NULL){
@ -231,7 +262,6 @@ main(int argc, char **argv)
}
if (xmldb_put(h, db, op, xt, NULL, cbret) < 1)
goto done;
}
else if (strcmp(cmd, "copy")==0){
if (argc != 2)

4
yang/clixon/clixon-config@2019-03-05.yang
View file

@ -90,6 +90,10 @@ module clixon-config {
enum json{
description "Save and load xmldb as JSON";
}
enum tree{
description "Save and load xmldb as Clixon record-based tree
file format";
}
}
}
typedef cli_genmodel_type{