Added CLICON_XMLDB_PRETTY option. If set to false, XML database files will be more compact.
Added CLICON_XMLDB_FORMAT option. Default is "xml". If set to "json", XML database files uses JSON format. Escape " in JSON names and strings and values Optimized search performance for large lists by sorting and binary search.
This commit is contained in:
Olof hagsand
parent
4b92dbdc10
commit
174cfc02c6
15 changed files with 428 additions and 161 deletions
|
|
@ -69,6 +69,12 @@
|
|||
*/
|
||||
#define VEC_ARRAY 1
|
||||
|
||||
/* Size of json read buffer when reading from file*/
|
||||
#define BUFLEN 1024
|
||||
|
||||
/* Name of xml top object created by xml parse functions */
|
||||
#define JSON_TOP_SYMBOL "top"
|
||||
|
||||
enum array_element_type{
|
||||
NO_ARRAY=0,
|
||||
FIRST_ARRAY,
|
||||
|
|
@ -185,32 +191,40 @@ json_escape(char *str)
|
|||
|
||||
j = 0;
|
||||
for (i=0;i<strlen(str);i++)
|
||||
if (str[i]=='\n')
|
||||
switch (str[i]){
|
||||
case '\n':
|
||||
case '\"':
|
||||
case '\\':
|
||||
j++;
|
||||
break;
|
||||
}
|
||||
if ((snew = malloc(strlen(str)+1+j))==NULL){
|
||||
clicon_err(OE_XML, errno, "malloc");
|
||||
return NULL;
|
||||
}
|
||||
j = 0;
|
||||
for (i=0;i<strlen(str);i++)
|
||||
if (str[i]=='\n'){
|
||||
switch (str[i]){
|
||||
case '\n':
|
||||
case '\"':
|
||||
case '\\':
|
||||
snew[j++]='\\';
|
||||
snew[j++]='n';
|
||||
}
|
||||
else
|
||||
default: /* fall thru */
|
||||
snew[j++]=str[i];
|
||||
}
|
||||
snew[j++]='\0';
|
||||
return snew;
|
||||
}
|
||||
|
||||
/*! Do the actual work of translating XML to JSON
|
||||
* @param[out] cb Cligen text buffer containing json on exit
|
||||
* @param[in] x XML tree structure containing XML to translate
|
||||
* @param[out] cb Cligen text buffer containing json on exit
|
||||
* @param[in] x XML tree structure containing XML to translate
|
||||
* @param[in] arraytype Does x occur in a array (of its parent) and how?
|
||||
* @param[in] level Indentation level
|
||||
* @param[in] pretty Pretty-print output (2 means debug)
|
||||
* @param[in] flat Dont print NO_ARRAY object name (for _vec call)
|
||||
*
|
||||
* @note Does not work with XML attributes
|
||||
* The following matrix explains how the mapping is done.
|
||||
* You need to understand what arraytype means (no/first/middle/last)
|
||||
* and what childtype is (null,body,any)
|
||||
|
|
@ -246,10 +260,11 @@ xml2json1_cbuf(cbuf *cb,
|
|||
int pretty,
|
||||
int flat)
|
||||
{
|
||||
int retval = -1;
|
||||
int i;
|
||||
cxobj *xc;
|
||||
int retval = -1;
|
||||
int i;
|
||||
cxobj *xc;
|
||||
enum childtype childt;
|
||||
enum array_element_type xc_arraytype;
|
||||
|
||||
childt = childtype(x);
|
||||
if (pretty==2)
|
||||
|
|
@ -326,7 +341,6 @@ xml2json1_cbuf(cbuf *cb,
|
|||
break;
|
||||
}
|
||||
for (i=0; i<xml_child_nr(x); i++){
|
||||
enum array_element_type xc_arraytype;
|
||||
xc = xml_child_i(x, i);
|
||||
xc_arraytype = array_eval(i?xml_child_i(x,i-1):NULL,
|
||||
xc,
|
||||
|
|
@ -628,6 +642,82 @@ json_parse_str(char *str,
|
|||
return json_parse(str, "", *xt);
|
||||
}
|
||||
|
||||
/*! Read a JSON definition from file and parse it into a parse-tree.
|
||||
*
|
||||
* @param[in] fd A file descriptor containing the JSON file (as ASCII characters)
|
||||
* @param[in] yspec Yang specification, or NULL
|
||||
* @param[in,out] xt Pointer to (XML) parse tree. If empty, create.
|
||||
* @retval 0 OK
|
||||
* @retval -1 Error with clicon_err called
|
||||
*
|
||||
* @code
|
||||
* cxobj *xt = NULL;
|
||||
* if (json_parse_file(0, NULL, &xt) < 0)
|
||||
* err;
|
||||
* xml_free(xt);
|
||||
* @endcode
|
||||
* @note you need to free the xml parse tree after use, using xml_free()
|
||||
* @note, If xt empty, a top-level symbol will be added so that <tree../> will be: <top><tree.../></tree></top>
|
||||
* @note May block on file I/O
|
||||
*/
|
||||
int
|
||||
json_parse_file(int fd,
|
||||
yang_spec *yspec,
|
||||
cxobj **xt)
|
||||
{
|
||||
int retval = -1;
|
||||
int ret;
|
||||
char *jsonbuf = NULL;
|
||||
int jsonbuflen = BUFLEN; /* start size */
|
||||
int oldjsonbuflen;
|
||||
char *ptr;
|
||||
char ch;
|
||||
int len = 0;
|
||||
|
||||
if ((jsonbuf = malloc(jsonbuflen)) == NULL){
|
||||
clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__);
|
||||
goto done;
|
||||
}
|
||||
memset(jsonbuf, 0, jsonbuflen);
|
||||
ptr = jsonbuf;
|
||||
while (1){
|
||||
if ((ret = read(fd, &ch, 1)) < 0){
|
||||
clicon_err(OE_XML, errno, "%s: read: [pid:%d]\n",
|
||||
__FUNCTION__,
|
||||
(int)getpid());
|
||||
break;
|
||||
}
|
||||
if (ret != 0)
|
||||
jsonbuf[len++] = ch;
|
||||
if (ret == 0){
|
||||
if (*xt == NULL)
|
||||
if ((*xt = xml_new(JSON_TOP_SYMBOL, NULL, NULL)) == NULL)
|
||||
goto done;
|
||||
if (len && json_parse(ptr, "", *xt) < 0)
|
||||
goto done;
|
||||
break;
|
||||
}
|
||||
if (len>=jsonbuflen-1){ /* Space: one for the null character */
|
||||
oldjsonbuflen = jsonbuflen;
|
||||
jsonbuflen *= 2;
|
||||
if ((jsonbuf = realloc(jsonbuf, jsonbuflen)) == NULL){
|
||||
clicon_err(OE_XML, errno, "%s: realloc", __FUNCTION__);
|
||||
goto done;
|
||||
}
|
||||
memset(jsonbuf+oldjsonbuflen, 0, jsonbuflen-oldjsonbuflen);
|
||||
ptr = jsonbuf;
|
||||
}
|
||||
}
|
||||
retval = 0;
|
||||
done:
|
||||
if (retval < 0 && *xt){
|
||||
free(*xt);
|
||||
*xt = NULL;
|
||||
}
|
||||
if (jsonbuf)
|
||||
free(jsonbuf);
|
||||
return retval;
|
||||
}
|
||||
|
||||
/*
|
||||
* Turn this on to get a json parse and pretty print test program
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@
|
|||
#include "clixon_handle.h"
|
||||
#include "clixon_yang.h"
|
||||
#include "clixon_xml.h"
|
||||
#include "clixon_xml_sort.h"
|
||||
#include "clixon_xml_parse.h"
|
||||
|
||||
/*
|
||||
|
|
@ -528,6 +529,7 @@ xml_childvec_get(cxobj *x)
|
|||
* @endcode
|
||||
* @note yspec may be NULL either because it is not known or it is irrelevant,
|
||||
* eg for body or attribute
|
||||
* @see xml_sort_insert
|
||||
*/
|
||||
cxobj *
|
||||
xml_new(char *name,
|
||||
|
|
@ -545,7 +547,7 @@ xml_new(char *name,
|
|||
return NULL;
|
||||
|
||||
xml_parent_set(x, xp);
|
||||
if (xp && xml_child_append(xp, x) < 0)
|
||||
if (xp && xml_child_append(xp, x) < 0)
|
||||
return NULL;
|
||||
x->x_spec = spec; /* Can be NULL */
|
||||
return x;
|
||||
|
|
@ -1290,7 +1292,6 @@ xml_parse_file(int fd,
|
|||
|
||||
if (endtag != NULL)
|
||||
endtaglen = strlen(endtag);
|
||||
*xt = NULL;
|
||||
if ((xmlbuf = malloc(xmlbuflen)) == NULL){
|
||||
clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__);
|
||||
goto done;
|
||||
|
|
@ -1558,7 +1559,7 @@ cxvec_append(cxobj *x,
|
|||
* The tree is traversed depth-first, which at least guarantees that a parent is
|
||||
* traversed before a child.
|
||||
* @param[in] xn XML node
|
||||
* @param[in] type matching type or -1 for any
|
||||
* @param[in] type Matching type or -1 for any
|
||||
* @param[in] fn Callback
|
||||
* @param[in] arg Argument
|
||||
* @retval -1 Error, aborted at first error encounter
|
||||
|
|
@ -1610,6 +1611,10 @@ xml_apply(cxobj *xn,
|
|||
}
|
||||
|
||||
/*! Apply a function call on top object and all xml node children recursively
|
||||
* @param[in] xn XML node
|
||||
* @param[in] type Matching type or -1 for any
|
||||
* @param[in] fn Callback
|
||||
* @param[in] arg Argument
|
||||
* @retval -1 Error, aborted at first error encounter
|
||||
* @retval 0 OK, all nodes traversed (subparts may have been skipped)
|
||||
* @retval 1 OK, aborted on first fn returned 1
|
||||
|
|
@ -1836,93 +1841,6 @@ xml_operation2str(enum operation_type op)
|
|||
}
|
||||
}
|
||||
|
||||
/*! Help function to qsort for sorting entries in xml child vector
|
||||
* @param[in] arg1
|
||||
* @param[in] arg2
|
||||
* @retval 0 If equal
|
||||
* @retval <0 if arg1 is less than arg2
|
||||
* @retval >0 if arg1 is greater than arg2
|
||||
* @note must be in clixon_xml.c since it uses internal (hidden) struct xml
|
||||
*/
|
||||
static int
|
||||
xml_cmp(const void* arg1,
|
||||
const void* arg2)
|
||||
{
|
||||
struct xml *x1 = *(struct xml**)arg1;
|
||||
struct xml *x2 = *(struct xml**)arg2;
|
||||
yang_stmt *y1;
|
||||
yang_stmt *y2;
|
||||
int yi1;
|
||||
int yi2;
|
||||
cvec *cvk = NULL; /* vector of index keys */
|
||||
cg_var *cvi;
|
||||
int equal = 0;
|
||||
char *b1;
|
||||
char *b2;
|
||||
char *keyname;
|
||||
|
||||
if (x1 == NULL){
|
||||
if (x2 == NULL)
|
||||
return 0;
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
else if (x2 == NULL)
|
||||
return 1;
|
||||
y1 = xml_spec(x1);
|
||||
y2 = xml_spec(x2);
|
||||
if (y1==NULL || y2==NULL)
|
||||
return 0; /* just ignore */
|
||||
if (y1 != y2){
|
||||
yi1 = yang_order(y1);
|
||||
yi2 = yang_order(y2);
|
||||
if ((equal = yi1-yi2) != 0)
|
||||
return equal;
|
||||
}
|
||||
/* Now y1=y2, same Yang spec, can only be list or leaf-list,
|
||||
* sort according to key
|
||||
*/
|
||||
if (yang_find((yang_node*)y1, Y_ORDERED_BY, "user") != NULL)
|
||||
return 0; /* Ordered by user: maintain existing order */
|
||||
switch (y1->ys_keyword){
|
||||
case Y_LEAF_LIST: /* Match with name and value */
|
||||
equal = strcmp(xml_body(x1), xml_body(x2));
|
||||
break;
|
||||
case Y_LIST: /* Match with key values
|
||||
* Use Y_LIST cache (see struct yang_stmt)
|
||||
*/
|
||||
cvk = y1->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
|
||||
cvi = NULL;
|
||||
while ((cvi = cvec_each(cvk, cvi)) != NULL) {
|
||||
keyname = cv_string_get(cvi);
|
||||
b1 = xml_find_body(x1, keyname);
|
||||
b2 = xml_find_body(x2, keyname);
|
||||
if ((equal = strcmp(b1,b2)) != 0)
|
||||
goto done;
|
||||
}
|
||||
equal = 0;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
done:
|
||||
return equal;
|
||||
}
|
||||
|
||||
/*! Sort children of an XML node
|
||||
* Assume populated by yang spec.
|
||||
* @param[in] x0 XML node
|
||||
* @param[in] arg Dummy so it can be called by xml_apply()
|
||||
* @note must be in clixon_xml.c since it uses internal (hidden) struct xml
|
||||
*/
|
||||
int
|
||||
xml_sort(cxobj *x,
|
||||
void *arg)
|
||||
{
|
||||
qsort(x->x_childvec, x->x_childvec_len, sizeof(struct xml*), xml_cmp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Turn this on to get a xml parse and pretty print test program
|
||||
|
|
|
|||
|
|
@ -229,7 +229,10 @@ xml2cli(FILE *f,
|
|||
term = xml_name(x);
|
||||
if (prepend0)
|
||||
fprintf(f, "%s ", prepend0);
|
||||
fprintf(f, "%s\n", term);
|
||||
if (index(term, ' '))
|
||||
fprintf(f, "\"%s\"\n", term);
|
||||
else
|
||||
fprintf(f, "%s\n", term);
|
||||
retval = 0;
|
||||
goto done;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,6 +100,80 @@ xml_child_spec(char *name,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*! Help function to qsort for sorting entries in xml child vector
|
||||
* @param[in] arg1 - actually cxobj**
|
||||
* @param[in] arg2 - actually cxobj**
|
||||
* @retval 0 If equal
|
||||
* @retval <0 if arg1 is less than arg2
|
||||
* @retval >0 if arg1 is greater than arg2
|
||||
* @note args are pointer ot pointers, to fit into qsort cmp function
|
||||
* @see xml_cmp1 Similar, but for one object
|
||||
*/
|
||||
int
|
||||
xml_cmp(const void* arg1,
|
||||
const void* arg2)
|
||||
{
|
||||
cxobj *x1 = *(struct xml**)arg1;
|
||||
cxobj *x2 = *(struct xml**)arg2;
|
||||
yang_stmt *y1;
|
||||
yang_stmt *y2;
|
||||
int yi1;
|
||||
int yi2;
|
||||
cvec *cvk = NULL; /* vector of index keys */
|
||||
cg_var *cvi;
|
||||
int equal = 0;
|
||||
char *b1;
|
||||
char *b2;
|
||||
char *keyname;
|
||||
|
||||
if (x1 == NULL){
|
||||
if (x2 == NULL)
|
||||
return 0;
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
else if (x2 == NULL)
|
||||
return 1;
|
||||
y1 = xml_spec(x1);
|
||||
y2 = xml_spec(x2);
|
||||
if (y1==NULL || y2==NULL)
|
||||
return 0; /* just ignore */
|
||||
if (y1 != y2){
|
||||
yi1 = yang_order(y1);
|
||||
yi2 = yang_order(y2);
|
||||
if ((equal = yi1-yi2) != 0)
|
||||
return equal;
|
||||
}
|
||||
/* Now y1=y2, same Yang spec, can only be list or leaf-list,
|
||||
* sort according to key
|
||||
*/
|
||||
if (yang_find((yang_node*)y1, Y_ORDERED_BY, "user") != NULL)
|
||||
return 0; /* Ordered by user: maintain existing order */
|
||||
switch (y1->ys_keyword){
|
||||
case Y_LEAF_LIST: /* Match with name and value */
|
||||
equal = strcmp(xml_body(x1), xml_body(x2));
|
||||
break;
|
||||
case Y_LIST: /* Match with key values
|
||||
* Use Y_LIST cache (see struct yang_stmt)
|
||||
*/
|
||||
cvk = y1->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
|
||||
cvi = NULL;
|
||||
while ((cvi = cvec_each(cvk, cvi)) != NULL) {
|
||||
keyname = cv_string_get(cvi);
|
||||
b1 = xml_find_body(x1, keyname);
|
||||
b2 = xml_find_body(x2, keyname);
|
||||
if ((equal = strcmp(b1,b2)) != 0)
|
||||
goto done;
|
||||
}
|
||||
equal = 0;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
done:
|
||||
return equal;
|
||||
}
|
||||
|
||||
/*!
|
||||
* @param[in] yangi Yang order
|
||||
* @param[in] keynr Length of keyvec/keyval vector when applicable
|
||||
|
|
@ -109,6 +183,7 @@ xml_child_spec(char *name,
|
|||
* @retval 0 If equal (or userorder set)
|
||||
* @retval <0 if arg1 is less than arg2
|
||||
* @retval >0 if arg1 is greater than arg2
|
||||
* @see xml_cmp Similar, but for two objects
|
||||
*/
|
||||
static int
|
||||
xml_cmp1(cxobj *x,
|
||||
|
|
@ -155,6 +230,21 @@ xml_cmp1(cxobj *x,
|
|||
return 0; /* should not reach here */
|
||||
}
|
||||
|
||||
/*! Sort children of an XML node
|
||||
* Assume populated by yang spec.
|
||||
* @param[in] x0 XML node
|
||||
* @param[in] arg Dummy so it can be called by xml_apply()
|
||||
*/
|
||||
int
|
||||
xml_sort(cxobj *x,
|
||||
void *arg)
|
||||
{
|
||||
qsort(xml_childvec_get(x), xml_child_nr(x), sizeof(cxobj *), xml_cmp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! Special case search for ordered-by user where linear sort is used
|
||||
*/
|
||||
static cxobj *
|
||||
xml_search_userorder(cxobj *x0,
|
||||
yang_stmt *y,
|
||||
|
|
@ -236,7 +326,7 @@ xml_search1(cxobj *x0,
|
|||
return NULL;
|
||||
}
|
||||
|
||||
/*!
|
||||
/*! Find XML children using binary search
|
||||
* @param[in] yangi yang child order
|
||||
* @param[in] keynr Length of keyvec/keyval vector when applicable
|
||||
* @param[in] keyvec Array of of yang key identifiers
|
||||
|
|
@ -255,6 +345,63 @@ xml_search(cxobj *x0,
|
|||
0, xml_child_nr(x0));
|
||||
}
|
||||
|
||||
static int
|
||||
xml_insert_pos(cxobj *x0,
|
||||
cxobj *x,
|
||||
int low,
|
||||
int upper)
|
||||
{
|
||||
int mid;
|
||||
cxobj *xc;
|
||||
int cmp;
|
||||
int i;
|
||||
|
||||
if (upper < low)
|
||||
return low; /* not found */
|
||||
mid = (low + upper) / 2;
|
||||
if (mid >= xml_child_nr(x0))
|
||||
return xml_child_nr(x0);
|
||||
xc = xml_child_i(x0, mid);
|
||||
cmp = xml_cmp(&x, &xc);
|
||||
if (cmp == 0){
|
||||
/* Special case: append last of equals if ordered by user */
|
||||
for (i=mid+1;i<xml_child_nr(x0);i++){
|
||||
xc = xml_child_i(x0, i);
|
||||
if (xml_cmp(&x, &xc) != 0)
|
||||
break;
|
||||
mid=i; /* still ok */
|
||||
}
|
||||
return mid;
|
||||
}
|
||||
else if (cmp < 0)
|
||||
return xml_insert_pos(x0, x, low, mid-1);
|
||||
else
|
||||
return xml_insert_pos(x0, x, mid+1, upper);
|
||||
}
|
||||
|
||||
/*! Add xml object in sorted position
|
||||
* @param[in] x0 XML parent node.
|
||||
* @param[in] x XML node (to insert)
|
||||
* Assume already under x0
|
||||
* XXX WORK IN PROGRESS
|
||||
*/
|
||||
cxobj *
|
||||
xml_sort_insert(cxobj *x0,
|
||||
cxobj *x)
|
||||
{
|
||||
int pos;
|
||||
/* find closest to x, insert after pos. */
|
||||
xml_rm(x);
|
||||
pos = xml_insert_pos(x0, x, 0, xml_child_nr(x0));
|
||||
fprintf(stderr, "%d\n", pos);
|
||||
#if 0
|
||||
if (pos < xml_child_nr(x0))
|
||||
xml_child_i_set(x0, pos);
|
||||
xml_parent_set(x, x0);
|
||||
#endif
|
||||
return x;
|
||||
}
|
||||
|
||||
/*! Find matching xml child given name and optional key values
|
||||
* container: x0, y->keyword, name
|
||||
* list: x0, y->keyword, y->key, name
|
||||
|
|
@ -350,6 +497,34 @@ xml_match(cxobj *x0,
|
|||
ok:
|
||||
return x;
|
||||
}
|
||||
|
||||
/*! Verify all children of XML node are sorted according to xml_sort()
|
||||
* @param[in] x XML node. Check its children
|
||||
* @param[in] arg Dummy. Ensures xml_apply can be used with this fn
|
||||
@ @retval 0 Sorted
|
||||
@ @retval -1 Not sorted
|
||||
* @see xml_apply
|
||||
*/
|
||||
int
|
||||
xml_sort_verify(cxobj *x0,
|
||||
void *arg)
|
||||
{
|
||||
int retval = -1;
|
||||
cxobj *x = NULL;
|
||||
cxobj *xprev = NULL;
|
||||
|
||||
while ((x = xml_child_each(x0, x, -1)) != NULL) {
|
||||
if (xprev != NULL){ /* Check xprev <= x */
|
||||
if (xml_cmp(&xprev, &x) > 0)
|
||||
goto done;
|
||||
}
|
||||
xprev = x;
|
||||
}
|
||||
retval = 0;
|
||||
done:
|
||||
return retval;
|
||||
}
|
||||
|
||||
/*! Given child tree x1c, find matching child in base tree x0
|
||||
* param[in] x0 Base tree node
|
||||
* param[in] x1c Modification tree child
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue