experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* Optimized validation of large lists

Browse source 
* New xmldb_get1() returning actual cache - not a copy. This has lead to some householding instead of just deleting the copy
  * xml_diff rewritten to work linearly instead of O(2)
  * New xml_insert function using tree search. The new code uses this in insertion xmldb_put and defaults. (Note previous xml_insert renamed to xml_wrap_all)
This commit is contained in:
Olof hagsand 2019-04-14 14:36:41 +02:00
parent 9b9b53c4ee
commit c79baf1b1f
16 changed files with 937 additions and 301 deletions
Download patch file Download diff file Expand all files Collapse all files

5
CHANGELOG.md
View file

@ -92,7 +92,10 @@
``` ```
### Minor changes ### Minor changes
* Optimized validation by making xml_diff work on raw cache tree (not copies) * Optimized validation of large lists
* New xmldb_get1() returning actual cache - not a copy. This has lead to some householding instead of just deleting the copy
* xml_diff rewritten to work linearly instead of O(2)
* New xml_insert function using tree search. The new code uses this in insertion xmldb_put and defaults. (Note previous xml_insert renamed to xml_wrap_all)
* Added syntactic check for yang status: current, deprecated or obsolete. * Added syntactic check for yang status: current, deprecated or obsolete.
* Added `xml_wrap` function that adds an XML node above a node as a wrapper * Added `xml_wrap` function that adds an XML node above a node as a wrapper
* also renamed `xml_insert` to `xml_wrap_all`. * also renamed `xml_insert` to `xml_wrap_all`.

4
include/clixon_custom.h
View file

@ -41,4 +41,6 @@
*/ */
#undef RPC_USERNAME_ASSERT #undef RPC_USERNAME_ASSERT
/* Use new xml_insert code on sorted xml lists
*/
#define USE_XML_INSERT

4
lib/clixon/clixon_xml.h
View file

@ -118,8 +118,9 @@ cxobj *xml_child_i_type(cxobj *xn, int i, enum cxobj_type type);
cxobj *xml_child_i_set(cxobj *xt, int i, cxobj *xc); cxobj *xml_child_i_set(cxobj *xt, int i, cxobj *xc);
cxobj *xml_child_each(cxobj *xparent, cxobj *xprev, enum cxobj_type type); cxobj *xml_child_each(cxobj *xparent, cxobj *xprev, enum cxobj_type type);
cxobj **xml_childvec_get(cxobj *x); int xml_child_insert_pos(cxobj *x, cxobj *xc, int i);
int xml_childvec_set(cxobj *x, int len); int xml_childvec_set(cxobj *x, int len);
cxobj **xml_childvec_get(cxobj *x);
cxobj *xml_new(char *name, cxobj *xn_parent, yang_stmt *spec); cxobj *xml_new(char *name, cxobj *xn_parent, yang_stmt *spec);
yang_stmt *xml_spec(cxobj *x); yang_stmt *xml_spec(cxobj *x);
int xml_spec_set(cxobj *x, yang_stmt *spec); int xml_spec_set(cxobj *x, yang_stmt *spec);
@ -130,7 +131,6 @@ cxobj *xml_find(cxobj *xn_parent, char *name);
int xml_addsub(cxobj *xp, cxobj *xc); int xml_addsub(cxobj *xp, cxobj *xc);
cxobj *xml_wrap_all(cxobj *xp, char *tag); cxobj *xml_wrap_all(cxobj *xp, char *tag);
cxobj *xml_wrap(cxobj *xc, char *tag); cxobj *xml_wrap(cxobj *xc, char *tag);
#define xml_insert(x,t) xml_wrap_all((x),(t))
int xml_purge(cxobj *xc); int xml_purge(cxobj *xc);
int xml_child_rm(cxobj *xp, int i); int xml_child_rm(cxobj *xp, int i);
int xml_rm(cxobj *xc); int xml_rm(cxobj *xc);

2
lib/clixon/clixon_xml_sort.h
View file

@ -40,7 +40,9 @@
* Prototypes * Prototypes
*/ */
int xml_child_spec(cxobj *x, cxobj *xp, yang_stmt *yspec, yang_stmt **yp); int xml_child_spec(cxobj *x, cxobj *xp, yang_stmt *yspec, yang_stmt **yp);
int xml_cmp(cxobj *x1, cxobj *x2, int enm);
int xml_sort(cxobj *x0, void *arg); int xml_sort(cxobj *x0, void *arg);
int xml_insert(cxobj *xp, cxobj *xc);
int xml_sort_verify(cxobj *x, void *arg); int xml_sort_verify(cxobj *x, void *arg);
int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp); int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp);

36
lib/src/clixon_datastore_write.c
View file

@ -129,8 +129,6 @@ text_modify(clicon_handle h,
int ret; int ret;
int changed = 0; /* Only if x0p's children have changed-> sort is necessary */ int changed = 0; /* Only if x0p's children have changed-> sort is necessary */
assert(x1 && xml_type(x1) == CX_ELMNT);
assert(y0);
/* Check for operations embedded in tree according to netconf */ /* Check for operations embedded in tree according to netconf */
if ((opstr = xml_find_value(x1, "operation")) != NULL) if ((opstr = xml_find_value(x1, "operation")) != NULL)
if (xml_operation(opstr, &op) < 0) if (xml_operation(opstr, &op) < 0)
@ -157,8 +155,16 @@ text_modify(clicon_handle h,
permit = 1; permit = 1;
} }
// int iamkey=0; // int iamkey=0;
#ifdef USE_XML_INSERT
/* Add new xml node but without parent - insert when node fully
copied (see changed conditional below) */
if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL)
goto done;
#else
if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
goto done; goto done;
#endif
changed++; changed++;
/* Copy xmlns attributes */ /* Copy xmlns attributes */
@ -204,6 +210,12 @@ text_modify(clicon_handle h,
} }
} }
} }
#ifdef USE_XML_INSERT
if (changed){
if (xml_insert(x0p, x0) < 0)
goto done;
}
#endif
break; break;
case OP_DELETE: case OP_DELETE:
if (x0==NULL){ if (x0==NULL){
@ -282,8 +294,15 @@ text_modify(clicon_handle h,
goto fail; goto fail;
permit = 1; permit = 1;
} }
#ifdef USE_XML_INSERT
/* Add new xml node but without parent - insert when node fully
copied (see changed conditional below) */
if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL)
goto done;
#else
if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL)
goto done; goto done;
#endif
changed++; changed++;
/* Copy xmlns attributes */ /* Copy xmlns attributes */
x1a = NULL; x1a = NULL;
@ -345,6 +364,12 @@ text_modify(clicon_handle h,
if (ret == 0) if (ret == 0)
goto fail; goto fail;
} }
#ifdef USE_XML_INSERT
if (changed){
if (xml_insert(x0p, x0) < 0)
goto done;
}
#endif
break; break;
case OP_DELETE: case OP_DELETE:
if (x0==NULL){ if (x0==NULL){
@ -362,15 +387,16 @@ text_modify(clicon_handle h,
} }
if (xml_purge(x0) < 0) if (xml_purge(x0) < 0)
goto done; goto done;
changed++;
} }
break; break;
default: default:
break; break;
} /* CONTAINER switch op */ } /* CONTAINER switch op */
} /* else Y_CONTAINER */ } /* else Y_CONTAINER */
#ifndef USE_XML_INSERT
if (changed) if (changed)
xml_sort(x0p, NULL); xml_sort(x0p, NULL);
#endif
retval = 1; retval = 1;
done: done:
if (x0vec) if (x0vec)
@ -417,8 +443,8 @@ text_modify_top(clicon_handle h,
int ret; int ret;
/* Assure top-levels are 'config' */ /* Assure top-levels are 'config' */
assert(x0 && strcmp(xml_name(x0),"config")==0); // assert(x0 && strcmp(xml_name(x0),"config")==0);
assert(x1 && strcmp(xml_name(x1),"config")==0); // assert(x1 && strcmp(xml_name(x1),"config")==0);
/* Check for operations embedded in tree according to netconf */ /* Check for operations embedded in tree according to netconf */
if ((opstr = xml_find_value(x1, "operation")) != NULL) if ((opstr = xml_find_value(x1, "operation")) != NULL)

75
lib/src/clixon_json.c
View file

@ -915,79 +915,4 @@ json_parse_file(int fd,
return retval; return retval;
} }
/*
* Turn this on to get a json parse and pretty print test program
* Usage: json
* read json from input
* Example compile:
gcc -g -o json -I. -I../clixon ./clixon_json.c -lclixon -lcligen
* Example run:
echo '{"foo": -23}' | ./json
*/
#if 0 /* Test program */
static int
usage(char *argv0)
{
fprintf(stderr, "usage:%s.\n\tInput on stdin\n", argv0);
exit(0);
}
int
main(int argc,
char **argv)
{
cxobj *xt;
cxobj *xc;
cbuf *cb = cbuf_new();
char *buf = NULL;
int i;
int c;
int len;
FILE *f = stdin;
if (argc != 1){
usage(argv[0]);
return 0;
}
clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR);
len = 1024; /* any number is fine */
if ((buf = malloc(len)) == NULL){
perror("malloc");
return -1;
}
memset(buf, 0, len);
i = 0; /* position in buf */
while (1){ /* read the whole file */
if ((c = fgetc(f)) == EOF)
break;
if (len==i){
if ((buf = realloc(buf, 2*len)) == NULL){
fprintf(stderr, "%s: realloc: %s\n", __FUNCTION__, strerror(errno));
goto done;
}
memset(buf+len, 0, len);
len *= 2;
}
buf[i++] = (char)(c&0xff);
} /* read a line */
if (json_parse_str(buf, &xt) < 0)
return -1;
xc = NULL;
while ((xc = xml_child_each(xt, xc, -1)) != NULL) {
xmltree2cbuf(cb, xc, 0); /* dump data structures */
//clicon_xml2cbuf(cb, xc, 0, 1); /* print xml */
}
fprintf(stdout, "%s", cbuf_get(cb));
if (xt)
xml_free(xt);
if (cb)
cbuf_free(cb);
done:
return 0;
}
#endif /* Test program */

42
lib/src/clixon_xml.c
View file

@ -562,6 +562,7 @@ xml_child_nr_type(cxobj *xn,
* @param[in] i the number of the child, eg order in children vector * @param[in] i the number of the child, eg order in children vector
* @retval xml The child xml node * @retval xml The child xml node
* @retval NULL if no such child, or empty child * @retval NULL if no such child, or empty child
* @see xml_child_i_type
*/ */
cxobj * cxobj *
xml_child_i(cxobj *xn, xml_child_i(cxobj *xn,
@ -652,7 +653,7 @@ xml_child_each(cxobj *xparent,
} }
/*! Extend child vector with one and insert xml node there /*! Extend child vector with one and insert xml node there
* Note: does not do anything with child, you may need to set its parent, etc * @note does not do anything with child, you may need to set its parent, etc
*/ */
static int static int
xml_child_append(cxobj *x, xml_child_append(cxobj *x,
@ -668,7 +669,31 @@ xml_child_append(cxobj *x,
return 0; return 0;
} }
/*! Set a a childvec to a specific size, fill with children after /*! Insert child xc at position i under parent xp
*
* @see xml_child_append
* @note does not do anything with child, you may need to set its parent, etc
*/
int
xml_child_insert_pos(cxobj *xp,
cxobj *xc,
int i)
{
size_t size;
xp->x_childvec_len++;
xp->x_childvec = realloc(xp->x_childvec, xp->x_childvec_len*sizeof(cxobj*));
if (xp->x_childvec == NULL){
clicon_err(OE_XML, errno, "realloc");
return -1;
}
size = (xml_child_nr(xp) - i - 1)*sizeof(cxobj *);
memmove(&xp->x_childvec[i+1], &xp->x_childvec[i], size);
xp->x_childvec[i] = xc;
return 0;
}
/*! Set a childvec to a specific size, fill with children after
* @code * @code
* xml_childvec_set(x, 2); * xml_childvec_set(x, 2);
* xml_child_i_set(x, 0, xc0) * xml_child_i_set(x, 0, xc0)
@ -731,6 +756,7 @@ xml_new(char *name,
xml_parent_set(x, xp); xml_parent_set(x, xp);
if (xml_child_append(xp, x) < 0) if (xml_child_append(xp, x) < 0)
return NULL; return NULL;
x->_x_i = xml_child_nr(xp)-1;
} }
x->x_spec = yspec; /* Can be NULL */ x->x_spec = yspec; /* Can be NULL */
return x; return x;
@ -966,15 +992,14 @@ xml_child_rm(cxobj *xp,
int int
xml_rm(cxobj *xc) xml_rm(cxobj *xc)
{ {
int retval = 0; int retval = -1;
cxobj *xp; cxobj *xp;
cxobj *x; cxobj *x;
int i; int i;
if ((xp = xml_parent(xc)) == NULL) if ((xp = xml_parent(xc)) == NULL)
goto done; goto ok;
retval = -1; /* Find child in parent XXX: search? */
/* Find child in parent */
x = NULL; i = 0; x = NULL; i = 0;
while ((x = xml_child_each(xp, x, -1)) != NULL) { while ((x = xml_child_each(xp, x, -1)) != NULL) {
if (x == xc) if (x == xc)
@ -982,7 +1007,10 @@ xml_rm(cxobj *xc)
i++; i++;
} }
if (x != NULL) if (x != NULL)
retval = xml_child_rm(xp, i); if (xml_child_rm(xp, i) < 0)
goto done;
ok:
retval = 0;
done: done:
return retval; return retval;
} }

96
lib/src/clixon_xml_map.c
View file

@ -1060,6 +1060,13 @@ cvec2xml_1(cvec *cvv,
* @param[out] changed_x0 Pointervector to XML nodes changed orig value * @param[out] changed_x0 Pointervector to XML nodes changed orig value
* @param[out] changed_x1 Pointervector to XML nodes changed wanted value * @param[out] changed_x1 Pointervector to XML nodes changed wanted value
* @param[out] changedlen Length of changed vector * @param[out] changedlen Length of changed vector
* Algorithm to compare two sorted lists A, B:
* A 0 1 2 3 5 6
* B 0 2 4 5 6
* Let a,b be first elements of A,B respectively
* a = b : recurse; get next a,b
* a < b : add a in x0, get next a
* a > b : add b in x1, get next b
*/ */
static int static int
xml_diff1(yang_stmt *ys, xml_diff1(yang_stmt *ys,
@ -1079,26 +1086,45 @@ xml_diff1(yang_stmt *ys,
yang_stmt *yc; yang_stmt *yc;
char *b1; char *b1;
char *b2; char *b2;
int eq;
clicon_debug(2, "%s: %s", __FUNCTION__, ys->ys_argument?ys->ys_argument:"yspec"); /* Traverse x0 and x1 in lock-step */
/* Check nodes present in x0 and x1 + nodes only in x0 x0c = x1c = NULL;
* Loop over x0 x0c = xml_child_each(x0, x0c, CX_ELMNT);
* XXX: room for improvement. Compare with match_base_child() x1c = xml_child_each(x1, x1c, CX_ELMNT);
*/ for (;;){
x0c = NULL; if (x0c == NULL && x1c == NULL)
while ((x0c = xml_child_each(x0, x0c, CX_ELMNT)) != NULL){ goto ok;
else if (x0c == NULL){
if (cxvec_append(x1c, x1vec, x1veclen) < 0)
goto done;
x1c = xml_child_each(x1, x1c, CX_ELMNT);
continue;
}
else if (x1c == NULL){
if (cxvec_append(x0c, x0vec, x0veclen) < 0)
goto done;
x0c = xml_child_each(x0, x0c, CX_ELMNT);
continue;
}
/* Both x0c and x1c exists, check if they are equal. */
eq = xml_cmp(x0c, x1c, 0);
if (eq < 0){
if (cxvec_append(x0c, x0vec, x0veclen) < 0)
goto done;
x0c = xml_child_each(x0, x0c, CX_ELMNT);
}
else if (eq > 0){
if (cxvec_append(x1c, x1vec, x1veclen) < 0)
goto done;
x1c = xml_child_each(x1, x1c, CX_ELMNT);
}
else{ /* equal */
if ((yc = xml_spec(x0c)) == NULL){ if ((yc = xml_spec(x0c)) == NULL){
clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x0c)); clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x0c));
goto done; goto done;
} }
/* Does x1 have a child matching x0c? */ if (yang_choice(yc)){
if (match_base_child(x1, x0c, yc, &x1c) < 0)
goto done;
if (x1c == NULL){
if (cxvec_append(x0c, x0vec, x0veclen) < 0)
goto done;
}
else if (yang_choice(yc)){
/* if x0c and x1c are choice/case, then they are changed */ /* if x0c and x1c are choice/case, then they are changed */
if (cxvec_append(x0c, changed_x0, changedlen) < 0) if (cxvec_append(x0c, changed_x0, changedlen) < 0)
goto done; goto done;
@ -1106,8 +1132,8 @@ xml_diff1(yang_stmt *ys,
if (cxvec_append(x1c, changed_x1, changedlen) < 0) if (cxvec_append(x1c, changed_x1, changedlen) < 0)
goto done; goto done;
} }
else{ /* if x0c and x1c are leafs w bodies, then they are changed */ else if (yc->ys_keyword == Y_LEAF){
if (yc->ys_keyword == Y_LEAF){ /* if x0c and x1c are leafs w bodies, then they are changed */
if ((b1 = xml_body(x0c)) == NULL) /* empty type */ if ((b1 = xml_body(x0c)) == NULL) /* empty type */
break; break;
if ((b2 = xml_body(x1c)) == NULL) /* empty type */ if ((b2 = xml_body(x1c)) == NULL) /* empty type */
@ -1120,29 +1146,16 @@ xml_diff1(yang_stmt *ys,
goto done; goto done;
} }
} }
if (xml_diff1(yc, x0c, x1c, else if (xml_diff1(yc, x0c, x1c,
x0vec, x0veclen, x0vec, x0veclen,
x1vec, x1veclen, x1vec, x1veclen,
changed_x0, changed_x1, changedlen)< 0) changed_x0, changed_x1, changedlen)< 0)
goto done; goto done;
} }
} /* while x0 */ x0c = xml_child_each(x0, x0c, CX_ELMNT);
/* Check nodes present only in x1 x1c = xml_child_each(x1, x1c, CX_ELMNT);
* Loop over x1
*/
x1c = NULL;
while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL){
if ((yc = xml_spec(x1c)) == NULL){
clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x1c));
goto done;
} }
/* Does x0 have a child matching x1c? */ ok:
if (match_base_child(x0, x1c, yc, &x0c) < 0)
goto done;
if (x0c == NULL)
if (cxvec_append(x1c, x1vec, x1veclen) < 0)
goto done;
} /* while x0 */
retval = 0; retval = 0;
done: done:
return retval; return retval;
@ -1622,6 +1635,7 @@ xml_default(cxobj *xt,
cxobj *xc; cxobj *xc;
cxobj *xb; cxobj *xb;
char *str; char *str;
int added=0;
if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){ if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){
retval = 0; retval = 0;
@ -1637,8 +1651,14 @@ xml_default(cxobj *xt,
assert(y->ys_cv); assert(y->ys_cv);
if (!cv_flag(y->ys_cv, V_UNSET)){ /* Default value exists */ if (!cv_flag(y->ys_cv, V_UNSET)){ /* Default value exists */
if (!xml_find(xt, y->ys_argument)){ if (!xml_find(xt, y->ys_argument)){
#ifdef USE_XML_INSERT
if ((xc = xml_new(y->ys_argument, NULL, y)) == NULL)
goto done;
#else
if ((xc = xml_new(y->ys_argument, xt, y)) == NULL) if ((xc = xml_new(y->ys_argument, xt, y)) == NULL)
goto done; goto done;
#endif
xml_flag_set(xc, XML_FLAG_DEFAULT); xml_flag_set(xc, XML_FLAG_DEFAULT);
if ((xb = xml_new("body", xc, NULL)) == NULL) if ((xb = xml_new("body", xc, NULL)) == NULL)
goto done; goto done;
@ -1650,11 +1670,19 @@ xml_default(cxobj *xt,
if (xml_value_set(xb, str) < 0) if (xml_value_set(xb, str) < 0)
goto done; goto done;
free(str); free(str);
added++;
#ifdef USE_XML_INSERT
if (xml_insert(xt, xc) < 0)
goto done;
#endif
} }
} }
} }
} }
#ifndef USE_XML_INSERT
if (added)
xml_sort(xt, NULL); xml_sort(xt, NULL);
#endif
retval = 0; retval = 0;
done: done:
return retval; return retval;

203
lib/src/clixon_xml_sort.c
View file

@ -188,19 +188,22 @@ xml_child_spec(cxobj *x,
} }
/*! Help function to qsort for sorting entries in xml child vector same parent /*! Help function to qsort for sorting entries in xml child vector same parent
* @param[in] xml object 1 * @param[in] x1 object 1
* @param[in] xml object 2 * @param[in] x2 object 2
* @param[in] same If set, x1 and x2 are member of same parent & enumeration
* is used
* @retval 0 If equal * @retval 0 If equal
* @retval <0 if x1 is less than x2 * @retval <0 If x1 is less than x2
* @retval >0 if x1 is greater than x2 * @retval >0 If x1 is greater than x2
* @see xml_cmp1 Similar, but for one object * @see xml_cmp1 Similar, but for one object
* @note empty value/NULL is smallest value * @note empty value/NULL is smallest value
* @note xml_enumerate_children must have been called prior to this call * @note xml_enumerate_children must have been called prior to this call
* @note some error cases return as -1 (qsort cant handle errors) * @note some error cases return as -1 (qsort cant handle errors)
*/ */
static int int
xml_cmp(cxobj *x1, xml_cmp(cxobj *x1,
cxobj *x2) cxobj *x2,
int same)
{ {
yang_stmt *y1; yang_stmt *y1;
yang_stmt *y2; yang_stmt *y2;
@ -218,17 +221,17 @@ xml_cmp(cxobj *x1,
int nr2 = 0; int nr2 = 0;
cxobj *x1b; cxobj *x1b;
cxobj *x2b; cxobj *x2b;
int e;
e=0;
if (x1==NULL || x2==NULL) if (x1==NULL || x2==NULL)
goto done; /* shouldnt happen */ goto done; /* shouldnt happen */
e=1;
y1 = xml_spec(x1); y1 = xml_spec(x1);
y2 = xml_spec(x2); y2 = xml_spec(x2);
if (same){
nr1 = xml_enumerate_get(x1); nr1 = xml_enumerate_get(x1);
nr2 = xml_enumerate_get(x2); nr2 = xml_enumerate_get(x2);
}
if (y1==NULL && y2==NULL){ if (y1==NULL && y2==NULL){
if (same)
equal = nr1-nr2; equal = nr1-nr2;
goto done; goto done;
} }
@ -240,24 +243,22 @@ xml_cmp(cxobj *x1,
equal = 1; equal = 1;
goto done; goto done;
} }
e=2;
if (y1 != y2){ if (y1 != y2){
yi1 = yang_order(y1); yi1 = yang_order(y1);
yi2 = yang_order(y2); yi2 = yang_order(y2);
if ((equal = yi1-yi2) != 0) if ((equal = yi1-yi2) != 0)
goto done; goto done;
} }
e=3;
/* Now y1==y2, same Yang spec, can only be list or leaf-list, /* Now y1==y2, same Yang spec, can only be list or leaf-list,
* But first check exceptions, eg config false or ordered-by user * But first check exceptions, eg config false or ordered-by user
* otherwise sort according to key * otherwise sort according to key
*/ */
if (yang_config(y1)==0 || if (yang_config(y1)==0 ||
yang_find(y1, Y_ORDERED_BY, "user") != NULL){ yang_find(y1, Y_ORDERED_BY, "user") != NULL){
if (same)
equal = nr1-nr2; equal = nr1-nr2;
goto done; /* Ordered by user or state data : maintain existing order */ goto done; /* Ordered by user or state data : maintain existing order */
} }
e=4;
switch (y1->ys_keyword){ switch (y1->ys_keyword){
case Y_LEAF_LIST: /* Match with name and value */ case Y_LEAF_LIST: /* Match with name and value */
if ((b1 = xml_body(x1)) == NULL) if ((b1 = xml_body(x1)) == NULL)
@ -299,9 +300,8 @@ xml_cmp(cxobj *x1,
default: default:
break; break;
} }
e=5;
done: done:
clicon_debug(2, "%s %s %s %d %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, e, nr1, nr2, yi1, yi2); clicon_debug(2, "%s %s %s %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, nr1, nr2, yi1, yi2);
return equal; return equal;
} }
@ -312,7 +312,7 @@ static int
xml_cmp_qsort(const void* arg1, xml_cmp_qsort(const void* arg1,
const void* arg2) const void* arg2)
{ {
return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2); return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1);
} }
/*! Compare xml object /*! Compare xml object
@ -531,76 +531,151 @@ xml_search(cxobj *x0,
{ {
cxobj *xa; cxobj *xa;
int low = 0; int low = 0;
int high = xml_child_nr(x0); int upper = xml_child_nr(x0);
/* Assume if there are any attributes, they are first in the list, mask /* Assume if there are any attributes, they are first in the list, mask
them by raising low to skip them */ them by raising low to skip them */
for (low=0; low<high; low++) for (low=0; low<upper; low++)
if ((xa = xml_child_i(x0, low)) == NULL || xml_type(xa)!=CX_ATTR) if ((xa = xml_child_i(x0, low)) == NULL || xml_type(xa)!=CX_ATTR)
break; break;
return xml_search1(x0, name, yangi, keyword, keynr, keyvec, keyval, keycvec, return xml_search1(x0, name, yangi, keyword, keynr, keyvec, keyval, keycvec,
low, high); low, upper);
} }
#ifdef NOTUSED /*! Insert xn in xp:s sorted child list
/*! Position where to insert xml object into a list of children nodes * Find a point in xp childvec with two adjacent nodes xi,xi+1 such that
* @note EXPERIMENTAL * xi<=xn<=xi+1 or xn<=x0 or xmax<=xn
* Insert after position returned
* @param[in] x0 XML parent node.
* @param[in] low Lower bound
* @param[in] upper Upper bound (+1)
* @retval position
* XXX: Problem with this is that evrything must be known before insertion
*/ */
int static int
xml_insert_pos(cxobj *x0, xml_insert2(cxobj *xp,
char *name, cxobj *xn,
int yangi, yang_stmt *yn,
enum rfc_6020 keyword, int yni,
int keynr, int userorder,
char **keyvec,
char **keyval,
int low, int low,
int upper) int upper)
{ {
int retval = -1;
int mid; int mid;
cxobj *xc;
yang_stmt *y;
int cmp; int cmp;
cxobj *xc;
yang_stmt *yc;
int i; int i;
int userorder= 0;
if (upper < low) if (low > upper){ /* beyond range */
return low; /* not found */ clicon_err(OE_XML, 0, "low>upper %d %d", low, upper);
goto done;
}
if (low == upper){
retval = low;
goto done;
}
mid = (low + upper) / 2; mid = (low + upper) / 2;
if (mid >= xml_child_nr(x0)) if (mid >= xml_child_nr(xp)){ /* beyond range */
return xml_child_nr(x0); /* upper range */ clicon_err(OE_XML, 0, "Beyond range %d %d %d", low, mid, upper);
xc = xml_child_i(x0, mid); goto done;
y = xml_spec(xc); }
cmp = yangi-yang_order(y); xc = xml_child_i(xp, mid);
if ((yc = xml_spec(xc)) == NULL){
clicon_err(OE_XML, 0, "No spec found %s", xml_name(xc));
goto done;
}
if (yc == yn){ /* Same yang */
if (userorder){ /* append: increment linearly until no longer equal */
for (i=mid+1; i<xml_child_nr(xp); i++){ /* First increment */
xc = xml_child_i(xp, i);
yc = xml_spec(xc);
if (yc != yn){
retval = i;
goto done;
}
}
retval = i;
goto done;
}
else /* Ordered by system */
cmp = xml_cmp(xn, xc, 0);
}
else{ /* Not equal yang - compute diff */
cmp = yni - yang_order(yc);
/* One case is a choice where
* xc = <tcp/>, xn = <udp/>
* same order but different yang spec
*/
}
if (low +1 == upper){ /* termination criterium */
#if 0
if (xml_child_nr(xp) <= mid+1){
retval = mid;
goto done;
}
#endif
if (cmp<0) {
retval = mid;
goto done;
}
retval = mid+1;
goto done;
}
if (cmp == 0){ if (cmp == 0){
cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, &userorder); retval = mid;
if (userorder){ /* Look inside this yangi order */ goto done;
/* Special case: append last of equals if ordered by user */
for (i=mid+1;i<xml_child_nr(x0);i++){
xc = xml_child_i(x0, i);
if (strcmp(xml_name(xc), name))
break;
mid=i; /* still ok */
} }
return mid;
}
}
if (cmp == 0)
return mid;
else if (cmp < 0) else if (cmp < 0)
return xml_insert_pos(x0, name, yangi, keyword, return xml_insert2(xp, xn, yn, yni, userorder, low, mid);
keynr, keyvec, keyval, keycvec, low, mid-1);
else else
return xml_insert_pos(x0, name, yangi, keyword, return xml_insert2(xp, xn, yn, yni, userorder, mid+1, upper);
keynr, keyvec, keyval, keycvec, mid+1, upper); done:
return retval;
}
/*! Insert xc as child to xp in sorted place. Remove xc from previous parent.
* @param[in] xp Parent xml node. If NULL just remove from old parent.
* @param[in] x Child xml node to insert under xp
* @retval 0 OK
* @retval -1 Error
* @see xml_addsub where xc is appended. xml_insert is xml_addsub();xml_sort()
*/
int
xml_insert(cxobj *xp,
cxobj *xi)
{
int retval = -1;
cxobj *xa;
int low = 0;
int upper;
yang_stmt *y;
int userorder= 0;
int yi; /* Global yang-stmt order */
int i;
/* Ensure the intermediate state that xp is parent of x but has not yet been
* added as a child
*/
// assert(xml_parent(xi) == NULL);
// assert(y = xml_spec(xi));
upper = xml_child_nr(xp);
/* Assume if there are any attributes, they are first in the list, mask
them by raising low to skip them */
for (low=0; low<upper; low++)
if ((xa = xml_child_i(xp, low)) == NULL || xml_type(xa)!=CX_ATTR)
break;
/* Find if non-config and if ordered-by-user */
if (yang_config(y)==0)
userorder = 1;
else if (y->ys_keyword == Y_LIST || y->ys_keyword == Y_LEAF_LIST)
userorder = (yang_find(y, Y_ORDERED_BY, "user") != NULL);
yi = yang_order(y);
if ((i = xml_insert2(xp, xi, y, yi, userorder, low, upper)) < 0)
goto done;
if (xml_child_insert_pos(xp, xi, i) < 0)
goto done;
xml_parent_set(xi, xp);
retval = 0;
done:
return retval;
} }
#endif /* NOTUSED */
/*! Verify all children of XML node are sorted according to xml_sort() /*! Verify all children of XML node are sorted according to xml_sort()
* @param[in] x XML node. Check its children * @param[in] x XML node. Check its children
@ -626,7 +701,7 @@ xml_sort_verify(cxobj *x0,
xml_enumerate_children(x0); xml_enumerate_children(x0);
while ((x = xml_child_each(x0, x, -1)) != NULL) { while ((x = xml_child_each(x0, x, -1)) != NULL) {
if (xprev != NULL){ /* Check xprev <= x */ if (xprev != NULL){ /* Check xprev <= x */
if (xml_cmp(xprev, x) > 0) if (xml_cmp(xprev, x, 1) > 0)
goto done; goto done;
} }
xprev = x; xprev = x;

38
lib/src/clixon_yang.c
View file

@ -660,6 +660,30 @@ yang_choice(yang_stmt *y)
return NULL; return NULL;
} }
static int
order1_choice(yang_stmt *yp,
yang_stmt *y)
{
yang_stmt *ys;
yang_stmt *yc;
int i;
int j;
for (i=0; i<yp->ys_len; i++){
ys = yp->ys_stmt[i];
if (ys->ys_keyword == Y_CASE){
for (j=0; j<ys->ys_len; j++){
yc = ys->ys_stmt[j];
if (yang_datanode(yc) && yc == y)
return 1;
}
}
else if (yang_datanode(ys) && ys == y)
return 1;
}
return 0;
}
/*! Find matching y in yp:s children, return 0 and index or -1 if not found. /*! Find matching y in yp:s children, return 0 and index or -1 if not found.
* @param[in] yp Parent * @param[in] yp Parent
* @param[in] y Yang datanode to find * @param[in] y Yang datanode to find
@ -677,10 +701,16 @@ order1(yang_stmt *yp,
for (i=0; i<yp->ys_len; i++){ for (i=0; i<yp->ys_len; i++){
ys = yp->ys_stmt[i]; ys = yp->ys_stmt[i];
if (ys->ys_keyword == Y_CHOICE){
if (order1_choice(ys, y) == 1) /* If one of the choices is "y" */
return 1;
}
else {
if (!yang_datanode(ys)) if (!yang_datanode(ys))
continue; continue;
if (ys==y) if (ys==y)
return 1; return 1;
}
(*index)++; (*index)++;
} }
return 0; return 0;
@ -703,7 +733,14 @@ yang_order(yang_stmt *y)
int j=0; int j=0;
int tot = 0; int tot = 0;
/* Some special handling if yp is choice (or case) and maybe union?
* if so, the real parent (from an xml point of view) is the parents
* parent.
*/
yp = y->ys_parent; yp = y->ys_parent;
while (yp->ys_keyword == Y_CASE || yp->ys_keyword == Y_CHOICE)
yp = yp->ys_parent;
/* XML nodes with yang specs that are children of modules are special - /* XML nodes with yang specs that are children of modules are special -
* In clixon, they are seen as an "implicit" container where the XML can come from different * In clixon, they are seen as an "implicit" container where the XML can come from different
* modules. The order must therefore be global among yang top-symbols to be unique. * modules. The order must therefore be global among yang top-symbols to be unique.
@ -1890,6 +1927,7 @@ yang_parse_str(char *str,
* @param[in] ysp Yang specification. Should have been created by caller using yspec_new * @param[in] ysp Yang specification. Should have been created by caller using yspec_new
* @retval ymod Top-level yang (sub)module * @retval ymod Top-level yang (sub)module
* @retval NULL Error * @retval NULL Error
* @note this function simply parse a yang spec, no dependencies or checks
*/ */
yang_stmt * yang_stmt *
yang_parse_file(int fd, yang_parse_file(int fd,

4
test/README.md
View file

@ -60,6 +60,10 @@ The `mem.sh` runs memory checks using valgrind. Start it with no arguments to te
mem.sh restconf backend # Only backend and cli mem.sh restconf backend # Only backend and cli
``` ```
## Performance plots
The script `plot_perf.sh` produces gnuplots for some testcases.
## Site.sh ## Site.sh
You may add your site-specific modifications in a `site.sh` file. Example: You may add your site-specific modifications in a `site.sh` file. Example:
``` ```

168
test/plot_perf.sh
View file

@ -1,10 +1,33 @@
#!/bin/bash #!/bin/bash
# Transactions per second for large lists read/write plotter using gnuplot # Transactions per second for large lists read/write plotter using gnuplot
# WORK IN PROGRESS # What do I want to plot?
. ./lib.sh # 1. How long to write 100K entries?
max=2000 # Nr of db entries # - netconf / restconf
step=200 # - list / leaf-list
reqs=500 # 2. How long to read 100K entries?
# - netconf/ restconf
# - list / leaf-list
# 3. How long to commit 100K entries? (netconf)
# - list / leaf-list
# 4. In database 100K entries. How many read operations per second?
# - netconf/ restconf
# - list / leaf-list
# 5. 100K entries. How many write operations per second?
# - netconf / restconf
# - list / leaf-list
# 6. 100K entries. How may delete operations per second?
# - netconf / restconf
# - list / leaf-list
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
#max=2000 # Nr of db entries
#step=200
#reqs=500
# Global variables
APPNAME=example
cfg=$dir/plot-conf.xml cfg=$dir/plot-conf.xml
fyang=$dir/plot.yang fyang=$dir/plot.yang
fconfig=$dir/config fconfig=$dir/config
@ -15,7 +38,10 @@ fconfig=$dir/config
clixon_netconf=clixon_netconf clixon_netconf=clixon_netconf
cat <<EOF > $fyang cat <<EOF > $fyang
module ietf-ip{ module scaling{
yang-version 1.1;
namespace "urn:example:clixon";
prefix sc;
container x { container x {
list y { list y {
key "a"; key "a";
@ -34,23 +60,32 @@ module ietf-ip{
EOF EOF
cat <<EOF > $cfg cat <<EOF > $cfg
<config> <clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE> <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_YANG_DIR>$fyang</CLICON_YANG_DIR> <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
<CLICON_YANG_MODULE_MAIN>ietf-ip</CLICON_YANG_MODULE_MAIN> <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
<CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK> <CLICON_YANG_MODULE_MAIN>scaling</CLICON_YANG_MODULE_MAIN>
<CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE> <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY> <CLICON_BACKEND_PIDFILE>/usr/local/var/example/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR> <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
</config> <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
</clixon-config>
EOF EOF
run(){ # Run function
# args: <i> <reqs> <mode>
# where mode is one of:
# readlist writelist restreadlist restwritelist
runfn(){
nr=$1 # Number of entries in DB nr=$1 # Number of entries in DB
reqs=$2 reqs=$2
mode=$3 operation=$3
echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x>" > $fconfig # echo "runfn nr=$nr reqs=$reqs mode=$mode"
# new "generate config with $nr list entries"
echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fconfig
for (( i=0; i<$nr; i++ )); do for (( i=0; i<$nr; i++ )); do
case $mode in case $mode in
readlist|writelist|restreadlist|restwritelist) readlist|writelist|restreadlist|restwritelist)
@ -64,100 +99,135 @@ run(){
echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
# new "netconf write $nr entry to backend"
expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
case $mode in case $mode in
readlist) readlist)
# new "netconf GET list $reqs"
time -p for (( i=0; i<$reqs; i++ )); do time -p for (( i=0; i<$reqs; i++ )); do
rnd=$(( ( RANDOM % $nr ) )) rnd=$(( ( RANDOM % $nr ) ))
echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>" echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
done | $clixon_netconf -qf $cfg -y $fyang > /dev/null done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
;; ;;
writelist) writelist)
# new "netconf WRITE list $reqs"
time -p for (( i=0; i<$reqs; i++ )); do time -p for (( i=0; i<$reqs; i++ )); do
rnd=$(( ( RANDOM % $nr ) )) rnd=$(( ( RANDOM % $nr ) ))
echo "<rpc><edit-config><target><candidate/></target><config><x><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>" echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
done | $clixon_netconf -qf $cfg -y $fyang > /dev/null done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
;; ;;
restreadlist) restreadlist)
# new "restconf GET list $reqs"
time -p for (( i=0; i<$reqs; i++ )); do time -p for (( i=0; i<$reqs; i++ )); do
rnd=$(( ( RANDOM % $nr ) )) rnd=$(( ( RANDOM % $nr ) ))
curl -sSG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null curl -sSG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null
done done
;; ;;
writeleaflist) writeleaflist)
# new "netconf GET leaf-list $reqs"
time -p for (( i=0; i<$reqs; i++ )); do time -p for (( i=0; i<$reqs; i++ )); do
rnd=$(( ( RANDOM % $nr ) )) rnd=$(( ( RANDOM % $nr ) ))
echo "<rpc><edit-config><target><candidate/></target><config><x><c>$rnd</c></x></config></edit-config></rpc>]]>]]>" echo "<rpc><edit-config><target><candidate/></target><config><x xmlns=\"urn:example:clixon\"><c>$rnd</c></x></config></edit-config></rpc>]]>]]>"
done | $clixon_netconf -qf $cfg -y $fyang > /dev/null done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
;; ;;
esac esac
# new "discard test"
expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
} }
step(){ # Step
# args: <i> <reqs> <mode>
stepfn(){
i=$1 i=$1
mode=$2 reqs=$2
mode=$3
>&2 echo "stepfn $mode: i=$i reqs=$reqs"
echo -n "" > $fconfig echo -n "" > $fconfig
t=$(TEST=%e run $i $reqs $mode 2>&1 | awk '/real/ {print $2}') t=$(TEST=%e runfn $i $reqs $mode 2>&1 | awk '/real/ {print $2}')
#TEST=%e run $i $reqs $mode 2>&1 #TEST=%e runfn $i $reqs $mode 2>&1
# t is time in secs of $reqs -> transactions per second. $reqs # t is time in secs of $reqs -> transactions per second. $reqs
p=$(echo "$reqs/$t" | bc -lq) p=$(echo "$reqs/$t" | bc -lq)
# p is transactions per second. # p is transactions per second.
# write to gnuplot file: $dir/$mode
echo "$i $p" >> $dir/$mode echo "$i $p" >> $dir/$mode
# echo "m:$mode i:$i t=$t p=$p"
} }
# Run once
#args: <step> <reqs> <max>
once(){ once(){
# kill old backend (if any) # Input Parameters
step=$1
reqs=$2
max=$3
echo "oncefn step=$step reqs=$reqs max=$max"
new "test params: -f $cfg -y $fyang"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg -y $fyang sudo clixon_backend -zf $cfg -y $fyang
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
err err
fi fi
new "start backend -s init -f $cfg -y $fyang"
# start new backend start_backend -s init -f $cfg -y $fyang
sudo clixon_backend -s init -f $cfg -y $fyang
if [ $? -ne 0 ]; then
err
fi fi
# Always as a start new "kill old restconf daemon"
sudo pkill -u www-data -f "/www-data/clixon_restconf"
new "start restconf daemon"
start_restconf -f $cfg -y $fyang
new "waiting"
sleep $RCWAIT
new "Intial steps as start"
for (( i=10; i<=$step; i=i+10 )); do for (( i=10; i<=$step; i=i+10 )); do
step $i readlist stepfn $i $reqs readlist
step $i writelist stepfn $i $reqs writelist
step $i restreadlist stepfn $i $reqs restreadlist
step $i writeleaflist stepfn $i $reqs writeleaflist
done done
# Actual steps rnd=$(( ( RANDOM % $step ) ))
echo "curl -sSG http://localhost/restconf/data/scaling:x/y=$rnd"
curl -sSG http://localhost/restconf/data/scaling:x/y=$rnd
exit
new "Actual steps"
for (( i=$step; i<=$max; i=i+$step )); do for (( i=$step; i<=$max; i=i+$step )); do
step $i readlist stepfn $i $reqs readlist
step $i writelist stepfn $i $reqs writelist
step $i restreadlist stepfn $i $reqs restreadlist
step $i writeleaflist stepfn $i $reqs writeleaflist
done done
# Check if still alive new "Kill restconf daemon"
pid=`pgrep clixon_backend` stop_restconf
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=`pgrep -u root -f clixon_backend`
if [ -z "$pid" ]; then if [ -z "$pid" ]; then
err "backend already dead" err "backend already dead"
fi fi
# kill backend # kill backend
sudo clixon_backend -zf $cfg stop_backend -f $cfg
if [ $? -ne 0 ]; then
err "kill backend"
fi fi
} }
once # step=200 reqs=500 max=2000
once 200 500 1000
gnuplot -persist <<EOF gnuplot -persist <<EOF
set title "Clixon transactions per second r/w large lists" font ",14" textcolor rgbcolor "royalblue" set title "Clixon transactions per second r/w large lists" font ",14" textcolor rgbcolor "royalblue"
set xlabel "entries" set xlabel "entries"
set ylabel "transactions per second" set ylabel "transactions per second"
set terminal wxt enhanced title "Clixon transactions " persist raise set terminal x11 enhanced title "Clixon transactions " persist raise
plot "$dir/readlist" with linespoints title "read list", "$dir/writelist" with linespoints title "write list", "$dir/writeleaflist" with linespoints title "write leaf-list" , "$dir/restreadlist" with linespoints title "rest get list" plot "$dir/readlist" with linespoints title "read list", "$dir/writelist" with linespoints title "write list", "$dir/writeleaflist" with linespoints title "write leaf-list" , "$dir/restreadlist" with linespoints title "rest get list"
EOF EOF
rm -rf $dir #rm -rf $dir

215
test/test_insert.sh Executable file
View file

@ -0,0 +1,215 @@
#!/bin/bash
# XML Insert unit test
# First a list with 0-5 base elements, insert in different places
# Second varying yangs: container, leaf, list, leaf-list, choice, user-order list
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
: ${clixon_util_insert:=clixon_util_insert}
OPTS="-D $DBG -s"
APPNAME=example
cfg=$dir/conf_yang.xml
fyang=$dir/example.yang
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
<CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_CACHE>true</CLICON_XMLDB_CACHE>
</clixon-config>
EOF
cat <<EOF > $fyang
module example {
yang-version 1.1;
namespace "urn:example:example";
prefix ex;
revision 2019-01-13;
container c{
list a{
key x;
leaf x{
type int32;
}
}
}
}
EOF
testrun(){
x0=$1
xi="<c xmlns=\"urn:example:example\">$2</c>"
xp=c
new "random list add leaf-list"
# First run sorted (assume this is the refernce == correct)
rs=$($clixon_util_insert -y $fyang -x "$xi" -b "$x0" -p $xp $OPTS -s)
# Then run actual insert
r0=$($clixon_util_insert -y $fyang -x "$xi" -b "$x0" -p $xp $OPTS)
# If both are null something is amiss
if [ -z "$r0" -a -z "$rs" ]; then
err "length of retval is zero"
fi
# echo "rs:$rs"
# echo "r0:$r0"
# Check they are equal
if [[ "$r0" != "$rs" ]]; then
err "$rs" "$r0"
fi
}
new "test params: -y $fyang $OPTS"
# Empty element base list
x0='<c xmlns="urn:example:example"></c>'
new "empty list"
testrun "$x0" "<a><x>1</x></a>"
# One element base list
x0='<c xmlns="urn:example:example"><a><x>99</x></a></c>'
new "one element list first"
testrun "$x0" "<a><x>1</x></a>"
new "one element list last"
testrun "$x0" "<a><x>100</x></a>"
# Two element base list
x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a></c>'
new "two element list first"
testrun "$x0" "<a><x>1</x></a>"
new "two element list mid"
testrun "$x0" "<a><x>12</x></a>"
new "two element list last"
testrun "$x0" "<a><x>3000</x></a>"
# Three element base list
x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a><a><x>101</x></a></c>'
new "three element list first"
testrun "$x0" "<a><x>1</x></a>"
new "three element list second"
testrun "$x0" "<a><x>10</x></a>"
new "three element list third"
testrun "$x0" "<a><x>100</x></a>"
new "three element list last"
testrun "$x0" "<a><x>1000</x></a>"
# Four element base list
x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a><a><x>101</x></a><a><x>200</x></a></c>'
new "four element list first"
testrun "$x0" "<a><x>1</x></a>"
new "four element list second"
testrun "$x0" "<a><x>10</x></a>"
new "four element list third"
testrun "$x0" "<a><x>100</x></a>"
new "four element list fourth"
testrun "$x0" "<a><x>102</x></a>"
new "four element list last"
testrun "$x0" "<a><x>1000</x></a>"
# Five element base list
x0='<c xmlns="urn:example:example"><a><x>2</x></a><a><x>99</x></a><a><x>101</x></a><a><x>200</x></a><a><x>300</x></a></c>'
new "five element list first"
testrun "$x0" "<a><x>1</x></a>"
new "five element list mid"
testrun "$x0" "<a><x>100</x></a>"
new "five element list last"
testrun "$x0" "<a><x>1000</x></a>"
cat <<EOF > $fyang
module example {
yang-version 1.1;
namespace "urn:example:example";
prefix ex;
revision 2019-01-13;
container c{
leaf a{
type string;
}
container b{
leaf a {
type string;
}
}
choice c1{
case a{
leaf x{
type string;
}
}
case b{
leaf y{
type int32;
}
}
}
choice c2{
leaf z{
type string;
}
leaf t{
type int32;
}
}
list d{
key x;
leaf x{
type int32;
}
ordered-by user;
}
leaf-list e{
type int32;
}
}
}
EOF
# Advanced list
# Empty base list
x0='<c xmlns="urn:example:example"></c>'
xp=c
new "adv empty list add leaf"
testrun "$x0" "<a>leaf</a>"
new "adv empty list add choice c1"
testrun "$x0" "<x>choice1</x>"
xi='<c xmlns="urn:example:example"><e>33</e></c>'
new "adv empty list add leaf-list"
testrun "$x0" "<e>33</e>"
# base list
x0='<c xmlns="urn:example:example"><a>leaf</a><x>choice</x><d><x>1</x></d><e>33</e></c>'
new "adv list add leaf-list"
testrun "$x0" "<e>32</e>"
new "adv list add leaf-list"
testrun "$x0" "<e>32</e>"
rm -rf $dir

29
test/test_perf.sh
View file

@ -5,7 +5,7 @@
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
# Number of list/leaf-list entries in file # Number of list/leaf-list entries in file
: ${perfnr:=1000} : ${perfnr:=2000}
# Number of requests made get/put # Number of requests made get/put
: ${perfreq:=100} : ${perfreq:=100}
@ -15,6 +15,7 @@ APPNAME=example
cfg=$dir/scaling-conf.xml cfg=$dir/scaling-conf.xml
fyang=$dir/scaling.yang fyang=$dir/scaling.yang
fconfig=$dir/large.xml fconfig=$dir/large.xml
fconfig2=$dir/large2.xml
cat <<EOF > $fyang cat <<EOF > $fyang
module scaling{ module scaling{
@ -43,12 +44,11 @@ cat <<EOF > $cfg
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE> <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_YANG_DIR>$dir</CLICON_YANG_DIR> <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
<CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR> <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
<CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
<CLICON_YANG_MODULE_MAIN>scaling</CLICON_YANG_MODULE_MAIN> <CLICON_YANG_MODULE_MAIN>scaling</CLICON_YANG_MODULE_MAIN>
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK> <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE> <CLICON_BACKEND_PIDFILE>/usr/local/var/example/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY> <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
<CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR> <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY> <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
</clixon-config> </clixon-config>
EOF EOF
@ -90,9 +90,6 @@ expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfi
new "netconf write large config again" new "netconf write large config again"
expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
# Remove the file, its used for different purposes further down
rm $fconfig
# Now commit it from candidate to running # Now commit it from candidate to running
new "netconf commit large config" new "netconf commit large config"
expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
@ -119,7 +116,7 @@ done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
new "restconf get $perfreq small config" new "restconf get $perfreq small config"
time -p for (( i=0; i<$perfreq; i++ )); do time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) )) rnd=$(( ( RANDOM % $perfnr ) ))
curl -sG http://localhost/restconf/data/scaling:x/y=$rnd,$rnd > /dev/null curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null
done done
new "restconf add $perfreq small config" new "restconf add $perfreq small config"
@ -135,19 +132,23 @@ expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><get
new "restconf get large config" new "restconf get large config"
expecteof "/usr/bin/time -f %e curl -sG http://localhost/restconf/data" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" '^{"data": {"scaling:x": {"y": \[{"a": 0,"b": 0},{ "a": 1,"b": 1},{ "a": 2,"b": 2},{ "a": 3,"b": 3},' expecteof "/usr/bin/time -f %e curl -sG http://localhost/restconf/data" 0 "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" '^{"data": {"scaling:x": {"y": \[{"a": 0,"b": 0},{ "a": 1,"b": 1},{ "a": 2,"b": 2},{ "a": 3,"b": 3},'
new "restconf delete $perfreq small config"
time -p for (( i=0; i<$perfreq; i++ )); do
rnd=$(( ( RANDOM % $perfnr ) ))
curl -s -X DELETE http://localhost/restconf/data/scaling:x/y=$rnd
done
# Now do leaf-lists istead of leafs # Now do leaf-lists istead of leafs
new "generate large leaf-list config" new "generate large leaf-list config"
echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fconfig echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x xmlns=\"urn:example:clixon\">" > $fconfig2
for (( i=0; i<$perfnr; i++ )); do for (( i=0; i<$perfnr; i++ )); do
echo -n "<c>$i</c>" >> $fconfig echo -n "<c>$i</c>" >> $fconfig2
done done
echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig2
new "netconf replace large list-leaf config" new "netconf replace large list-leaf config"
expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig2" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
rm $fconfig
new "netconf commit large leaf-list config" new "netconf commit large leaf-list config"
expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><commit/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"

4
util/Makefile.in
View file

@ -73,6 +73,7 @@ APPSRC += clixon_util_json.c
APPSRC += clixon_util_yang.c APPSRC += clixon_util_yang.c
APPSRC += clixon_util_xpath.c APPSRC += clixon_util_xpath.c
APPSRC += clixon_util_datastore.c APPSRC += clixon_util_datastore.c
APPSRC += clixon_util_insert.c
ifeq ($(with_restconf),yes) ifeq ($(with_restconf),yes)
APPSRC += clixon_util_stream.c # Needs curl APPSRC += clixon_util_stream.c # Needs curl
endif endif
@ -107,6 +108,9 @@ clixon_util_stream: clixon_util_stream.c $(LIBDEPS)
clixon_util_datastore: clixon_util_datastore.c $(LIBDEPS) clixon_util_datastore: clixon_util_datastore.c $(LIBDEPS)
$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@
clixon_util_insert: clixon_util_insert.c $(LIBDEPS)
$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@
distclean: clean distclean: clean
rm -f Makefile *~ .depend rm -f Makefile *~ .depend

215
util/clixon_util_insert.c Normal file
View file

@ -0,0 +1,215 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
See https://www.w3.org/TR/xpath/
* Turn this on to get an xpath test program
* Usage: xpath [<xpath>]
* read xpath on first line and xml on rest of lines from input
* Example compile:
gcc -g -o xpath -I. -I../clixon ./clixon_xsl.c -lclixon -lcligen
* Example run:
echo "a\n<a><b/></a>" | xpath
*/
#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <limits.h>
#include <fnmatch.h>
#include <stdint.h>
#include <assert.h>
#include <syslog.h>
#include <fcntl.h>
/* cligen */
#include <cligen/cligen.h>
/* clixon */
#include "clixon/clixon.h"
static int
usage(char *argv0)
{
fprintf(stderr, "usage:%s [options]\n"
"where options are\n"
"\t-h \t\tHelp\n"
"\t-D <level>\tDebug\n"
"\t-y <file> \tYANG spec file (or stdin)\n"
"\t-b <base> \tXML base expression\n"
"\t-x <xml> \tXML to insert\n"
"\t-p <xpath>\tXpath to where in base and XML\n"
"\t-s \tSort output after insert\n"
"Assume insert xml is first child of xpath. Ie if xml=<a><x>23 and xpath=a, then inserted element is <x>23\n",
argv0
);
exit(0);
}
int
main(int argc, char **argv)
{
int retval = -1;
char *argv0 = argv[0];
int c;
char *filename = NULL;
int fd = 0; /* unless overriden by argv[1] */
char *x0str = NULL;
char *xistr = NULL;
char *xpath = NULL;
yang_stmt *yspec;
cxobj *x0 = NULL;
cxobj *xb;
cxobj *xi = NULL;
int sort = 0;
clicon_handle h;
clicon_log_init("clixon_insert", LOG_DEBUG, CLICON_LOG_STDERR);
if ((h = clicon_handle_init()) == NULL)
goto done;
optind = 1;
opterr = 0;
while ((c = getopt(argc, argv, "hD:y:b:x:p:s")) != -1)
switch (c) {
case 'h':
usage(argv0);
break;
case 'D':
if (sscanf(optarg, "%d", &debug) != 1)
usage(argv0);
break;
case 'y': /* YANG spec file */
filename = optarg;
if (0 && (fd = open(filename, O_RDONLY)) < 0){
clicon_err(OE_UNIX, errno, "open(%s)", argv[1]);
goto done;
}
break;
case 'b': /* Base XML expression */
x0str = optarg;
break;
case 'x': /* XML to insert */
xistr = optarg;
break;
case 'p': /* XPATH base */
xpath = optarg;
break;
case 's': /* sort output after insert */
sort++;
break;
default:
usage(argv[0]);
break;
}
if (xistr == NULL || x0str == NULL)
usage(argv0);
if (xpath == NULL)
usage(argv0);
if (filename == NULL)
usage(argv0);
clicon_debug(1, "xistr:%s", xistr);
clicon_debug(1, "x0str:%s", x0str);
clicon_debug(1, "xpath:%s", xpath);
if ((yspec = yspec_new()) == NULL)
goto done;
#if 1
if (yang_spec_parse_file(h, filename, yspec) < 0)
goto done;
#else
if (yang_parse_file(fd, "yang test", yspec) == NULL)
goto done;
#endif
/* Parse base XML */
if (xml_parse_string(x0str, yspec, &x0) < 0){
clicon_err(OE_XML, 0, "Parsing base xml: %s", x0str);
goto done;
}
if (xml_apply(x0, CX_ELMNT, xml_spec_populate, yspec) < 0)
goto done;
if ((xb = xpath_first(x0, "%s", xpath)) == NULL){
clicon_err(OE_XML, 0, "xpath: %s not found in x0", xpath);
goto done;
}
if (debug){
clicon_debug(1, "xb:");
xml_print(stderr, xb);
}
/* Parse insert XML */
if (xml_parse_string(xistr, yspec, &xi) < 0){
clicon_err(OE_XML, 0, "Parsing insert xml: %s", xistr);
goto done;
}
if (xml_apply(xi, CX_ELMNT, xml_spec_populate, yspec) < 0)
goto done;
if ((xi = xpath_first(xi, "%s", xpath)) == NULL){
clicon_err(OE_XML, 0, "xpath: %s not found in xi", xpath);
goto done;
}
/* Find first element child */
if ((xi = xml_child_i_type(xi, 0, CX_ELMNT)) == NULL){
clicon_err(OE_XML, 0, "xi has no element child");
goto done;
}
/* Remove it from parent */
if (xml_rm(xi) < 0)
goto done;
if (debug){
clicon_debug(1, "xi:");
xml_print(stderr, xi);
}
if (xml_insert(xb, xi) < 0)
goto done;
if (debug){
clicon_debug(1, "x0:");
xml_print(stderr, x0);
}
if (sort)
xml_sort(xb, NULL);
clicon_xml2file(stdout, xb, 0, 0);
retval = 0;
done:
if (x0)
xml_free(x0);
if (yspec)
yspec_free(yspec);
if (fd > 0)
close(fd);
return retval;
}