diff --git a/datastore/text/clixon_xmldb_text.c b/datastore/text/clixon_xmldb_text.c index e00583df..429329f8 100644 --- a/datastore/text/clixon_xmldb_text.c +++ b/datastore/text/clixon_xmldb_text.c @@ -312,64 +312,75 @@ singleconfigroot(cxobj *xt, return retval; } +/*! Given XML tree x0 with marked nodes, copy marked nodes to new tree x1 + * Two marks are used: XML_FLAG_MARK and XML_FLAG_CHANGE + * + * The algorithm works as following: + * (1) Copy individual nodes marked with XML_FLAG_CHANGE + * until nodes marked with XML_FLAG_MARK are reached, where + * (2) the complete subtree of that node is copied. + * (3) Special case: key nodes in lists are copied if any node in list is marked + */ static int xml_copy_marked(cxobj *x0, - cxobj *x1, - int flag, - int test, - int *upmark) + cxobj *x1) { int retval = -1; - int submark; int mark; cxobj *x; cxobj *xcopy; int iskey; - yang_node *yt; + yang_stmt *yt; char *name; + assert(x0 && x1); + yt = xml_spec(x0); /* can be null */ + /* Go through children to detect any marked nodes: + * (3) Special case: key nodes in lists are copied if any + * node in list is marked + */ mark = 0; - yt = xml_spec(x0); /* xan be null */ + x = NULL; + while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) { + if (xml_flag(x, XML_FLAG_MARK|XML_FLAG_CHANGE)){ + mark++; + break; + } + } x = NULL; while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) { name = xml_name(x); - if (xml_flag(x, flag) == test?flag:0){ - /* Pass test */ - mark++; - if (x1){ - if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL) - goto done; - if (xml_copy(x, xcopy) < 0) - goto done; - } - continue; /* mark and stop here */ - } - /* If it is key dont remove it yet (see second round) */ - if (yt){ - if ((iskey = yang_key_match(yt, name)) < 0) + if (xml_flag(x, XML_FLAG_MARK)){ + /* (2) the complete subtree of that node is copied. */ + if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL) goto done; - if (iskey) - continue; + if (xml_copy(x, xcopy) < 0) + goto done; + continue; } - if (xml_copy_marked(x, NULL, flag, test, &submark) < 0) - goto done; - /* if x0 is list and submark anywhere, then key subs are also marked - */ - if (submark){ - mark++; /* copy */ - if (x1){ + if (xml_flag(x, XML_FLAG_CHANGE)){ + /* Copy individual nodes marked with XML_FLAG_CHANGE */ + if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL) + goto done; + if (xml_copy_marked(x, xcopy) < 0) /* */ + goto done; + } + /* (3) Special case: key nodes in lists are copied if any + * node in list is marked */ + if (mark && yt && yt->ys_keyword == Y_LIST){ + /* XXX: I think yang_key_match is suboptimal here */ + if ((iskey = yang_key_match((yang_node*)yt, name)) < 0) + goto done; + if (iskey){ if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL) goto done; - if (xml_copy_marked(x, xcopy, flag, test, &submark) < 0) + if (xml_copy(x, xcopy) < 0) goto done; } - } } retval = 0; done: - if (upmark) - *upmark = mark; return retval; } @@ -458,20 +469,29 @@ text_get(xmldb_handle xh, * otherwise return complete tree. */ if (xvec != NULL) - for (i=0; i1) clicon_xml2file(stderr, xt, 0, 1); *xtop = xt; @@ -572,7 +586,7 @@ text_modify(cxobj *x0, case OP_REPLACE: if (x0==NULL){ // int iamkey=0; - if ((x0 = xml_new(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) goto done; #if 0 /* If it is key I dont want to mark it */ @@ -598,9 +612,10 @@ text_modify(cxobj *x0, if (xml_value_set(x0b, x1bstr) < 0) goto done; } - if (xml_child_hash && - xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) - goto done; + if (xml_child_hash && + xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) + goto done; + break; case OP_DELETE: if (x0==NULL){ @@ -638,21 +653,22 @@ text_modify(cxobj *x0, if (x0){ xml_purge(x0); } - if ((x0 = xml_new(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) goto done; if (xml_copy(x1, x0) < 0) goto done; break; } if (x0==NULL){ - if ((x0 = xml_new(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) goto done; if (op==OP_NONE) xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */ + } - if (xml_child_hash && - xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) - goto done; + if (xml_child_hash && + xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) + goto done; /* First pass: mark existing children in base */ /* Loop through children of the modification tree */ if ((x0vec = calloc(xml_child_nr(x1), sizeof(x1))) == NULL){ @@ -880,10 +896,11 @@ text_put(xmldb_handle xh, if (xml_apply(x1, CX_ELMNT, xml_spec_populate, yspec) < 0) goto done; /* Add hash */ +#if 0 if (xml_child_hash && xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) goto done; - +#endif /* * Modify base tree x with modification x1 */ @@ -1133,7 +1150,17 @@ text_delete(xmldb_handle xh, int retval = -1; char *filename = NULL; struct text_handle *th = handle(xh); + struct db_element *de = NULL; + cxobj *xt = NULL; + if (datastore_cache){ + if ((de = hash_value(th->th_dbs, db, NULL)) != NULL){ + if ((xt = de->de_xml) != NULL){ + xml_free(xt); + de->de_xml = NULL; + } + } + } if (text_db2file(th, db, &filename) < 0) goto done; if (unlink(filename) < 0){ @@ -1162,7 +1189,18 @@ text_create(xmldb_handle xh, struct text_handle *th = handle(xh); char *filename = NULL; int fd = -1; + struct db_element *de = NULL; + cxobj *xt = NULL; + if (datastore_cache){ /* XXX This should nt really happen? */ + if ((de = hash_value(th->th_dbs, db, NULL)) != NULL){ + if ((xt = de->de_xml) != NULL){ + assert(xt==NULL); /* XXX */ + xml_free(xt); + de->de_xml = NULL; + } + } + } if (text_db2file(th, db, &filename) < 0) goto done; if ((fd = open(filename, O_CREAT|O_WRONLY, S_IRWXU)) == -1) { diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 541f6d8d..de1dd636 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -113,9 +113,9 @@ cxobj *xml_child_each(cxobj *xparent, cxobj *xprev, enum cxobj_type type); cxobj **xml_childvec_get(cxobj *x); int xml_childvec_set(cxobj *x, int len); -cxobj *xml_new(char *name, cxobj *xn_parent, void *spec); -void *xml_spec(cxobj *x); -void *xml_spec_set(cxobj *x, void *spec); +cxobj *xml_new(char *name, cxobj *xn_parent, yang_stmt *spec); +yang_stmt *xml_spec(cxobj *x); +int xml_spec_set(cxobj *x, yang_stmt *spec); cxobj *xml_find(cxobj *xn_parent, char *name); int xml_addsub(cxobj *xp, cxobj *xc); diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index baa809fe..2c002a68 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -513,9 +513,9 @@ xml_childvec_get(cxobj *x) * eg for body or attribute */ cxobj * -xml_new(char *name, - cxobj *xp, - void *spec) +xml_new(char *name, + cxobj *xp, + yang_stmt *spec) { cxobj *x; @@ -531,7 +531,7 @@ xml_new(char *name, if (xp && xml_child_append(xp, x) < 0) return NULL; x->x_spec = spec; /* Can be NULL */ - if (xml_child_hash && xml_hash_add(x) < 0) + if (xml_child_hash && spec && xml_hash_add(x) < 0) return NULL; return x; } @@ -539,15 +539,15 @@ xml_new(char *name, /*! Return yang spec of node. * Not necessarily set. Either has not been set yet (by xml_spec_set( or anyxml. */ -void * +yang_stmt * xml_spec(cxobj *x) { return x->x_spec; } -void * -xml_spec_set(cxobj *x, - void *spec) +int +xml_spec_set(cxobj *x, + yang_stmt *spec) { x->x_spec = spec; return 0; @@ -1459,6 +1459,7 @@ cxvec_append(cxobj *x, * @note do not delete or move around any children during this function * @note return value > 0 aborts the traversal * @see xml_apply0 including top object + * @see xml_apply_ancestor for marking all parents recursively */ int xml_apply(cxobj *xn, @@ -1847,25 +1848,33 @@ xml_hash_key(cxobj *x, /*! XML hash add. Create hash and add key/value to parent * - * @param[in] arg 0: rm entry, 1: add + * @param[in] arg add flag. If 1, else if 0 remove. * Typically called for a whole tree. */ int xml_hash_op(cxobj *x, void *arg) { + int add = (intptr_t)arg; +#if 1 + if (add) + return xml_hash_add(x); + else + return xml_hash_rm_entry(x); +#else + int retval = -1; cxobj *xp; clicon_hash_t *ph; yang_stmt *y; cbuf *key = NULL; /* cligen buffer hash key */ - int op = (intptr_t)arg; + if (xml_hash(x)==NULL){ - if (op==1) + if (add) xml_hash_init(x); } - else if (op==0) + else if (!add) xml_hash_rm_only(x); if ((xp = xml_parent(x)) == NULL) goto ok; @@ -1881,7 +1890,7 @@ xml_hash_op(cxobj *x, goto done; if (cbuf_len(key)){ // fprintf(stderr, "%s add %s = 0x%x\n", __FUNCTION__, cbuf_get(key), (unsigned int)x); - if (op == 1){ + if (add){ if (hash_add(ph, cbuf_get(key), &x, sizeof(x)) == NULL) goto done; } @@ -1895,6 +1904,7 @@ xml_hash_op(cxobj *x, if (key) cbuf_free(key); return retval; +#endif } /*! XML hash add. Create hash and add key/value to parent @@ -1909,6 +1919,7 @@ xml_hash_add(cxobj *x) cxobj *xp; clicon_hash_t *ph; yang_stmt *y; + yang_stmt *yp; cbuf *key = NULL; /* cligen buffer hash key */ if ((ph = xml_hash(x))==NULL){ @@ -1917,6 +1928,9 @@ xml_hash_add(cxobj *x) } if ((xp = xml_parent(x)) == NULL) goto ok; + yp = xml_spec(xp); + if (yp && yp->ys_keyword != Y_LIST) + goto ok; if ((y = xml_spec(x)) == NULL) goto ok; if ((key = cbuf_new()) == NULL){ diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 58ad338f..08746aed 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -620,9 +620,9 @@ match_base_child(cxobj *x0, int equal; char **b1vec = NULL; int i; - cxobj **p; - cbuf *key = NULL; /* cligen buffer hash key */ - size_t vlen; + cxobj **p; + cbuf *key = NULL; /* cligen buffer hash key */ + size_t vlen; if (xml_child_hash){ *x0cp = NULL; /* return value */ @@ -1204,7 +1204,7 @@ xml_tree_prune_flagged_sub(cxobj *xt, cxobj *xprev; int iskey; int anykey=0; - yang_node *yt; + yang_stmt *yt; mark = 0; yt = xml_spec(xt); /* xan be null */ @@ -1219,7 +1219,7 @@ xml_tree_prune_flagged_sub(cxobj *xt, } /* If it is key dont remove it yet (see second round) */ if (yt){ - if ((iskey = yang_key_match(yt, xml_name(x))) < 0) + if ((iskey = yang_key_match((yang_node*)yt, xml_name(x))) < 0) goto done; if (iskey){ anykey++; @@ -1247,7 +1247,7 @@ xml_tree_prune_flagged_sub(cxobj *xt, while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) { /* If it is key remove it here */ if (yt){ - if ((iskey = yang_key_match(yt, xml_name(x))) < 0) + if ((iskey = yang_key_match((yang_node*)yt, xml_name(x))) < 0) goto done; if (iskey && xml_purge(x) < 0) goto done; @@ -1858,7 +1858,7 @@ xml_merge1(cxobj *x0, if (y0->yn_keyword == Y_LEAF_LIST || y0->yn_keyword == Y_LEAF){ x1bstr = xml_body(x1); if (x0==NULL){ - if ((x0 = xml_new(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) goto done; if (x1bstr){ /* empty type does not have body */ if ((x0b = xml_new("body", x0, NULL)) == NULL) @@ -1879,7 +1879,7 @@ xml_merge1(cxobj *x0, } /* if LEAF|LEAF_LIST */ else { /* eg Y_CONTAINER, Y_LIST */ if (x0==NULL){ - if ((x0 = xml_new(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) goto done; } /* Loop through children of the modification tree */ diff --git a/test/test_datastore_scaling.sh b/test/test_datastore_scaling.sh deleted file mode 100755 index ca1f739c..00000000 --- a/test/test_datastore_scaling.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# Scaling test - -if [ $# = 0 ]; then - number=1000 - req=10 -elif [ $# = 1 ]; then - number=$1 - req=10 -elif [ $# = 2 ]; then - number=$1 - req=$2 -else - echo "Usage: $0 [ []]" - exit 1 -fi -rnd=$(( ( RANDOM % $number ) )) - -fyang=/tmp/scaling.yang -db=/tmp/text/candidate_db -name=text -dir=/tmp/text -conf="-d candidate -b $dir -p ../datastore/$name/$name.so -y /tmp -m ietf-ip" - -# include err() and new() functions -. ./lib.sh -clixon_cf=/tmp/scaling-conf.xml - - -# For memcheck -# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -# clixon_netconf="valgrind --tool=callgrind clixon_netconf -clixon_netconf=clixon_netconf - - -cat < $fyang -module example{ - container x { - list y { - key "a"; - leaf a { - type string; - } - leaf b { - type string; - } - } - leaf-list c { - type string; - } - } -} -EOF - -cat < $clixon_cf - - /tmp/test_yang.xml - /usr/local/share/routing/yang - example - /usr/local/var/routing/routing.sock - /usr/local/var/routing/routing.pidfile - /usr/local/var/routing - /usr/local/lib/xmldb/text.so - -EOF - -if [ ! -d $dir ]; then - mkdir $dir -fi - -echo "datastore_client $conf mget $req /x/y[a=$rnd][b=$rnd]" - -new "generate large list config" -echo -n "" > $db -for (( i=0; i<$number; i++ )); do - echo -n "$i$i" >> $db -done -echo "" >> $db - -new "datastore_client $name init" -expectfn "datastore_client $conf init" "" - -new "datastore $name mget" -expectfn "datastore_client $conf mget 1 /x/y[a=$rnd][b=$rnd]" "^$rnd$rnd$" - -new "make $req gets" -time datastore_client $conf mget $req "/x/y[a=$rnd][b=$rnd]" > /dev/null - diff --git a/test/test_perf.sh b/test/test_perf.sh index fa31f4d8..fcadc6f0 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -1,12 +1,17 @@ #!/bin/bash # Scaling test +number=1000 +req=100 if [ $# = 0 ]; then number=1000 elif [ $# = 1 ]; then number=$1 +elif [ $# = 2 ]; then + number=$1 + req=$2 else - echo "Usage: $0 []" + echo "Usage: $0 [ []]" exit 1 fi @@ -15,6 +20,7 @@ fconfig=/tmp/config # include err() and new() functions . ./lib.sh +clixon_cf=/tmp/scaling-conf.xml # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -40,6 +46,19 @@ module ietf-ip{ } EOF +cat < $clixon_cf + + $clixon_cf + $fyang + ietf-ip + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + + # kill old backend (if any) new "kill old backend" sudo clixon_backend -zf $clixon_cf -y $fyang @@ -61,26 +80,44 @@ for (( i=0; i<$number; i++ )); do done echo "]]>]]>" >> $fconfig +# Just for manual dbg +echo "$clixon_netconf -qf $clixon_cf -y $fyang" + new "netconf edit large config" -expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^]]>]]>$" + +echo ']]>]]>' | $clixon_netconf -qf $clixon_cf -y $fyang new "netconf edit large config again" -expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^]]>]]>$" + +echo ']]>]]>' | $clixon_netconf -qf $clixon_cf -y $fyang rm $fconfig new "netconf commit large config" expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "]]>]]>" "^]]>]]>$" -new "netconf add small config" +new "netconf add one small config" expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "xy]]>]]>" "^]]>]]>$" +new "netconf add $req small config" +time -p for (( i=0; i<$req; i++ )); do + rnd=$(( ( RANDOM % $number ) )) + echo "$rnd$rnd]]>]]>" +done | $clixon_netconf -qf $clixon_cf -y $fyang > /dev/null + new "netconf commit small config" expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "]]>]]>" "^]]>]]>$" new "netconf get large config" expecteof "time -p $clixon_netconf -qf $clixon_cf -y $fyang" "]]>]]>" "^0011" +new "netconf get $req small config" +time -p for (( i=0; i<$req; i++ )); do + rnd=$(( ( RANDOM % $number ) )) + echo "]]>]]>" +done | $clixon_netconf -qf $clixon_cf -y $fyang > /dev/null new "generate large leaf-list config" echo -n "replace" > $fconfig diff --git a/test/test_yang.sh b/test/test_yang.sh index 779cb84a..47f1131c 100755 --- a/test/test_yang.sh +++ b/test/test_yang.sh @@ -128,13 +128,12 @@ expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" " new "netconf set presence and not present" expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "]]>]]>" "^]]>]]>$" -new "netconf get" -expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "]]>]]>" "^]]>]]>$" +new "netconf get presence only" +expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" ']]>]]>' "^]]>]]>$" new "netconf discard-changes" expecteof "$clixon_netconf -qf $clixon_cf" "]]>]]>" "^]]>]]>$" - new "netconf anyxml" expecteof "$clixon_netconf -qf $clixon_cf -y /tmp/test.yang" "]]>]]>" "^]]>]]>$"