diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f6d347b..6752cf31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,11 +21,11 @@ * [3.3.1](#331) June 7 2017 ## 4.4.0 -Expected: February 2020 +Expected: Early March 2020 ### Major New features -* New "general-purpose" datastore upgrade callback added which i called once on startup, intended for low-level general upgrades and as a complement to module-specific upgrade. +* New "general-purpose" datastore upgrade callback called once on startup, intended for low-level general upgrades and as a complement to module-specific upgrade. * Called on startup after initial XML parsing, but before module-specific upgrades * Enabled by definign the `.ca_datastore_upgrade` * [General-purpose upgrade documentation](https://clixon-docs.readthedocs.io/en/latest/backend.html#general-purpose) @@ -39,8 +39,10 @@ Expected: February 2020 [search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml) ### API changes on existing features (you may need to change your code) +* Bugfix of config false statement may cause change of sorting of lists in GET opertions (lists that were sorted should not have been sorted) * New clixon-config@2020-02-22.yang revision - * Added search index extension + * Search index extension `search_index` for declaring which non-key variables are search indexes + * Added `clixon-stats` for clixon XML and memory statistics. * JSON parse error messages change from ` on line x: syntax error,..` to `json_parse: line x: syntax error` * Unknown-element error message is more descriptive, eg from `namespace is: urn:example:clixon` to: `Failed to find YANG spec of XML node: x with parent: xp in namespace urn:example:clixon`. * C-API parse and validation API more capable @@ -76,7 +78,7 @@ Expected: February 2020 ### Corrected Bugs -* Fixed: Search function checked only own not for config false statement, should have checked all ancestors. +* Fixed: Search function checked only own not for config false statement, should have checked all ancestors. This may affect some state returned in GET calls * Fixed: Some restconf errors were wrongly formatted such as: `{"ietf-restconf:errors":{"error":{"rpc-error":` . There should be no `"rpc-error"` level. * Fixed: Enabling modstate (CLICON_XMLDB_MODSTATE), changing a revision on a yang, and restarting made the backend daemon exit at start (thanks Matt) * Also: ensure to load `ietf-yang-library.yang ` if CLICON_XMLDB_MODSTATE is set diff --git a/LICENSE.md b/LICENSE.md index d0d73b96..ad067e78 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,5 +1,6 @@ Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren -Copyright (C) 2017-2020 Olof Hagsand +Copyright (C) 2017-2019 Olof Hagsand +Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC CLIXON is dual license. diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index f4ae36bf..ee69740c 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -256,6 +257,84 @@ client_get_streams(clicon_handle h, goto done; } +/*! Get clixon per datastore stats + * @param[in] h Clicon handle + * @param[in] dbname Datastore name + * @param[in,out] cb Cligen buf + * @retval 0 OK + * @retval -1 Error + */ +static int +clixon_stats_get_db(clicon_handle h, + char *name, + cbuf *cb) +{ + int retval = -1; + cxobj *xt = NULL; + uint64_t nr = 0; + size_t sz = 0; + + if (xmldb_get(h, "running", NULL, NULL, &xt) < 0) + goto done; + xml_stats(xt, &nr, &sz); + cprintf(cb, "%s%" PRIu64 "" + "%" PRIu64 "", + name, nr, sz); + retval = 0; + done: + if (xt) + free(xt); + return retval; +} + +/*! Get clixon stats + * @param[in] h Clicon handle + * @param[in] yspec Yang spec + * @param[in] xpath XML Xpath + * @param[in] nsc XML Namespace context for xpath + * @param[in,out] xret Existing XML tree, merge x into this + * @retval 0 OK +* @retval -1 Error + + */ +int +clixon_stats_get(clicon_handle h, + yang_stmt *yspec, + char *xpath, + cvec *nsc, + cxobj **xret) +{ + int retval = -1; + cbuf *cb = NULL; + uint64_t nr; + int ret; + + if ((cb = cbuf_new()) == NULL){ + clicon_err(OE_UNIX, errno, "cbuf_new"); + goto done; + } + cprintf(cb, "", CLIXON_CONF_NS); + nr=0; + xml_stats_global(&nr); + cprintf(cb, "%" PRIu64 "", nr); + clixon_stats_get_db(h, "running", cb); + clixon_stats_get_db(h, "candidate", cb); + clixon_stats_get_db(h, "startup", cb); + cprintf(cb, ""); + if ((ret = xml_parse_string2(cbuf_get(cb), YB_TOP, yspec, xret, NULL)) < 0) + goto done; + if (ret == 0){ + clicon_err(OE_XML, EINVAL, "Internal error"); + goto done; + } + retval = 0; + done: + clicon_debug(1, "%s %d", __FUNCTION__, retval); + if (cb) + cbuf_free(cb); + return retval; +} + /*! Get system state-data, including streams and plugins * @param[in] h Clicon handle * @param[in] xpath Xpath selection, not used but may be to filter early @@ -331,6 +410,11 @@ client_statedata(clicon_handle h, if (ret == 0) goto fail; } + /* Clixon-config has a state data, if yang is present */ + if (yang_find(yspec, Y_MODULE, "clixon-config") != NULL){ + if (clixon_stats_get(h, yspec, xpath, nsc, xret) < 0) + goto done; + } if ((ret = clixon_plugin_statedata(h, yspec, nsc, xpath, xret)) < 0) goto done; if (ret == 0) diff --git a/lib/clixon/clixon_options.h b/lib/clixon/clixon_options.h index 51eaedc7..554ec763 100644 --- a/lib/clixon/clixon_options.h +++ b/lib/clixon/clixon_options.h @@ -2,7 +2,9 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2019 Olof Hagsand + Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC This file is part of CLIXON. @@ -38,6 +40,15 @@ #ifndef _CLIXON_OPTIONS_H_ #define _CLIXON_OPTIONS_H_ +/* + * Constants + */ +/*! Clixon configuration namespace + * Probably should be defined somewhere else or extracted from yang + * @see clixon-config.yang + */ +#define CLIXON_CONF_NS "http://clicon.org/config" + /* * Types */ diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index 8a81bd6a..ccc1550c 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -80,12 +80,6 @@ #include "clixon_validate.h" #include "clixon_xml_map.h" -/*! Clixon configuration namespace - * Probably should be defined somewhere else or extracted from yang - * @see clixon-config.yang - */ -#define CLIXON_CONF_NS "http://clicon.org/config" - /* Mapping between Cli generation from Yang string <--> constants, see clixon-config.yang type cli_genmodel_type */ static const map_str2int cli_genmodel_map[] = { diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index a32cb619..2dc1f773 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -237,8 +237,19 @@ xml_stats_one(cxobj *x, sz += cv_size(x->x_cv); if (x->x_ns_cache) sz += cvec_size(x->x_ns_cache); +#ifdef XML_EXPLICIT_INDEX + if (x->x_search_index){ + /* XXX: only one */ + sz += sizeof(struct search_index); + if (x->x_search_index->si_name) + sz += strlen(x->x_search_index->si_name)+1; + if (x->x_search_index->si_xvec) + sz += clixon_xvec_len(x->x_search_index->si_xvec)*sizeof(struct cxobj*); + } +#endif if (szp) *szp = sz; + clicon_debug(1, "%s %" PRIu64, __FUNCTION__, sz); return 0; } @@ -261,10 +272,12 @@ xml_stats(cxobj *xt, *szp += sz; xc = NULL; while ((xc = xml_child_each(xt, xc, -1)) != NULL) { + sz=0; xml_stats(xc, nrp, &sz); if (szp) *szp += sz; } + clicon_debug(1, "%s %" PRIu64, __FUNCTION__, *szp); return 0; } diff --git a/lib/src/clixon_xpath_optimize.c b/lib/src/clixon_xpath_optimize.c index 9b664e17..85dbb543 100644 --- a/lib/src/clixon_xpath_optimize.c +++ b/lib/src/clixon_xpath_optimize.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2020 Olof Hagsand + Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC This file is part of CLIXON. diff --git a/test/test_perf.sh b/test/test_perf.sh index 927358bf..3a0064cc 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -267,4 +267,4 @@ rm -rf $dir # unset conditional parameters unset format unset perfnr -unset perfreg +unset perfreq diff --git a/test/test_perf_mem.sh b/test/test_perf_mem.sh new file mode 100755 index 00000000..8b1cbdde --- /dev/null +++ b/test/test_perf_mem.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Backend Memory tests, footprint using the clixon-conf state statistics +# Create a large datastore, load it and measure + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +clixon_util_xpath=clixon_util_xpath + +# Number of list/leaf-list entries in file +: ${perfnr:=10000} + +APPNAME=example + +cfg=$dir/scaling-conf.xml +fyang=$dir/scaling.yang +pidfile=$dir/pidfile + +cat < $fyang +module scaling{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix ex; + import "clixon-config" { + prefix cc; + } + container x { + list y { + key "a"; + leaf a { + type int32; + } + leaf b { + type int32; + } + } + } +} +EOF + +cat < $cfg + + $cfg + ietf-netconf:startup + $dir + /usr/local/share/clixon + $fyang + /usr/local/var/$APPNAME/$APPNAME.sock + $pidfile + false + $dir + false + example + /usr/local/lib/example/cli + /usr/local/lib/example/clispec + 0 + ietf-netconf:startup + +EOF + +testrun(){ + nr=$1 + + new "generate config with $nr list entries" + echo -n "" > $dir/startup_db + for (( i=0; i<$nr; i++ )); do + echo -n "$i$i" >> $dir/startup_db + done + echo "" >> $dir/startup_db + + new "test params: -f $cfg" + if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s startup -f $cfg" + start_backend -s startup -f $cfg + fi + + new "waiting" + wait_backend + + pid=$(cat $pidfile) + + new "netconf get state" + res=$(echo "]]>]]>" | $clixon_netconf -qf $cfg) + echo "Total" + echo -n " objects: " + echo $res | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/global/xmlnr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' + echo -n " mem: " + # This ony works on Linux + cat /proc/$pid/statm|awk '{print $1*4/1000 "M"}' + for db in running candidate startup; do + echo "$db" + resdb=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/datastore[name=\"$db\"]") + resdb=${resdb#"nodeset:0:"} +# echo "resdb:$resdb" + echo -n " objects: " + echo $resdb | $clixon_util_xpath -p "datastore/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' + echo -n " mem: " + echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}' + done + + if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + fi +} + +new "Memory test for backend with 1 $perfnr entries" +testrun $perfnr + +rm -rf $dir + +# unset conditional parameters +unset perfnr + diff --git a/test/test_perf_state.sh b/test/test_perf_state.sh index 34fb5af7..86851abc 100755 --- a/test/test_perf_state.sh +++ b/test/test_perf_state.sh @@ -160,6 +160,6 @@ rm -rf $dir # unset conditional parameters unset format unset perfnr -unset perfreg +unset perfreq diff --git a/util/clixon_util_xpath.c b/util/clixon_util_xpath.c index 7d8f3599..8ff79a3f 100644 --- a/util/clixon_util_xpath.c +++ b/util/clixon_util_xpath.c @@ -324,23 +324,8 @@ main(int argc, } else x = x0; -#ifdef XPATH_LIST_OPTIMIZE /* Experimental */ - { - int hits = 0; - int j; - - xpath_list_optimize_stats(&hits); - for (j=0;j<1;j++){ - if (xpath_vec_ctx(x, nsc, xpath, 0, &xc) < 0) - return -1; - } - xpath_list_optimize_stats(&hits); - fprintf(stderr, "hits after:%d\n", hits); - } -#else if (xpath_vec_ctx(x, nsc, xpath, 0, &xc) < 0) return -1; -#endif /* Print results */ cb = cbuf_new(); ctx_print2(cb, xc); diff --git a/yang/clixon/clixon-config@2020-02-22.yang b/yang/clixon/clixon-config@2020-02-22.yang index 4d27a381..385c4f16 100644 --- a/yang/clixon/clixon-config@2020-02-22.yang +++ b/yang/clixon/clixon-config@2020-02-22.yang @@ -41,7 +41,9 @@ module clixon-config { ***** END LICENSE BLOCK *****"; revision 2020-02-22 { - description "Added search index extension"; + description + "Added: search index extension, + Added: clixon-stats state for clixon XML and memory statistics."; } revision 2019-09-11 { description @@ -673,12 +675,11 @@ module clixon-config { description "Clixon backend statistics."; container global{ description "Clixon global statistics"; - leaf nr{ - description "Number of cxobj:ects. That is number of residing xml/json objects + leaf xmlnr{ + description "Number of XML objects. That is number of residing xml/json objects in the internal 'cxobj' representation."; type uint64; } - } list datastore{ description "Datastore statistics"; @@ -688,7 +689,8 @@ module clixon-config { type string; } leaf nr{ - description "Number bytes of internal datastore cache of datastore tree."; + description "Number of XML objects. That is number of residing xml/json objects + in the internal 'cxobj' representation."; type uint64; } leaf size{