* Added clixon-stats in clixon-config.yang for clixon XML and memory statistics.
This commit is contained in:
Olof hagsand
parent
f1ceec689a
commit
2d521d52c8
12 changed files with 253 additions and 35 deletions
10
CHANGELOG.md
10
CHANGELOG.md
|
|
@ -21,11 +21,11 @@
|
|||
* [3.3.1](#331) June 7 2017
|
||||
|
||||
## 4.4.0
|
||||
Expected: February 2020
|
||||
Expected: Early March 2020
|
||||
|
||||
### Major New features
|
||||
|
||||
* New "general-purpose" datastore upgrade callback added which i called once on startup, intended for low-level general upgrades and as a complement to module-specific upgrade.
|
||||
* New "general-purpose" datastore upgrade callback called once on startup, intended for low-level general upgrades and as a complement to module-specific upgrade.
|
||||
* Called on startup after initial XML parsing, but before module-specific upgrades
|
||||
* Enabled by definign the `.ca_datastore_upgrade`
|
||||
* [General-purpose upgrade documentation](https://clixon-docs.readthedocs.io/en/latest/backend.html#general-purpose)
|
||||
|
|
@ -39,8 +39,10 @@ Expected: February 2020
|
|||
[search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml)
|
||||
|
||||
### API changes on existing features (you may need to change your code)
|
||||
* Bugfix of config false statement may cause change of sorting of lists in GET opertions (lists that were sorted should not have been sorted)
|
||||
* New clixon-config@2020-02-22.yang revision
|
||||
* Added search index extension
|
||||
* Search index extension `search_index` for declaring which non-key variables are search indexes
|
||||
* Added `clixon-stats` for clixon XML and memory statistics.
|
||||
* JSON parse error messages change from ` on line x: syntax error,..` to `json_parse: line x: syntax error`
|
||||
* Unknown-element error message is more descriptive, eg from `namespace is: urn:example:clixon` to: `Failed to find YANG spec of XML node: x with parent: xp in namespace urn:example:clixon`.
|
||||
* C-API parse and validation API more capable
|
||||
|
|
@ -76,7 +78,7 @@ Expected: February 2020
|
|||
|
||||
### Corrected Bugs
|
||||
|
||||
* Fixed: Search function checked only own not for config false statement, should have checked all ancestors.
|
||||
* Fixed: Search function checked only own not for config false statement, should have checked all ancestors. This may affect some state returned in GET calls
|
||||
* Fixed: Some restconf errors were wrongly formatted such as: `{"ietf-restconf:errors":{"error":{"rpc-error":` . There should be no `"rpc-error"` level.
|
||||
* Fixed: Enabling modstate (CLICON_XMLDB_MODSTATE), changing a revision on a yang, and restarting made the backend daemon exit at start (thanks Matt)
|
||||
* Also: ensure to load `ietf-yang-library.yang ` if CLICON_XMLDB_MODSTATE is set
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
|
||||
Copyright (C) 2017-2020 Olof Hagsand
|
||||
Copyright (C) 2017-2019 Olof Hagsand
|
||||
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
|
||||
|
||||
|
||||
CLIXON is dual license.
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@
|
|||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <stdarg.h>
|
||||
#include <inttypes.h>
|
||||
#include <errno.h>
|
||||
#include <signal.h>
|
||||
#include <fcntl.h>
|
||||
|
|
@ -256,6 +257,84 @@ client_get_streams(clicon_handle h,
|
|||
goto done;
|
||||
}
|
||||
|
||||
/*! Get clixon per datastore stats
|
||||
* @param[in] h Clicon handle
|
||||
* @param[in] dbname Datastore name
|
||||
* @param[in,out] cb Cligen buf
|
||||
* @retval 0 OK
|
||||
* @retval -1 Error
|
||||
*/
|
||||
static int
|
||||
clixon_stats_get_db(clicon_handle h,
|
||||
char *name,
|
||||
cbuf *cb)
|
||||
{
|
||||
int retval = -1;
|
||||
cxobj *xt = NULL;
|
||||
uint64_t nr = 0;
|
||||
size_t sz = 0;
|
||||
|
||||
if (xmldb_get(h, "running", NULL, NULL, &xt) < 0)
|
||||
goto done;
|
||||
xml_stats(xt, &nr, &sz);
|
||||
cprintf(cb, "<datastore><name>%s</name><nr>%" PRIu64 "</nr>"
|
||||
"<size>%" PRIu64 "</size></datastore>",
|
||||
name, nr, sz);
|
||||
retval = 0;
|
||||
done:
|
||||
if (xt)
|
||||
free(xt);
|
||||
return retval;
|
||||
}
|
||||
|
||||
/*! Get clixon stats
|
||||
* @param[in] h Clicon handle
|
||||
* @param[in] yspec Yang spec
|
||||
* @param[in] xpath XML Xpath
|
||||
* @param[in] nsc XML Namespace context for xpath
|
||||
* @param[in,out] xret Existing XML tree, merge x into this
|
||||
* @retval 0 OK
|
||||
* @retval -1 Error
|
||||
|
||||
*/
|
||||
int
|
||||
clixon_stats_get(clicon_handle h,
|
||||
yang_stmt *yspec,
|
||||
char *xpath,
|
||||
cvec *nsc,
|
||||
cxobj **xret)
|
||||
{
|
||||
int retval = -1;
|
||||
cbuf *cb = NULL;
|
||||
uint64_t nr;
|
||||
int ret;
|
||||
|
||||
if ((cb = cbuf_new()) == NULL){
|
||||
clicon_err(OE_UNIX, errno, "cbuf_new");
|
||||
goto done;
|
||||
}
|
||||
cprintf(cb, "<clixon-stats xmlns=\"%s\">", CLIXON_CONF_NS);
|
||||
nr=0;
|
||||
xml_stats_global(&nr);
|
||||
cprintf(cb, "<global><xmlnr>%" PRIu64 "</xmlnr></global>", nr);
|
||||
clixon_stats_get_db(h, "running", cb);
|
||||
clixon_stats_get_db(h, "candidate", cb);
|
||||
clixon_stats_get_db(h, "startup", cb);
|
||||
cprintf(cb, "</clixon-stats>");
|
||||
if ((ret = xml_parse_string2(cbuf_get(cb), YB_TOP, yspec, xret, NULL)) < 0)
|
||||
goto done;
|
||||
if (ret == 0){
|
||||
clicon_err(OE_XML, EINVAL, "Internal error");
|
||||
goto done;
|
||||
}
|
||||
retval = 0;
|
||||
done:
|
||||
clicon_debug(1, "%s %d", __FUNCTION__, retval);
|
||||
if (cb)
|
||||
cbuf_free(cb);
|
||||
return retval;
|
||||
}
|
||||
|
||||
/*! Get system state-data, including streams and plugins
|
||||
* @param[in] h Clicon handle
|
||||
* @param[in] xpath Xpath selection, not used but may be to filter early
|
||||
|
|
@ -331,6 +410,11 @@ client_statedata(clicon_handle h,
|
|||
if (ret == 0)
|
||||
goto fail;
|
||||
}
|
||||
/* Clixon-config has a state data, if yang is present */
|
||||
if (yang_find(yspec, Y_MODULE, "clixon-config") != NULL){
|
||||
if (clixon_stats_get(h, yspec, xpath, nsc, xret) < 0)
|
||||
goto done;
|
||||
}
|
||||
if ((ret = clixon_plugin_statedata(h, yspec, nsc, xpath, xret)) < 0)
|
||||
goto done;
|
||||
if (ret == 0)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
|
||||
Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
|
||||
Copyright (C) 2017-2019 Olof Hagsand
|
||||
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
|
||||
|
||||
This file is part of CLIXON.
|
||||
|
||||
|
|
@ -38,6 +40,15 @@
|
|||
#ifndef _CLIXON_OPTIONS_H_
|
||||
#define _CLIXON_OPTIONS_H_
|
||||
|
||||
/*
|
||||
* Constants
|
||||
*/
|
||||
/*! Clixon configuration namespace
|
||||
* Probably should be defined somewhere else or extracted from yang
|
||||
* @see clixon-config.yang
|
||||
*/
|
||||
#define CLIXON_CONF_NS "http://clicon.org/config"
|
||||
|
||||
/*
|
||||
* Types
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -80,12 +80,6 @@
|
|||
#include "clixon_validate.h"
|
||||
#include "clixon_xml_map.h"
|
||||
|
||||
/*! Clixon configuration namespace
|
||||
* Probably should be defined somewhere else or extracted from yang
|
||||
* @see clixon-config.yang
|
||||
*/
|
||||
#define CLIXON_CONF_NS "http://clicon.org/config"
|
||||
|
||||
/* Mapping between Cli generation from Yang string <--> constants,
|
||||
see clixon-config.yang type cli_genmodel_type */
|
||||
static const map_str2int cli_genmodel_map[] = {
|
||||
|
|
|
|||
|
|
@ -237,8 +237,19 @@ xml_stats_one(cxobj *x,
|
|||
sz += cv_size(x->x_cv);
|
||||
if (x->x_ns_cache)
|
||||
sz += cvec_size(x->x_ns_cache);
|
||||
#ifdef XML_EXPLICIT_INDEX
|
||||
if (x->x_search_index){
|
||||
/* XXX: only one */
|
||||
sz += sizeof(struct search_index);
|
||||
if (x->x_search_index->si_name)
|
||||
sz += strlen(x->x_search_index->si_name)+1;
|
||||
if (x->x_search_index->si_xvec)
|
||||
sz += clixon_xvec_len(x->x_search_index->si_xvec)*sizeof(struct cxobj*);
|
||||
}
|
||||
#endif
|
||||
if (szp)
|
||||
*szp = sz;
|
||||
clicon_debug(1, "%s %" PRIu64, __FUNCTION__, sz);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -261,10 +272,12 @@ xml_stats(cxobj *xt,
|
|||
*szp += sz;
|
||||
xc = NULL;
|
||||
while ((xc = xml_child_each(xt, xc, -1)) != NULL) {
|
||||
sz=0;
|
||||
xml_stats(xc, nrp, &sz);
|
||||
if (szp)
|
||||
*szp += sz;
|
||||
}
|
||||
clicon_debug(1, "%s %" PRIu64, __FUNCTION__, *szp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@
|
|||
*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright (C) 2009-2020 Olof Hagsand
|
||||
Copyright (C) 2009-2019 Olof Hagsand
|
||||
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
|
||||
|
||||
This file is part of CLIXON.
|
||||
|
||||
|
|
|
|||
|
|
@ -267,4 +267,4 @@ rm -rf $dir
|
|||
# unset conditional parameters
|
||||
unset format
|
||||
unset perfnr
|
||||
unset perfreg
|
||||
unset perfreq
|
||||
|
|
|
|||
125
test/test_perf_mem.sh
Executable file
125
test/test_perf_mem.sh
Executable file
|
|
@ -0,0 +1,125 @@
|
|||
#!/usr/bin/env bash
|
||||
# Backend Memory tests, footprint using the clixon-conf state statistics
|
||||
# Create a large datastore, load it and measure
|
||||
|
||||
# Magic line must be first in script (see README.md)
|
||||
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
|
||||
|
||||
clixon_util_xpath=clixon_util_xpath
|
||||
|
||||
# Number of list/leaf-list entries in file
|
||||
: ${perfnr:=10000}
|
||||
|
||||
APPNAME=example
|
||||
|
||||
cfg=$dir/scaling-conf.xml
|
||||
fyang=$dir/scaling.yang
|
||||
pidfile=$dir/pidfile
|
||||
|
||||
cat <<EOF > $fyang
|
||||
module scaling{
|
||||
yang-version 1.1;
|
||||
namespace "urn:example:clixon";
|
||||
prefix ex;
|
||||
import "clixon-config" {
|
||||
prefix cc;
|
||||
}
|
||||
container x {
|
||||
list y {
|
||||
key "a";
|
||||
leaf a {
|
||||
type int32;
|
||||
}
|
||||
leaf b {
|
||||
type int32;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
cat <<EOF > $cfg
|
||||
<clixon-config xmlns="http://clicon.org/config">
|
||||
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
|
||||
<CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
|
||||
<CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
|
||||
<CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
|
||||
<CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
|
||||
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
|
||||
<CLICON_BACKEND_PIDFILE>$pidfile</CLICON_BACKEND_PIDFILE>
|
||||
<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
|
||||
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
|
||||
<CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
|
||||
<CLICON_CLI_MODE>example</CLICON_CLI_MODE>
|
||||
<CLICON_CLI_DIR>/usr/local/lib/example/cli</CLICON_CLI_DIR>
|
||||
<CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
|
||||
<CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
|
||||
<CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
|
||||
</clixon-config>
|
||||
EOF
|
||||
|
||||
testrun(){
|
||||
nr=$1
|
||||
|
||||
new "generate config with $nr list entries"
|
||||
echo -n "<config><x xmlns=\"urn:example:clixon\">" > $dir/startup_db
|
||||
for (( i=0; i<$nr; i++ )); do
|
||||
echo -n "<y><a>$i</a><b>$i</b></y>" >> $dir/startup_db
|
||||
done
|
||||
echo "</x></config>" >> $dir/startup_db
|
||||
|
||||
new "test params: -f $cfg"
|
||||
if [ $BE -ne 0 ]; then
|
||||
new "kill old backend"
|
||||
sudo clixon_backend -zf $cfg
|
||||
if [ $? -ne 0 ]; then
|
||||
err
|
||||
fi
|
||||
new "start backend -s startup -f $cfg"
|
||||
start_backend -s startup -f $cfg
|
||||
fi
|
||||
|
||||
new "waiting"
|
||||
wait_backend
|
||||
|
||||
pid=$(cat $pidfile)
|
||||
|
||||
new "netconf get state"
|
||||
res=$(echo "<rpc><get><filter type=\"xpath\" select=\"/cc:clixon-stats\" xmlns:cc=\"http://clicon.org/config\"/></get></rpc>]]>]]>" | $clixon_netconf -qf $cfg)
|
||||
echo "Total"
|
||||
echo -n " objects: "
|
||||
echo $res | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/global/xmlnr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
|
||||
echo -n " mem: "
|
||||
# This ony works on Linux
|
||||
cat /proc/$pid/statm|awk '{print $1*4/1000 "M"}'
|
||||
for db in running candidate startup; do
|
||||
echo "$db"
|
||||
resdb=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/datastore[name=\"$db\"]")
|
||||
resdb=${resdb#"nodeset:0:"}
|
||||
# echo "resdb:$resdb"
|
||||
echo -n " objects: "
|
||||
echo $resdb | $clixon_util_xpath -p "datastore/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
|
||||
echo -n " mem: "
|
||||
echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
|
||||
done
|
||||
|
||||
if [ $BE -ne 0 ]; then
|
||||
new "Kill backend"
|
||||
# Check if premature kill
|
||||
pid=$(pgrep -u root -f clixon_backend)
|
||||
if [ -z "$pid" ]; then
|
||||
err "backend already dead"
|
||||
fi
|
||||
# kill backend
|
||||
stop_backend -f $cfg
|
||||
fi
|
||||
}
|
||||
|
||||
new "Memory test for backend with 1 $perfnr entries"
|
||||
testrun $perfnr
|
||||
|
||||
rm -rf $dir
|
||||
|
||||
# unset conditional parameters
|
||||
unset perfnr
|
||||
|
||||
|
|
@ -160,6 +160,6 @@ rm -rf $dir
|
|||
# unset conditional parameters
|
||||
unset format
|
||||
unset perfnr
|
||||
unset perfreg
|
||||
unset perfreq
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -324,23 +324,8 @@ main(int argc,
|
|||
}
|
||||
else
|
||||
x = x0;
|
||||
#ifdef XPATH_LIST_OPTIMIZE /* Experimental */
|
||||
{
|
||||
int hits = 0;
|
||||
int j;
|
||||
|
||||
xpath_list_optimize_stats(&hits);
|
||||
for (j=0;j<1;j++){
|
||||
if (xpath_vec_ctx(x, nsc, xpath, 0, &xc) < 0)
|
||||
return -1;
|
||||
}
|
||||
xpath_list_optimize_stats(&hits);
|
||||
fprintf(stderr, "hits after:%d\n", hits);
|
||||
}
|
||||
#else
|
||||
if (xpath_vec_ctx(x, nsc, xpath, 0, &xc) < 0)
|
||||
return -1;
|
||||
#endif
|
||||
/* Print results */
|
||||
cb = cbuf_new();
|
||||
ctx_print2(cb, xc);
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ module clixon-config {
|
|||
***** END LICENSE BLOCK *****";
|
||||
|
||||
revision 2020-02-22 {
|
||||
description "Added search index extension";
|
||||
description
|
||||
"Added: search index extension,
|
||||
Added: clixon-stats state for clixon XML and memory statistics.";
|
||||
}
|
||||
revision 2019-09-11 {
|
||||
description
|
||||
|
|
@ -673,12 +675,11 @@ module clixon-config {
|
|||
description "Clixon backend statistics.";
|
||||
container global{
|
||||
description "Clixon global statistics";
|
||||
leaf nr{
|
||||
description "Number of cxobj:ects. That is number of residing xml/json objects
|
||||
leaf xmlnr{
|
||||
description "Number of XML objects. That is number of residing xml/json objects
|
||||
in the internal 'cxobj' representation.";
|
||||
type uint64;
|
||||
}
|
||||
|
||||
}
|
||||
list datastore{
|
||||
description "Datastore statistics";
|
||||
|
|
@ -688,7 +689,8 @@ module clixon-config {
|
|||
type string;
|
||||
}
|
||||
leaf nr{
|
||||
description "Number bytes of internal datastore cache of datastore tree.";
|
||||
description "Number of XML objects. That is number of residing xml/json objects
|
||||
in the internal 'cxobj' representation.";
|
||||
type uint64;
|
||||
}
|
||||
leaf size{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue