experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* Added clixon-stats in clixon-config.yang for clixon XML and memory statistics.

Browse source
This commit is contained in:
Olof hagsand 2020-03-01 10:59:24 +01:00
parent f1ceec689a
commit 2d521d52c8
12 changed files with 253 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

10
CHANGELOG.md
View file

@ -21,11 +21,11 @@
* [3.3.1](#331) June 7 2017
## 4.4.0
Expected: February 2020
Expected: Early March 2020
### Major New features
* New "general-purpose" datastore upgrade callback added which i called once on startup, intended for low-level general upgrades and as a complement to module-specific upgrade.
* New "general-purpose" datastore upgrade callback called once on startup, intended for low-level general upgrades and as a complement to module-specific upgrade.
* Called on startup after initial XML parsing, but before module-specific upgrades
* Enabled by definign the `.ca_datastore_upgrade`
* [General-purpose upgrade documentation](https://clixon-docs.readthedocs.io/en/latest/backend.html#general-purpose)
@ -39,8 +39,10 @@ Expected: February 2020
[search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml)
### API changes on existing features (you may need to change your code)
* Bugfix of config false statement may cause change of sorting of lists in GET opertions (lists that were sorted should not have been sorted)
* New clixon-config@2020-02-22.yang revision
* Added search index extension
* Search index extension `search_index` for declaring which non-key variables are search indexes
* Added `clixon-stats` for clixon XML and memory statistics.
* JSON parse error messages change from ` on line x: syntax error,..` to `json_parse: line x: syntax error`
* Unknown-element error message is more descriptive, eg from `namespace is: urn:example:clixon` to: `Failed to find YANG spec of XML node: x with parent: xp in namespace urn:example:clixon`.
* C-API parse and validation API more capable
@ -76,7 +78,7 @@ Expected: February 2020
### Corrected Bugs
* Fixed: Search function checked only own not for config false statement, should have checked all ancestors.
* Fixed: Search function checked only own not for config false statement, should have checked all ancestors. This may affect some state returned in GET calls
* Fixed: Some restconf errors were wrongly formatted such as: `{"ietf-restconf:errors":{"error":{"rpc-error":` . There should be no `"rpc-error"` level.
* Fixed: Enabling modstate (CLICON_XMLDB_MODSTATE), changing a revision on a yang, and restarting made the backend daemon exit at start (thanks Matt)
* Also: ensure to load `ietf-yang-library.yang ` if CLICON_XMLDB_MODSTATE is set

3
LICENSE.md
View file

@ -1,5 +1,6 @@
Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
Copyright (C) 2017-2020 Olof Hagsand
Copyright (C) 2017-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
CLIXON is dual license.

84
apps/backend/backend_client.c
View file

@ -44,6 +44,7 @@
#include <stdlib.h>
#include <unistd.h>
#include <stdarg.h>
#include <inttypes.h>
#include <errno.h>
#include <signal.h>
#include <fcntl.h>
@ -256,6 +257,84 @@ client_get_streams(clicon_handle h,
goto done;
}
/*! Get clixon per datastore stats
* @param[in] h Clicon handle
* @param[in] dbname Datastore name
* @param[in,out] cb Cligen buf
* @retval 0 OK
* @retval -1 Error
*/
static int
clixon_stats_get_db(clicon_handle h,
char *name,
cbuf *cb)
{
int retval = -1;
cxobj *xt = NULL;
uint64_t nr = 0;
size_t sz = 0;
if (xmldb_get(h, "running", NULL, NULL, &xt) < 0)
goto done;
xml_stats(xt, &nr, &sz);
cprintf(cb, "<datastore><name>%s</name><nr>%" PRIu64 "</nr>"
"<size>%" PRIu64 "</size></datastore>",
name, nr, sz);
retval = 0;
done:
if (xt)
free(xt);
return retval;
}
/*! Get clixon stats
* @param[in] h Clicon handle
* @param[in] yspec Yang spec
* @param[in] xpath XML Xpath
* @param[in] nsc XML Namespace context for xpath
* @param[in,out] xret Existing XML tree, merge x into this
* @retval 0 OK
* @retval -1 Error
*/
int
clixon_stats_get(clicon_handle h,
yang_stmt *yspec,
char *xpath,
cvec *nsc,
cxobj **xret)
{
int retval = -1;
cbuf *cb = NULL;
uint64_t nr;
int ret;
if ((cb = cbuf_new()) == NULL){
clicon_err(OE_UNIX, errno, "cbuf_new");
goto done;
}
cprintf(cb, "<clixon-stats xmlns=\"%s\">", CLIXON_CONF_NS);
nr=0;
xml_stats_global(&nr);
cprintf(cb, "<global><xmlnr>%" PRIu64 "</xmlnr></global>", nr);
clixon_stats_get_db(h, "running", cb);
clixon_stats_get_db(h, "candidate", cb);
clixon_stats_get_db(h, "startup", cb);
cprintf(cb, "</clixon-stats>");
if ((ret = xml_parse_string2(cbuf_get(cb), YB_TOP, yspec, xret, NULL)) < 0)
goto done;
if (ret == 0){
clicon_err(OE_XML, EINVAL, "Internal error");
goto done;
}
retval = 0;
done:
clicon_debug(1, "%s %d", __FUNCTION__, retval);
if (cb)
cbuf_free(cb);
return retval;
}
/*! Get system state-data, including streams and plugins
* @param[in] h Clicon handle
* @param[in] xpath Xpath selection, not used but may be to filter early
@ -331,6 +410,11 @@ client_statedata(clicon_handle h,
if (ret == 0)
goto fail;
}
/* Clixon-config has a state data, if yang is present */
if (yang_find(yspec, Y_MODULE, "clixon-config") != NULL){
if (clixon_stats_get(h, yspec, xpath, nsc, xret) < 0)
goto done;
}
if ((ret = clixon_plugin_statedata(h, yspec, nsc, xpath, xret)) < 0)
goto done;
if (ret == 0)

13
lib/clixon/clixon_options.h
View file

@ -2,7 +2,9 @@
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
Copyright (C) 2017-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
This file is part of CLIXON.
@ -38,6 +40,15 @@
#ifndef _CLIXON_OPTIONS_H_
#define _CLIXON_OPTIONS_H_
/*
* Constants
*/
/*! Clixon configuration namespace
* Probably should be defined somewhere else or extracted from yang
* @see clixon-config.yang
*/
#define CLIXON_CONF_NS "http://clicon.org/config"
/*
* Types
*/

6
lib/src/clixon_options.c
View file

@ -80,12 +80,6 @@
#include "clixon_validate.h"
#include "clixon_xml_map.h"
/*! Clixon configuration namespace
* Probably should be defined somewhere else or extracted from yang
* @see clixon-config.yang
*/
#define CLIXON_CONF_NS "http://clicon.org/config"
/* Mapping between Cli generation from Yang string <--> constants,
see clixon-config.yang type cli_genmodel_type */
static const map_str2int cli_genmodel_map[] = {

13
lib/src/clixon_xml.c
View file

@ -237,8 +237,19 @@ xml_stats_one(cxobj *x,
sz += cv_size(x->x_cv);
if (x->x_ns_cache)
sz += cvec_size(x->x_ns_cache);
#ifdef XML_EXPLICIT_INDEX
if (x->x_search_index){
/* XXX: only one */
sz += sizeof(struct search_index);
if (x->x_search_index->si_name)
sz += strlen(x->x_search_index->si_name)+1;
if (x->x_search_index->si_xvec)
sz += clixon_xvec_len(x->x_search_index->si_xvec)*sizeof(struct cxobj*);
}
#endif
if (szp)
*szp = sz;
clicon_debug(1, "%s %" PRIu64, __FUNCTION__, sz);
return 0;
}
@ -261,10 +272,12 @@ xml_stats(cxobj *xt,
*szp += sz;
xc = NULL;
while ((xc = xml_child_each(xt, xc, -1)) != NULL) {
sz=0;
xml_stats(xc, nrp, &sz);
if (szp)
*szp += sz;
}
clicon_debug(1, "%s %" PRIu64, __FUNCTION__, *szp);
return 0;
}

3
lib/src/clixon_xpath_optimize.c
View file

@ -2,7 +2,8 @@
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2020 Olof Hagsand
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
This file is part of CLIXON.

2
test/test_perf.sh
View file

@ -267,4 +267,4 @@ rm -rf $dir
# unset conditional parameters
unset format
unset perfnr
unset perfreg
unset perfreq

125
test/test_perf_mem.sh Executable file
View file

@ -0,0 +1,125 @@
#!/usr/bin/env bash
# Backend Memory tests, footprint using the clixon-conf state statistics
# Create a large datastore, load it and measure
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
clixon_util_xpath=clixon_util_xpath
# Number of list/leaf-list entries in file
: ${perfnr:=10000}
APPNAME=example
cfg=$dir/scaling-conf.xml
fyang=$dir/scaling.yang
pidfile=$dir/pidfile
cat <<EOF > $fyang
module scaling{
yang-version 1.1;
namespace "urn:example:clixon";
prefix ex;
import "clixon-config" {
prefix cc;
}
container x {
list y {
key "a";
leaf a {
type int32;
}
leaf b {
type int32;
}
}
}
}
EOF
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
<CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
<CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
<CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_PIDFILE>$pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
<CLICON_CLI_MODE>example</CLICON_CLI_MODE>
<CLICON_CLI_DIR>/usr/local/lib/example/cli</CLICON_CLI_DIR>
<CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
<CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
<CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
</clixon-config>
EOF
testrun(){
nr=$1
new "generate config with $nr list entries"
echo -n "<config><x xmlns=\"urn:example:clixon\">" > $dir/startup_db
for (( i=0; i<$nr; i++ )); do
echo -n "<y><a>$i</a><b>$i</b></y>" >> $dir/startup_db
done
echo "</x></config>" >> $dir/startup_db
new "test params: -f $cfg"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s startup -f $cfg"
start_backend -s startup -f $cfg
fi
new "waiting"
wait_backend
pid=$(cat $pidfile)
new "netconf get state"
res=$(echo "<rpc><get><filter type=\"xpath\" select=\"/cc:clixon-stats\" xmlns:cc=\"http://clicon.org/config\"/></get></rpc>]]>]]>" | $clixon_netconf -qf $cfg)
echo "Total"
echo -n " objects: "
echo $res | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/global/xmlnr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
echo -n " mem: "
# This ony works on Linux
cat /proc/$pid/statm|awk '{print $1*4/1000 "M"}'
for db in running candidate startup; do
echo "$db"
resdb=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/data/clixon-stats/datastore[name=\"$db\"]")
resdb=${resdb#"nodeset:0:"}
# echo "resdb:$resdb"
echo -n " objects: "
echo $resdb | $clixon_util_xpath -p "datastore/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
echo -n " mem: "
echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
done
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
}
new "Memory test for backend with 1 $perfnr entries"
testrun $perfnr
rm -rf $dir
# unset conditional parameters
unset perfnr

2
test/test_perf_state.sh
View file

@ -160,6 +160,6 @@ rm -rf $dir
# unset conditional parameters
unset format
unset perfnr
unset perfreg
unset perfreq

15
util/clixon_util_xpath.c
View file

@ -324,23 +324,8 @@ main(int argc,
}
else
x = x0;
#ifdef XPATH_LIST_OPTIMIZE /* Experimental */
{
int hits = 0;
int j;
xpath_list_optimize_stats(&hits);
for (j=0;j<1;j++){
if (xpath_vec_ctx(x, nsc, xpath, 0, &xc) < 0)
return -1;
}
xpath_list_optimize_stats(&hits);
fprintf(stderr, "hits after:%d\n", hits);
}
#else
if (xpath_vec_ctx(x, nsc, xpath, 0, &xc) < 0)
return -1;
#endif
/* Print results */
cb = cbuf_new();
ctx_print2(cb, xc);

12
yang/clixon/clixon-config@2020-02-22.yang
View file

@ -41,7 +41,9 @@ module clixon-config {
***** END LICENSE BLOCK *****";
revision 2020-02-22 {
description "Added search index extension";
description
"Added: search index extension,
Added: clixon-stats state for clixon XML and memory statistics.";
}
revision 2019-09-11 {
description
@ -673,12 +675,11 @@ module clixon-config {
description "Clixon backend statistics.";
container global{
description "Clixon global statistics";
leaf nr{
description "Number of cxobj:ects. That is number of residing xml/json objects
leaf xmlnr{
description "Number of XML objects. That is number of residing xml/json objects
in the internal 'cxobj' representation.";
type uint64;
}
}
list datastore{
description "Datastore statistics";
@ -688,7 +689,8 @@ module clixon-config {
type string;
}
leaf nr{
description "Number bytes of internal datastore cache of datastore tree.";
description "Number of XML objects. That is number of residing xml/json objects
in the internal 'cxobj' representation.";
type uint64;
}
leaf size{