experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* Restructured and extended <stats> rpc to schema mountpoints

Browse source 
* New revision of clixon-lib.yang
* CLI: Added `show statistics` example code for backend and CLI memory stats
This commit is contained in:
Olof hagsand 2023-05-05 17:43:03 +02:00
parent a0df12134a
commit b6116261a3
11 changed files with 285 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

7
CHANGELOG.md
View file

@ -43,8 +43,15 @@
## 6.3.0
Expected: July 2023
### API changes on existing protocol/config features
* New `clixon-lib@2023-05-01.yang` revision
* Restructured and extended stats rpc to schema mountpoints
* rpc `<stats>` is not backward compatible
### Minor features
* CLI: Added `show statistics` example code for backend and CLI memory stats
* [Support yang type union with are same subtypes with SNMP](https://github.com/clicon/clixon/pull/427)
* Removed obsolete compile options introduced in 6.1:
* `NETCONF_DEFAULT_RETRIEVAL_REPORT_ALL`

61
apps/backend/backend_client.c
View file

@ -334,20 +334,18 @@ clixon_stats_datastore_get(clicon_handle h,
if ((xt = xmldb_cache_get(h, dbname)) == NULL){
/* Trigger cache if no exist */
if (xmldb_get(h, dbname, NULL, "/", &xn) < 0)
goto done;
//goto done;
goto ok;
xt = xmldb_cache_get(h, dbname);
}
if (xt == NULL){
cprintf(cb, "<datastore xmlns=\"%s\"><name>%s</name><nr>0</nr><size>0</size></datastore>",
CLIXON_LIB_NS, dbname);
}
else{
if (xt != NULL){
if (xml_stats(xt, &nr, &sz) < 0)
goto done;
cprintf(cb, "<datastore xmlns=\"%s\"><name>%s</name><nr>%" PRIu64 "</nr>"
cprintf(cb, "<datastore><name>%s</name><nr>%" PRIu64 "</nr>"
"<size>%zu</size></datastore>",
CLIXON_LIB_NS, dbname, nr, sz);
dbname, nr, sz);
}
ok:
retval = 0;
done:
if (xn)
@ -377,9 +375,7 @@ clixon_stats_module_get(clicon_handle h,
return 0;
if (yang_stats(ys, &nr, &sz) < 0)
goto done;
cprintf(cb, "<module xmlns=\"%s\"><name>%s</name><nr>%" PRIu64 "</nr>"
"<size>%zu</size></module>",
CLIXON_LIB_NS, yang_argument_get(ys), nr, sz);
cprintf(cb, "<nr>%" PRIu64 "</nr><size>%zu</size>", nr, sz);
retval = 0;
done:
if (xn)
@ -1308,9 +1304,15 @@ from_client_stats(clicon_handle h,
int retval = -1;
uint64_t nr;
yang_stmt *ym;
char *str;
int modules = 0;
yang_stmt *yspec;
yang_stmt *ymodext;
cxobj *xt;
if ((str = xml_find_body(xe, "modules")) != NULL)
modules = strcmp(str, "true") == 0;
cprintf(cbret, "<rpc-reply xmlns=\"%s\">", NETCONF_BASE_NAMESPACE);
xml_stats_global(&nr);
cprintf(cbret, "<global xmlns=\"%s\">", CLIXON_LIB_NS);
nr=0;
xml_stats_global(&nr);
@ -1319,27 +1321,52 @@ from_client_stats(clicon_handle h,
yang_stats_global(&nr);
cprintf(cbret, "<yangnr>%" PRIu64 "</yangnr>", nr);
cprintf(cbret, "</global>");
cprintf(cbret, "<datastores xmlns=\"%s\">", CLIXON_LIB_NS);
if (clixon_stats_datastore_get(h, "running", cbret) < 0)
goto done;
if (clixon_stats_datastore_get(h, "candidate", cbret) < 0)
goto done;
if (clixon_stats_datastore_get(h, "startup", cbret) < 0)
goto done;
cprintf(cbret, "</datastores>");
/* per module-set, first configuration, then main dbspec, then mountpoints */
cprintf(cbret, "<module-sets xmlns=\"%s\">", CLIXON_LIB_NS);
cprintf(cbret, "<module-set><name>clixon-config</name>");
yspec = clicon_config_yang(h);
if (clixon_stats_module_get(h, yspec, cbret) < 0)
goto done;
if (modules){
ym = NULL;
while ((ym = yn_each(clicon_config_yang(h), ym)) != NULL) {
while ((ym = yn_each(yspec, ym)) != NULL) {
cprintf(cbret, "<module><name>%s</name>", yang_argument_get(ym));
if (clixon_stats_module_get(h, ym, cbret) < 0)
goto done;
cprintf(cbret, "</module>");
}
}
cprintf(cbret, "</module-set>");
cprintf(cbret, "<module-set><name>main</name>");
yspec = clicon_dbspec_yang(h);
if (clixon_stats_module_get(h, yspec, cbret) < 0)
goto done;
if (modules){
ym = NULL;
while ((ym = yn_each(clicon_dbspec_yang(h), ym)) != NULL) {
while ((ym = yn_each(yspec, ym)) != NULL) {
cprintf(cbret, "<module><name>%s</name>", yang_argument_get(ym));
if (clixon_stats_module_get(h, ym, cbret) < 0)
goto done;
cprintf(cbret, "</module>");
}
ym = NULL;
while ((ym = yn_each(clicon_nacm_ext_yang(h), ym)) != NULL) {
if (clixon_stats_module_get(h, ym, cbret) < 0)
}
cprintf(cbret, "</module-set>");
/* Mountpoints */
if ((ymodext = yang_find(yspec, Y_MODULE, "ietf-yang-schema-mount")) != NULL){
if (xmldb_get(h, "running", NULL, "/", &xt) < 0)
goto done;
if (xt && yang_schema_mount_statistics(h, xt, modules, cbret) < 0)
goto done;
}
cprintf(cbret, "</module-sets>");
cprintf(cbret, "</rpc-reply>");
retval = 0;
done:

71
apps/cli/cli_show.c
View file

@ -53,6 +53,7 @@
#include <dirent.h>
#include <syslog.h>
#include <pwd.h>
#include <inttypes.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
@ -1487,3 +1488,73 @@ clixon_cli2file(clicon_handle h,
done:
return retval;
}
/*! CLI callback show statistics
*/
int
cli_show_statistics(clicon_handle h,
cvec *cvv,
cvec *argv)
{
int retval = -1;
cbuf *cb = NULL;
cxobj *xret = NULL;
cxobj *xerr;
cg_var *cv;
int modules = 0;
pt_head *ph;
parse_tree *pt;
uint64_t nr = 0;
size_t sz = 0;
if (argv != NULL && cvec_len(argv) != 1){
clicon_err(OE_PLUGIN, EINVAL, "Expected arguments: [modules]");
goto done;
}
if (argv){
cv = cvec_i(argv, 0);
modules = (strcmp(cv_string_get(cv), "modules") == 0);
}
if ((cb = cbuf_new()) == NULL){
clicon_err(OE_PLUGIN, errno, "cbuf_new");
goto done;
}
/* CLI */
cligen_output(stdout, "CLI:\n");
ph = NULL;
while ((ph = cligen_ph_each(cli_cligen(h), ph)) != NULL) {
if ((pt = cligen_ph_parsetree_get(ph)) == NULL)
continue;
nr = 0; sz = 0;
pt_stats(pt, &nr, &sz);
cligen_output(stdout, "%s: nr=%" PRIu64 " size:%zu\n",
cligen_ph_name_get(ph), nr, sz);
}
/* Backend */
cprintf(cb, "<rpc xmlns=\"%s\"", NETCONF_BASE_NAMESPACE);
cprintf(cb, " %s", NETCONF_MESSAGE_ID_ATTR); /* XXX: use incrementing sequence */
cprintf(cb, ">");
cprintf(cb, "<stats xmlns=\"%s\">", CLIXON_LIB_NS);
if (modules)
cprintf(cb, "<modules>true</modules>");
cprintf(cb, "</stats>");
cprintf(cb, "</rpc>");
if (clicon_rpc_netconf(h, cbuf_get(cb), &xret, NULL) < 0)
goto done;
if ((xerr = xpath_first(xret, NULL, "//rpc-error")) != NULL){
clixon_netconf_error(xerr, "Get configuration", NULL);
goto done;
}
fprintf(stdout, "Backend:\n");
if (clixon_xml2file(stdout, xml_child_i(xret, 0), 0, 1, NULL, cligen_output, 0, 1) < 0)
goto done;
fprintf(stdout, "CLI:\n");
retval = 0;
done:
if (xret)
xml_free(xret);
if (cb)
cbuf_free(cb);
return retval;
}

4
example/main/example_cli.cli
View file

@ -130,6 +130,10 @@ show("Show a particular state of the system"){
yang("Show yang specs"), show_yang(); {
clixon-example("Show clixon-example yang spec"), show_yang("clixon-example");
}
statistics("Show statistics"), cli_show_statistics();{
brief, cli_show_statistics();
modules, cli_show_statistics("modules");
}
}
save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename", "xml");{

1
lib/clixon/clixon_yang_schema_mount.h
View file

@ -60,6 +60,7 @@ int xml_yang_mount_get(clicon_handle h, cxobj *x, validate_level *vl, yang_stmt
int xml_yang_mount_set(cxobj *x, yang_stmt *yspec);
int xml_yang_mount_freeall(cvec *cvv);
int yang_schema_mount_statedata(clicon_handle h, yang_stmt *yspec, char *xpath, cvec *nsc, cxobj **xret, cxobj **xerr);
int yang_schema_mount_statistics(clicon_handle h, cxobj *xt, int modules, cbuf *cb);
int yang_schema_yanglib_parse_mount(clicon_handle h, cxobj *xt);
int yang_schema_get_child(clicon_handle h, cxobj *x1, cxobj *x1c, yang_stmt **yc);

72
lib/src/clixon_yang_schema_mount.c
View file

@ -58,6 +58,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <inttypes.h>
#include <limits.h>
#include <string.h>
#include <sys/param.h>
@ -209,7 +210,7 @@ yang_mount_set(yang_stmt *yu,
*
* @param[in] h Clixon handle
* @param[in] x XML moint-point node
* @param[out] vallevel Do or dont do full RFC 7950 validation
* @param[out] vallevel Do or dont do full RFC 7950 validation if given
* @param[out] yspec YANG stmt spec
* @retval 1 x is a mount-point: yspec may be set
* @retval 0 x is not a mount point
@ -505,6 +506,75 @@ yang_schema_mount_statedata(clicon_handle h,
goto done;
}
/*! Statistics about mountpoints
* @see yang_schema_mount_statedata
*/
int
yang_schema_mount_statistics(clicon_handle h,
cxobj *xt,
int modules,
cbuf *cb)
{
int retval = -1;
cvec *cvv = NULL;
cg_var *cv;
cxobj *xmp; /* xml mount-point */
yang_stmt *yspec;
yang_stmt *ym;
int ret;
char *xpath;
uint64_t nr;
size_t sz;
if ((cvv = cvec_new(0)) == NULL){
clicon_err(OE_UNIX, errno, "cvec_new");
goto done;
}
if (xml_apply(xt, CX_ELMNT, find_schema_mounts, cvv) < 0)
goto done;
cv = NULL;
while ((cv = cvec_each(cvv, cv)) != NULL) {
if ((xmp = cv_void_get(cv)) == NULL)
continue;
if ((ret = xml_yang_mount_get(h, xmp, NULL, &yspec)) < 0)
goto done;
if (ret == 0)
continue;
if (xml2xpath(xmp, NULL, 1, 0, &xpath) < 0)
goto done;
cprintf(cb, "<module-set><name>mountpoint: ");
xml_chardata_cbuf_append(cb, xpath);
cprintf(cb, "</name>");
nr = 0; sz = 0;
if (yang_stats(yspec, &nr, &sz) < 0)
goto done;
cprintf(cb, "<nr>%" PRIu64 "</nr><size>%zu</size>", nr, sz);
if (modules){
ym = NULL;
while ((ym = yn_each(yspec, ym)) != NULL) {
cprintf(cb, "<module><name>%s</name>", yang_argument_get(ym));
nr = 0; sz = 0;
if (yang_stats(ym, &nr, &sz) < 0)
goto done;
cprintf(cb, "<nr>%" PRIu64 "</nr><size>%zu</size>", nr, sz);
cprintf(cb, "</module>");
}
}
cprintf(cb, "</module-set>");
if (xpath){
free(xpath);
xpath = NULL;
}
}
retval = 0;
done:
if (xpath)
free(xpath);
if (cvv)
cvec_free(cvv);
return retval;
}
/*! Get yanglib from user plugin callback, parse it and mount it
*
* @param[in] h Clixon handle

2
test/config.sh.in
View file

@ -73,7 +73,7 @@ DATASTORE_TOP="config"
# clixon yang revisions occuring in tests (see eg yang/clixon/Makefile.in)
CLIXON_AUTOCLI_REV="2022-02-11"
CLIXON_LIB_REV="2023-03-01"
CLIXON_LIB_REV="2023-05-01"
CLIXON_CONFIG_REV="2023-03-01"
CLIXON_RESTCONF_REV="2022-08-01"
CLIXON_EXAMPLE_REV="2022-11-01"

13
test/test_perf_mem.sh
View file

@ -91,7 +91,7 @@ function testrun(){
pid=$(cat $pidfile)
new "netconf get stats"
rpc=$(chunked_framing "<rpc $DEFAULTNS><stats $LIBNS/></rpc>")
rpc=$(chunked_framing "<rpc $DEFAULTNS><stats $LIBNS><modules>true</modules></stats></rpc>")
res=$(echo "$DEFAULTHELLO$rpc" | $clixon_netconf -qef $cfg)
# echo "res:$res"
err0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/rpc-error")
@ -112,7 +112,7 @@ function testrun(){
fi
for db in running candidate startup; do
echo "$db"
resdb0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastore[name=\"$db\"]")
resdb0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastores/datastore[name=\"$db\"]")
resdb=${resdb0#"nodeset:0:"}
if [ "$resdb0" = "$resdb" ]; then
err1 "nodeset:0:" "$resdb0"
@ -122,15 +122,6 @@ function testrun(){
echo -n " mem: "
echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
done
for mod in clixon-config; do
echo "$mod"
resmod0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/module[name=\"$mod\"]")
resmod=${resmod0#"nodeset:0:"}
echo -n " objects: "
echo $resmod | $clixon_util_xpath -p "module/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
echo -n " mem: "
echo $resmod | $clixon_util_xpath -p "module/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
done
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill

4
test/test_yang_schema_mount.sh
View file

@ -83,6 +83,10 @@ expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS>
new "get yang-lib at mountpoint"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"subtree\"><top xmlns=\"urn:example:clixon\"><mylist/></top>></filter></get></rpc>" "<rpc-reply $DEFAULTNS><data><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><yang-library xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"><module-set><name>mount</name><module><name>clixon-example</name><revision>2022-11-01</revision><namespace>urn:example:urn</namespace></module></module-set></yang-library></root></mylist><mylist><name>y</name><root><yang-library xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"><module-set><name>mount</name><module><name>clixon-example</name><revision>2022-11-01</revision><namespace>urn:example:urn</namespace></module></module-set></yang-library></root></mylist></top></data></rpc-reply>"
new "check there is statistics from mountpoint"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><stats xmlns=\"http://clicon.org/lib\"></stats></rpc>" '<module-set><name>mountpoint: /top/mylist\[name="x"\]/root</name><nr>'
#"<rpc-reply $DEFAULTNS></rpc-reply>"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill

2
yang/clixon/Makefile.in
View file

@ -43,7 +43,7 @@ YANG_INSTALLDIR = @YANG_INSTALLDIR@
# Note: mirror these to test/config.sh.in
YANGSPECS = clixon-config@2023-03-01.yang # 6.2
YANGSPECS += clixon-lib@2023-03-01.yang # 6.2
YANGSPECS += clixon-lib@2023-05-01.yang # 6.3
YANGSPECS += clixon-rfc5277@2008-07-01.yang
YANGSPECS += clixon-xml-changelog@2019-03-21.yang
YANGSPECS += clixon-restconf@2022-08-01.yang # 5.9

64
yang/clixon/clixon-lib@2022-12-01.yang → yang/clixon/clixon-lib@2023-05-01.yang
View file

@ -9,6 +9,9 @@ module clixon-lib {
import ietf-netconf-monitoring {
prefix ncm;
}
import ietf-yang-metadata {
prefix "md";
}
organization
"Clicon / Clixon";
@ -18,7 +21,7 @@ module clixon-lib {
description
"***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate)
This file is part of CLIXON
@ -51,7 +54,7 @@ module clixon-lib {
- RPCs for debug, stats and process-control
- Informal description of attributes
Additionally, Clixon extends NETCONF for internal use with some internal attributes. These
Clixon also extends NETCONF for internal use with some internal attributes. These
are not visible for external usage bit belongs to the namespace of this YANG.
The internal attributes are:
- content (also RESTCONF)
@ -65,6 +68,14 @@ module clixon-lib {
- objectexisted
";
revision 2023-05-01 {
description
"Restructured and extended stats rpc to schema mountpoints";
}
revision 2023-03-01 {
description
"Added creator meta-object";
}
revision 2022-12-01 {
description
"Added values of RFC6022 transport identityref
@ -138,7 +149,7 @@ module clixon-lib {
}
identity netconf {
description
"Just NETCONF without specitic underlying transport,
"Just NETCONF without specific underlying transport,
Clixon uses stdio for its netconf client and therefore does not know whether it is
invoked in a script, by a NETCONF/SSH subsystem, etc";
base ncm:transport;
@ -166,6 +177,17 @@ module clixon-lib {
argument cliop;
status obsolete;
}
md:annotation creator {
type string;
description
"This annotation contains the name of a creator of an object.
One application is the clixon controller where multiple services can
create the same object. When such a service is deleted (or changed) one needs to keep
track of which service created what.
Limitations: only objects that are actually added or deleted.
A sub-object wil not be noted";
}
rpc debug {
description "Set debug level of backend.";
input {
@ -177,8 +199,15 @@ module clixon-lib {
rpc ping {
description "Check aliveness of backend daemon.";
}
rpc stats {
description "Clixon XML statistics.";
rpc stats { /* Could be moved to state */
description "Clixon yang and datastore statistics.";
input {
leaf modules {
description "If enabled include per-module statistics";
type boolean;
mandatory false;
}
}
output {
container global{
description
@ -198,6 +227,7 @@ module clixon-lib {
type uint64;
}
}
container datastores{
list datastore{
description "Per datastore statistics for cxobj";
key "name";
@ -215,8 +245,27 @@ module clixon-lib {
type uint64;
}
}
}
container module-sets{
list module-set{
description "Statistics per group of module, eg top-level and mount-points";
key "name";
leaf name{
description "Name of YANG module.";
type string;
}
leaf nr{
description
"Total number of YANG objects in set";
type uint64;
}
leaf size{
description
"Total size in bytes of internal YANG object representation for module set";
type uint64;
}
list module{
description "Per YANG module statistics";
description "Statistics per module (if modules set in input)";
key "name";
leaf name{
description "Name of YANG module.";
@ -235,6 +284,8 @@ module clixon-lib {
}
}
}
}
}
rpc restart-plugin {
description "Restart specific backend plugins.";
input {
@ -244,7 +295,6 @@ module clixon-lib {
}
}
}
rpc process-control {
description
"Control a specific process or daemon: start/stop, etc.