* Restructured and extended <stats> rpc to schema mountpoints

* New revision of clixon-lib.yang
* CLI: Added `show statistics` example code for backend and CLI memory stats

CHANGELOG.md

@@ -43,8 +43,15 @@
 ## 6.3.0
 Expected: July 2023
 
+### API changes on existing protocol/config features
+
+* New `clixon-lib@2023-05-01.yang` revision
+  * Restructured and extended stats rpc to schema mountpoints
+  * rpc `<stats>` is not backward compatible
+  
 ### Minor features
 
+* CLI: Added `show statistics` example code for backend and CLI memory stats
 * [Support yang type union with are same subtypes with SNMP](https://github.com/clicon/clixon/pull/427)
 * Removed obsolete compile options introduced in 6.1:
   * `NETCONF_DEFAULT_RETRIEVAL_REPORT_ALL`

apps/backend/backend_client.c

@@ -334,20 +334,18 @@ clixon_stats_datastore_get(clicon_handle h,
     if ((xt = xmldb_cache_get(h, dbname)) == NULL){
         /* Trigger cache if no exist */
         if (xmldb_get(h, dbname, NULL, "/", &xn) < 0)
-            goto done;
+            //goto done;
+            goto ok;
         xt = xmldb_cache_get(h, dbname);
     }
-    if (xt == NULL){
+    if (xt != NULL){
-        cprintf(cb, "<datastore xmlns=\"%s\"><name>%s</name><nr>0</nr><size>0</size></datastore>",
-                CLIXON_LIB_NS, dbname);
-    }
-    else{
         if (xml_stats(xt, &nr, &sz) < 0)
             goto done;
-        cprintf(cb, "<datastore xmlns=\"%s\"><name>%s</name><nr>%" PRIu64 "</nr>"
+        cprintf(cb, "<datastore><name>%s</name><nr>%" PRIu64 "</nr>"
                 "<size>%zu</size></datastore>",
-                CLIXON_LIB_NS, dbname, nr, sz);
+                dbname, nr, sz);
     }
+ ok:
     retval = 0;
  done:
     if (xn)
@@ -377,9 +375,7 @@ clixon_stats_module_get(clicon_handle h,
         return 0;
     if (yang_stats(ys, &nr, &sz) < 0)
         goto done;
-    cprintf(cb, "<module xmlns=\"%s\"><name>%s</name><nr>%" PRIu64 "</nr>"
+    cprintf(cb, "<nr>%" PRIu64 "</nr><size>%zu</size>", nr, sz);
-            "<size>%zu</size></module>",
-            CLIXON_LIB_NS, yang_argument_get(ys), nr, sz);
     retval = 0;
  done:
     if (xn)
@@ -1308,9 +1304,15 @@ from_client_stats(clicon_handle h,
     int        retval = -1;
     uint64_t   nr;
     yang_stmt *ym;
+    char      *str;
+    int        modules = 0;
+    yang_stmt *yspec;
+    yang_stmt *ymodext;
+    cxobj     *xt;
     
+    if ((str = xml_find_body(xe, "modules")) != NULL)
+        modules = strcmp(str, "true") == 0;
     cprintf(cbret, "<rpc-reply xmlns=\"%s\">", NETCONF_BASE_NAMESPACE);
-    xml_stats_global(&nr);
     cprintf(cbret, "<global xmlns=\"%s\">", CLIXON_LIB_NS);
     nr=0;
     xml_stats_global(&nr);
@@ -1319,27 +1321,52 @@ from_client_stats(clicon_handle h,
     yang_stats_global(&nr);
     cprintf(cbret, "<yangnr>%" PRIu64 "</yangnr>", nr);
     cprintf(cbret, "</global>");
+    cprintf(cbret, "<datastores xmlns=\"%s\">", CLIXON_LIB_NS);
     if (clixon_stats_datastore_get(h, "running", cbret) < 0)
         goto done;
     if (clixon_stats_datastore_get(h, "candidate", cbret) < 0)
         goto done;
     if (clixon_stats_datastore_get(h, "startup", cbret) < 0)
         goto done;
+    cprintf(cbret, "</datastores>");
+    /* per module-set, first configuration, then main dbspec, then mountpoints */
+    cprintf(cbret, "<module-sets xmlns=\"%s\">", CLIXON_LIB_NS);
+    cprintf(cbret, "<module-set><name>clixon-config</name>");
+    yspec = clicon_config_yang(h);
+    if (clixon_stats_module_get(h, yspec, cbret) < 0)
+        goto done;
+    if (modules){
         ym = NULL;
-    while ((ym = yn_each(clicon_config_yang(h), ym)) != NULL) {    
+        while ((ym = yn_each(yspec, ym)) != NULL) {    
+            cprintf(cbret, "<module><name>%s</name>", yang_argument_get(ym));
             if (clixon_stats_module_get(h, ym, cbret) < 0)
                 goto done;
+            cprintf(cbret, "</module>");
         }
+    }
+    cprintf(cbret, "</module-set>");
+    cprintf(cbret, "<module-set><name>main</name>");
+    yspec = clicon_dbspec_yang(h);
+    if (clixon_stats_module_get(h, yspec, cbret) < 0)
+        goto done;
+    if (modules){
         ym = NULL;
-    while ((ym = yn_each(clicon_dbspec_yang(h), ym)) != NULL) {    
+        while ((ym = yn_each(yspec, ym)) != NULL) {
+            cprintf(cbret, "<module><name>%s</name>", yang_argument_get(ym));
             if (clixon_stats_module_get(h, ym, cbret) < 0)
                 goto done;
+            cprintf(cbret, "</module>");
         }
-    ym = NULL;
+    }
-    while ((ym = yn_each(clicon_nacm_ext_yang(h), ym)) != NULL) {    
+    cprintf(cbret, "</module-set>");
-        if (clixon_stats_module_get(h, ym, cbret) < 0)
+    /* Mountpoints */
+    if ((ymodext = yang_find(yspec, Y_MODULE, "ietf-yang-schema-mount")) != NULL){
+        if (xmldb_get(h, "running", NULL, "/", &xt) < 0)
+            goto done;
+        if (xt && yang_schema_mount_statistics(h, xt, modules, cbret) < 0)
             goto done;
     }
+    cprintf(cbret, "</module-sets>");
     cprintf(cbret, "</rpc-reply>");
     retval = 0;
  done:

apps/cli/cli_show.c

@@ -53,6 +53,7 @@
 #include <dirent.h>
 #include <syslog.h>
 #include <pwd.h>
+#include <inttypes.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <sys/types.h>
@@ -1487,3 +1488,73 @@ clixon_cli2file(clicon_handle     h,
  done:
     return retval;
 }
+
+/*! CLI callback show statistics
+ */
+int
+cli_show_statistics(clicon_handle h, 
+                    cvec         *cvv, 
+                    cvec         *argv)
+{
+    int         retval = -1;
+    cbuf       *cb = NULL;
+    cxobj      *xret = NULL;
+    cxobj      *xerr;
+    cg_var     *cv;
+    int         modules = 0;
+    pt_head    *ph;
+    parse_tree *pt;
+    uint64_t    nr = 0;
+    size_t      sz = 0;
+    
+    if (argv != NULL && cvec_len(argv) != 1){
+        clicon_err(OE_PLUGIN, EINVAL, "Expected arguments: [modules]");
+        goto done;
+    }
+    if (argv){
+        cv = cvec_i(argv, 0);
+        modules = (strcmp(cv_string_get(cv), "modules") == 0);
+    }
+    if ((cb = cbuf_new()) == NULL){
+        clicon_err(OE_PLUGIN, errno, "cbuf_new");
+        goto done;
+    }
+    /* CLI */
+    cligen_output(stdout, "CLI:\n");
+    ph = NULL;
+    while ((ph = cligen_ph_each(cli_cligen(h), ph)) != NULL) {
+        if ((pt = cligen_ph_parsetree_get(ph)) == NULL)
+            continue;
+        nr = 0; sz = 0;
+        pt_stats(pt, &nr, &sz);
+        cligen_output(stdout, "%s: nr=%" PRIu64 " size:%zu\n",
+                      cligen_ph_name_get(ph), nr, sz);        
+    }
+    /* Backend */
+    cprintf(cb, "<rpc xmlns=\"%s\"", NETCONF_BASE_NAMESPACE);
+    cprintf(cb, " %s", NETCONF_MESSAGE_ID_ATTR); /* XXX: use incrementing sequence */
+    cprintf(cb, ">");
+    cprintf(cb, "<stats xmlns=\"%s\">", CLIXON_LIB_NS);
+    if (modules)
+        cprintf(cb, "<modules>true</modules>");
+    cprintf(cb, "</stats>");
+    cprintf(cb, "</rpc>");
+    if (clicon_rpc_netconf(h, cbuf_get(cb), &xret, NULL) < 0)
+        goto done;
+    if ((xerr = xpath_first(xret, NULL, "//rpc-error")) != NULL){
+        clixon_netconf_error(xerr, "Get configuration", NULL);
+        goto done;
+    }
+    fprintf(stdout, "Backend:\n");
+    if (clixon_xml2file(stdout, xml_child_i(xret, 0), 0, 1, NULL, cligen_output, 0, 1) < 0)
+        goto done;
+    fprintf(stdout, "CLI:\n");
+    
+    retval = 0;
+ done:
+    if (xret)
+        xml_free(xret);
+    if (cb)
+        cbuf_free(cb);
+    return retval;
+}

example/main/example_cli.cli

@@ -130,6 +130,10 @@ show("Show a particular state of the system"){
     yang("Show yang specs"), show_yang(); {
         clixon-example("Show clixon-example yang spec"), show_yang("clixon-example");
     }		   
+    statistics("Show statistics"), cli_show_statistics();{
+      brief, cli_show_statistics();
+      modules, cli_show_statistics("modules");
+    }
 }
 
 save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename", "xml");{

lib/clixon/clixon_yang_schema_mount.h

@@ -60,6 +60,7 @@ int xml_yang_mount_get(clicon_handle h, cxobj *x, validate_level *vl, yang_stmt
 int xml_yang_mount_set(cxobj *x,  yang_stmt *yspec);
 int xml_yang_mount_freeall(cvec *cvv);
 int yang_schema_mount_statedata(clicon_handle h, yang_stmt *yspec, char *xpath, cvec *nsc, cxobj **xret, cxobj **xerr);
+int yang_schema_mount_statistics(clicon_handle h, cxobj *xt, int modules, cbuf *cb);
 int yang_schema_yanglib_parse_mount(clicon_handle h, cxobj *xt);
 int yang_schema_get_child(clicon_handle h, cxobj *x1, cxobj *x1c, yang_stmt **yc);
 

lib/src/clixon_yang_schema_mount.c

@@ -58,6 +58,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <inttypes.h>
 #include <limits.h>
 #include <string.h>
 #include <sys/param.h>
@@ -209,7 +210,7 @@ yang_mount_set(yang_stmt *yu,
  *
  * @param[in]  h     Clixon handle
  * @param[in]  x     XML moint-point node
- * @param[out] vallevel Do or dont do full RFC 7950 validation
+ * @param[out] vallevel Do or dont do full RFC 7950 validation if given
  * @param[out] yspec YANG stmt spec
  * @retval     1     x is a mount-point: yspec may be set
  * @retval     0     x is not a mount point
@@ -505,6 +506,75 @@ yang_schema_mount_statedata(clicon_handle h,
     goto done;
 }
 
+/*! Statistics about mountpoints
+ * @see yang_schema_mount_statedata
+ */
+int
+yang_schema_mount_statistics(clicon_handle h,
+                             cxobj        *xt,
+                             int           modules,
+                             cbuf         *cb)
+{
+    int        retval = -1;
+    cvec      *cvv = NULL;
+    cg_var    *cv;
+    cxobj     *xmp;          /* xml mount-point */
+    yang_stmt *yspec;
+    yang_stmt *ym;
+    int        ret;
+    char      *xpath;
+    uint64_t   nr;
+    size_t     sz;
+
+    if ((cvv = cvec_new(0)) == NULL){
+        clicon_err(OE_UNIX, errno, "cvec_new");
+        goto done;
+    }
+    if (xml_apply(xt, CX_ELMNT, find_schema_mounts, cvv) < 0)
+        goto done;
+    cv = NULL;
+    while ((cv = cvec_each(cvv, cv)) != NULL) {
+        if ((xmp = cv_void_get(cv)) == NULL)
+            continue;
+        if ((ret = xml_yang_mount_get(h, xmp, NULL, &yspec)) < 0)
+            goto done;
+        if (ret == 0)
+            continue;
+        if (xml2xpath(xmp, NULL, 1, 0, &xpath) < 0)
+            goto done;
+        cprintf(cb, "<module-set><name>mountpoint: ");
+        xml_chardata_cbuf_append(cb, xpath);
+        cprintf(cb, "</name>");
+        nr = 0; sz = 0;
+        if (yang_stats(yspec, &nr, &sz) < 0)
+            goto done;
+        cprintf(cb, "<nr>%" PRIu64 "</nr><size>%zu</size>", nr, sz);
+        if (modules){
+            ym = NULL;
+            while ((ym = yn_each(yspec, ym)) != NULL) {
+                cprintf(cb, "<module><name>%s</name>", yang_argument_get(ym));
+                nr = 0; sz = 0;
+                if (yang_stats(ym, &nr, &sz) < 0)
+                    goto done;
+                cprintf(cb, "<nr>%" PRIu64 "</nr><size>%zu</size>", nr, sz);
+                cprintf(cb, "</module>");
+            }
+        }
+        cprintf(cb, "</module-set>");
+        if (xpath){
+            free(xpath);
+            xpath = NULL;
+        }
+    }
+    retval = 0;
+ done:
+    if (xpath)
+        free(xpath);
+    if (cvv)
+        cvec_free(cvv);
+    return retval;
+}
+
 /*! Get yanglib from user plugin callback, parse it and mount it
  * 
  * @param[in]     h     Clixon handle

test/config.sh.in

@@ -73,7 +73,7 @@ DATASTORE_TOP="config"
 
 # clixon yang revisions occuring in tests (see eg yang/clixon/Makefile.in)
 CLIXON_AUTOCLI_REV="2022-02-11"
-CLIXON_LIB_REV="2023-03-01"
+CLIXON_LIB_REV="2023-05-01"
 CLIXON_CONFIG_REV="2023-03-01"
 CLIXON_RESTCONF_REV="2022-08-01"
 CLIXON_EXAMPLE_REV="2022-11-01"

test/test_perf_mem.sh

@@ -91,7 +91,7 @@ function testrun(){
     pid=$(cat $pidfile)
 
     new "netconf get stats"
-    rpc=$(chunked_framing "<rpc $DEFAULTNS><stats $LIBNS/></rpc>")
+    rpc=$(chunked_framing "<rpc $DEFAULTNS><stats $LIBNS><modules>true</modules></stats></rpc>")
     res=$(echo "$DEFAULTHELLO$rpc" | $clixon_netconf -qef $cfg)
 #    echo "res:$res"
     err0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/rpc-error")
@@ -112,7 +112,7 @@ function testrun(){
     fi
     for db in running candidate startup; do
         echo "$db"
-        resdb0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastore[name=\"$db\"]")
+        resdb0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastores/datastore[name=\"$db\"]")
         resdb=${resdb0#"nodeset:0:"}
         if [ "$resdb0" = "$resdb" ]; then
             err1 "nodeset:0:" "$resdb0"
@@ -122,15 +122,6 @@ function testrun(){
         echo -n "   mem: "
         echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
     done
-    for mod in clixon-config; do
-        echo "$mod"
-        resmod0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/module[name=\"$mod\"]")
-        resmod=${resmod0#"nodeset:0:"}
-        echo -n "   objects: "
-        echo $resmod | $clixon_util_xpath -p "module/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}'
-        echo -n "   mem: "
-        echo $resmod | $clixon_util_xpath -p "module/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}'
-    done
     if [ $BE -ne 0 ]; then
         new "Kill backend"
         # Check if premature kill

test/test_yang_schema_mount.sh

@@ -83,6 +83,10 @@ expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS>
 new "get yang-lib at mountpoint"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get><filter type=\"subtree\"><top xmlns=\"urn:example:clixon\"><mylist/></top>></filter></get></rpc>" "<rpc-reply $DEFAULTNS><data><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><yang-library xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"><module-set><name>mount</name><module><name>clixon-example</name><revision>2022-11-01</revision><namespace>urn:example:urn</namespace></module></module-set></yang-library></root></mylist><mylist><name>y</name><root><yang-library xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"><module-set><name>mount</name><module><name>clixon-example</name><revision>2022-11-01</revision><namespace>urn:example:urn</namespace></module></module-set></yang-library></root></mylist></top></data></rpc-reply>"
 
+new "check there is statistics from mountpoint"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><stats xmlns=\"http://clicon.org/lib\"></stats></rpc>" '<module-set><name>mountpoint: /top/mylist\[name="x"\]/root</name><nr>'
+#"<rpc-reply $DEFAULTNS></rpc-reply>"
+
 if [ $BE -ne 0 ]; then
     new "Kill backend"
     # Check if premature kill

yang/clixon/Makefile.in

@@ -43,7 +43,7 @@ YANG_INSTALLDIR   = @YANG_INSTALLDIR@
 
 # Note: mirror these to test/config.sh.in
 YANGSPECS	 = clixon-config@2023-03-01.yang   # 6.2
-YANGSPECS	+= clixon-lib@2023-03-01.yang      # 6.2
+YANGSPECS	+= clixon-lib@2023-05-01.yang      # 6.3
 YANGSPECS	+= clixon-rfc5277@2008-07-01.yang
 YANGSPECS	+= clixon-xml-changelog@2019-03-21.yang
 YANGSPECS	+= clixon-restconf@2022-08-01.yang # 5.9

yang/clixon/clixon-lib@2023-05-01.yang

@@ -9,6 +9,9 @@ module clixon-lib {
     import ietf-netconf-monitoring {
         prefix ncm;
     }    
+    import ietf-yang-metadata {
+        prefix "md";
+    }
     organization
         "Clicon / Clixon";
 
@@ -18,7 +21,7 @@ module clixon-lib {
     description
         "***** BEGIN LICENSE BLOCK *****
        Copyright (C) 2009-2019 Olof Hagsand
-       Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+       Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate)
        
        This file is part of CLIXON
 
@@ -51,7 +54,7 @@ module clixon-lib {
        - RPCs for debug, stats and process-control
        - Informal description of attributes
 
-       Additionally, Clixon extends NETCONF for internal use with some internal attributes. These
+       Clixon also extends NETCONF for internal use with some internal attributes. These
        are not visible for external usage bit belongs to the namespace of this YANG.
        The internal attributes are:
        - content (also RESTCONF)
@@ -65,6 +68,14 @@ module clixon-lib {
        - objectexisted
       ";
 
+    revision 2023-05-01 {
+        description
+            "Restructured and extended stats rpc to schema mountpoints";
+    }
+    revision 2023-03-01 {
+        description
+            "Added creator meta-object";
+    }
     revision 2022-12-01 {
         description
             "Added values of RFC6022 transport identityref 
@@ -138,7 +149,7 @@ module clixon-lib {
     }
     identity netconf {
         description
-            "Just NETCONF without specitic underlying transport, 
+            "Just NETCONF without specific underlying transport, 
              Clixon uses stdio for its netconf client and therefore does not know whether it is
              invoked in a script, by a NETCONF/SSH subsystem, etc";
         base ncm:transport;
@@ -166,6 +177,17 @@ module clixon-lib {
         argument cliop;
         status obsolete;
     }
+
+    md:annotation creator {
+        type string;
+        description
+            "This annotation contains the name of a creator of an object.
+             One application is the clixon controller where multiple services can
+             create the same object. When such a service is deleted (or changed) one needs to keep
+             track of which service created what.
+             Limitations: only objects that are actually added or deleted. 
+             A sub-object wil not be noted";
+    }
     rpc debug {
         description "Set debug level of backend.";
         input {
@@ -177,8 +199,15 @@ module clixon-lib {
     rpc ping {
         description "Check aliveness of backend daemon.";
     }
-    rpc stats {
+    rpc stats { /* Could be moved to state */
-        description "Clixon XML statistics.";
+        description "Clixon yang and datastore statistics.";
+        input {
+            leaf modules {
+                description "If enabled include per-module statistics";
+                type boolean;
+                mandatory false;
+            }
+        }
         output {
             container global{
                 description
@@ -198,6 +227,7 @@ module clixon-lib {
                     type uint64;
                 }
             }
+            container datastores{
               list datastore{
                 description "Per datastore statistics for cxobj";
                 key "name";
@@ -215,8 +245,27 @@ module clixon-lib {
                     type uint64;
                 }
               }
+            }
+            container module-sets{
+              list module-set{
+                description "Statistics per group of module, eg top-level and mount-points";
+                key "name";
+                leaf name{
+                    description "Name of YANG module.";
+                    type string;
+                }
+                leaf nr{
+                    description
+                        "Total number of YANG objects in set";
+                    type uint64;
+                }
+                leaf size{
+                    description
+                        "Total size in bytes of internal YANG object representation for module set";
+                    type uint64;
+                }
                 list module{
-                description "Per YANG module statistics";
+                    description "Statistics per module (if modules set in input)";
                     key "name";
                     leaf name{
                         description "Name of YANG module.";
@@ -235,6 +284,8 @@ module clixon-lib {
                 }
               }
             }
+        }
+    }
     rpc restart-plugin {
         description "Restart specific backend plugins.";
         input {
@@ -244,7 +295,6 @@ module clixon-lib {
             }
         }
     }
-
     rpc process-control {
         description
             "Control a specific process or daemon: start/stop, etc.