diff --git a/CHANGELOG.md b/CHANGELOG.md index e51347cd..a5bfb897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,8 +43,15 @@ ## 6.3.0 Expected: July 2023 +### API changes on existing protocol/config features + +* New `clixon-lib@2023-05-01.yang` revision + * Restructured and extended stats rpc to schema mountpoints + * rpc `` is not backward compatible + ### Minor features +* CLI: Added `show statistics` example code for backend and CLI memory stats * [Support yang type union with are same subtypes with SNMP](https://github.com/clicon/clixon/pull/427) * Removed obsolete compile options introduced in 6.1: * `NETCONF_DEFAULT_RETRIEVAL_REPORT_ALL` diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 31808577..e52f465e 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -334,20 +334,18 @@ clixon_stats_datastore_get(clicon_handle h, if ((xt = xmldb_cache_get(h, dbname)) == NULL){ /* Trigger cache if no exist */ if (xmldb_get(h, dbname, NULL, "/", &xn) < 0) - goto done; + //goto done; + goto ok; xt = xmldb_cache_get(h, dbname); } - if (xt == NULL){ - cprintf(cb, "%s00", - CLIXON_LIB_NS, dbname); - } - else{ + if (xt != NULL){ if (xml_stats(xt, &nr, &sz) < 0) goto done; - cprintf(cb, "%s%" PRIu64 "" + cprintf(cb, "%s%" PRIu64 "" "%zu", - CLIXON_LIB_NS, dbname, nr, sz); + dbname, nr, sz); } + ok: retval = 0; done: if (xn) @@ -377,9 +375,7 @@ clixon_stats_module_get(clicon_handle h, return 0; if (yang_stats(ys, &nr, &sz) < 0) goto done; - cprintf(cb, "%s%" PRIu64 "" - "%zu", - CLIXON_LIB_NS, yang_argument_get(ys), nr, sz); + cprintf(cb, "%" PRIu64 "%zu", nr, sz); retval = 0; done: if (xn) @@ -1308,9 +1304,15 @@ from_client_stats(clicon_handle h, int retval = -1; uint64_t nr; yang_stmt *ym; + char *str; + int modules = 0; + yang_stmt *yspec; + yang_stmt *ymodext; + cxobj *xt; + if ((str = xml_find_body(xe, "modules")) != NULL) + modules = strcmp(str, "true") == 0; cprintf(cbret, "", NETCONF_BASE_NAMESPACE); - xml_stats_global(&nr); cprintf(cbret, "", CLIXON_LIB_NS); nr=0; xml_stats_global(&nr); @@ -1319,27 +1321,52 @@ from_client_stats(clicon_handle h, yang_stats_global(&nr); cprintf(cbret, "%" PRIu64 "", nr); cprintf(cbret, ""); + cprintf(cbret, "", CLIXON_LIB_NS); if (clixon_stats_datastore_get(h, "running", cbret) < 0) goto done; if (clixon_stats_datastore_get(h, "candidate", cbret) < 0) goto done; if (clixon_stats_datastore_get(h, "startup", cbret) < 0) goto done; - ym = NULL; - while ((ym = yn_each(clicon_config_yang(h), ym)) != NULL) { - if (clixon_stats_module_get(h, ym, cbret) < 0) - goto done; - } - ym = NULL; - while ((ym = yn_each(clicon_dbspec_yang(h), ym)) != NULL) { - if (clixon_stats_module_get(h, ym, cbret) < 0) - goto done; - } - ym = NULL; - while ((ym = yn_each(clicon_nacm_ext_yang(h), ym)) != NULL) { - if (clixon_stats_module_get(h, ym, cbret) < 0) + cprintf(cbret, ""); + /* per module-set, first configuration, then main dbspec, then mountpoints */ + cprintf(cbret, "", CLIXON_LIB_NS); + cprintf(cbret, "clixon-config"); + yspec = clicon_config_yang(h); + if (clixon_stats_module_get(h, yspec, cbret) < 0) + goto done; + if (modules){ + ym = NULL; + while ((ym = yn_each(yspec, ym)) != NULL) { + cprintf(cbret, "%s", yang_argument_get(ym)); + if (clixon_stats_module_get(h, ym, cbret) < 0) + goto done; + cprintf(cbret, ""); + } + } + cprintf(cbret, ""); + cprintf(cbret, "main"); + yspec = clicon_dbspec_yang(h); + if (clixon_stats_module_get(h, yspec, cbret) < 0) + goto done; + if (modules){ + ym = NULL; + while ((ym = yn_each(yspec, ym)) != NULL) { + cprintf(cbret, "%s", yang_argument_get(ym)); + if (clixon_stats_module_get(h, ym, cbret) < 0) + goto done; + cprintf(cbret, ""); + } + } + cprintf(cbret, ""); + /* Mountpoints */ + if ((ymodext = yang_find(yspec, Y_MODULE, "ietf-yang-schema-mount")) != NULL){ + if (xmldb_get(h, "running", NULL, "/", &xt) < 0) + goto done; + if (xt && yang_schema_mount_statistics(h, xt, modules, cbret) < 0) goto done; } + cprintf(cbret, ""); cprintf(cbret, ""); retval = 0; done: diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index 16f4e1dd..f397df07 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -1487,3 +1488,73 @@ clixon_cli2file(clicon_handle h, done: return retval; } + +/*! CLI callback show statistics + */ +int +cli_show_statistics(clicon_handle h, + cvec *cvv, + cvec *argv) +{ + int retval = -1; + cbuf *cb = NULL; + cxobj *xret = NULL; + cxobj *xerr; + cg_var *cv; + int modules = 0; + pt_head *ph; + parse_tree *pt; + uint64_t nr = 0; + size_t sz = 0; + + if (argv != NULL && cvec_len(argv) != 1){ + clicon_err(OE_PLUGIN, EINVAL, "Expected arguments: [modules]"); + goto done; + } + if (argv){ + cv = cvec_i(argv, 0); + modules = (strcmp(cv_string_get(cv), "modules") == 0); + } + if ((cb = cbuf_new()) == NULL){ + clicon_err(OE_PLUGIN, errno, "cbuf_new"); + goto done; + } + /* CLI */ + cligen_output(stdout, "CLI:\n"); + ph = NULL; + while ((ph = cligen_ph_each(cli_cligen(h), ph)) != NULL) { + if ((pt = cligen_ph_parsetree_get(ph)) == NULL) + continue; + nr = 0; sz = 0; + pt_stats(pt, &nr, &sz); + cligen_output(stdout, "%s: nr=%" PRIu64 " size:%zu\n", + cligen_ph_name_get(ph), nr, sz); + } + /* Backend */ + cprintf(cb, ""); + cprintf(cb, "", CLIXON_LIB_NS); + if (modules) + cprintf(cb, "true"); + cprintf(cb, ""); + cprintf(cb, ""); + if (clicon_rpc_netconf(h, cbuf_get(cb), &xret, NULL) < 0) + goto done; + if ((xerr = xpath_first(xret, NULL, "//rpc-error")) != NULL){ + clixon_netconf_error(xerr, "Get configuration", NULL); + goto done; + } + fprintf(stdout, "Backend:\n"); + if (clixon_xml2file(stdout, xml_child_i(xret, 0), 0, 1, NULL, cligen_output, 0, 1) < 0) + goto done; + fprintf(stdout, "CLI:\n"); + + retval = 0; + done: + if (xret) + xml_free(xret); + if (cb) + cbuf_free(cb); + return retval; +} diff --git a/example/main/example_cli.cli b/example/main/example_cli.cli index 0da1a6a1..542a8184 100644 --- a/example/main/example_cli.cli +++ b/example/main/example_cli.cli @@ -130,6 +130,10 @@ show("Show a particular state of the system"){ yang("Show yang specs"), show_yang(); { clixon-example("Show clixon-example yang spec"), show_yang("clixon-example"); } + statistics("Show statistics"), cli_show_statistics();{ + brief, cli_show_statistics(); + modules, cli_show_statistics("modules"); + } } save("Save candidate configuration to XML file") ("Filename (local filename)"), save_config_file("candidate","filename", "xml");{ diff --git a/lib/clixon/clixon_yang_schema_mount.h b/lib/clixon/clixon_yang_schema_mount.h index 98ed6bce..8b9bb250 100644 --- a/lib/clixon/clixon_yang_schema_mount.h +++ b/lib/clixon/clixon_yang_schema_mount.h @@ -60,6 +60,7 @@ int xml_yang_mount_get(clicon_handle h, cxobj *x, validate_level *vl, yang_stmt int xml_yang_mount_set(cxobj *x, yang_stmt *yspec); int xml_yang_mount_freeall(cvec *cvv); int yang_schema_mount_statedata(clicon_handle h, yang_stmt *yspec, char *xpath, cvec *nsc, cxobj **xret, cxobj **xerr); +int yang_schema_mount_statistics(clicon_handle h, cxobj *xt, int modules, cbuf *cb); int yang_schema_yanglib_parse_mount(clicon_handle h, cxobj *xt); int yang_schema_get_child(clicon_handle h, cxobj *x1, cxobj *x1c, yang_stmt **yc); diff --git a/lib/src/clixon_yang_schema_mount.c b/lib/src/clixon_yang_schema_mount.c index 08799c39..85881791 100644 --- a/lib/src/clixon_yang_schema_mount.c +++ b/lib/src/clixon_yang_schema_mount.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -209,7 +210,7 @@ yang_mount_set(yang_stmt *yu, * * @param[in] h Clixon handle * @param[in] x XML moint-point node - * @param[out] vallevel Do or dont do full RFC 7950 validation + * @param[out] vallevel Do or dont do full RFC 7950 validation if given * @param[out] yspec YANG stmt spec * @retval 1 x is a mount-point: yspec may be set * @retval 0 x is not a mount point @@ -505,6 +506,75 @@ yang_schema_mount_statedata(clicon_handle h, goto done; } +/*! Statistics about mountpoints + * @see yang_schema_mount_statedata + */ +int +yang_schema_mount_statistics(clicon_handle h, + cxobj *xt, + int modules, + cbuf *cb) +{ + int retval = -1; + cvec *cvv = NULL; + cg_var *cv; + cxobj *xmp; /* xml mount-point */ + yang_stmt *yspec; + yang_stmt *ym; + int ret; + char *xpath; + uint64_t nr; + size_t sz; + + if ((cvv = cvec_new(0)) == NULL){ + clicon_err(OE_UNIX, errno, "cvec_new"); + goto done; + } + if (xml_apply(xt, CX_ELMNT, find_schema_mounts, cvv) < 0) + goto done; + cv = NULL; + while ((cv = cvec_each(cvv, cv)) != NULL) { + if ((xmp = cv_void_get(cv)) == NULL) + continue; + if ((ret = xml_yang_mount_get(h, xmp, NULL, &yspec)) < 0) + goto done; + if (ret == 0) + continue; + if (xml2xpath(xmp, NULL, 1, 0, &xpath) < 0) + goto done; + cprintf(cb, "mountpoint: "); + xml_chardata_cbuf_append(cb, xpath); + cprintf(cb, ""); + nr = 0; sz = 0; + if (yang_stats(yspec, &nr, &sz) < 0) + goto done; + cprintf(cb, "%" PRIu64 "%zu", nr, sz); + if (modules){ + ym = NULL; + while ((ym = yn_each(yspec, ym)) != NULL) { + cprintf(cb, "%s", yang_argument_get(ym)); + nr = 0; sz = 0; + if (yang_stats(ym, &nr, &sz) < 0) + goto done; + cprintf(cb, "%" PRIu64 "%zu", nr, sz); + cprintf(cb, ""); + } + } + cprintf(cb, ""); + if (xpath){ + free(xpath); + xpath = NULL; + } + } + retval = 0; + done: + if (xpath) + free(xpath); + if (cvv) + cvec_free(cvv); + return retval; +} + /*! Get yanglib from user plugin callback, parse it and mount it * * @param[in] h Clixon handle diff --git a/test/config.sh.in b/test/config.sh.in index 2bac50f0..a08e2f5d 100755 --- a/test/config.sh.in +++ b/test/config.sh.in @@ -73,7 +73,7 @@ DATASTORE_TOP="config" # clixon yang revisions occuring in tests (see eg yang/clixon/Makefile.in) CLIXON_AUTOCLI_REV="2022-02-11" -CLIXON_LIB_REV="2023-03-01" +CLIXON_LIB_REV="2023-05-01" CLIXON_CONFIG_REV="2023-03-01" CLIXON_RESTCONF_REV="2022-08-01" CLIXON_EXAMPLE_REV="2022-11-01" diff --git a/test/test_perf_mem.sh b/test/test_perf_mem.sh index 25a7f642..6012313f 100755 --- a/test/test_perf_mem.sh +++ b/test/test_perf_mem.sh @@ -91,7 +91,7 @@ function testrun(){ pid=$(cat $pidfile) new "netconf get stats" - rpc=$(chunked_framing "") + rpc=$(chunked_framing "true") res=$(echo "$DEFAULTHELLO$rpc" | $clixon_netconf -qef $cfg) # echo "res:$res" err0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/rpc-error") @@ -112,7 +112,7 @@ function testrun(){ fi for db in running candidate startup; do echo "$db" - resdb0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastore[name=\"$db\"]") + resdb0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/datastores/datastore[name=\"$db\"]") resdb=${resdb0#"nodeset:0:"} if [ "$resdb0" = "$resdb" ]; then err1 "nodeset:0:" "$resdb0" @@ -122,15 +122,6 @@ function testrun(){ echo -n " mem: " echo $resdb | $clixon_util_xpath -p "datastore/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}' done - for mod in clixon-config; do - echo "$mod" - resmod0=$(echo "$res" | $clixon_util_xpath -p "/rpc-reply/module[name=\"$mod\"]") - resmod=${resmod0#"nodeset:0:"} - echo -n " objects: " - echo $resmod | $clixon_util_xpath -p "module/nr" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' - echo -n " mem: " - echo $resmod | $clixon_util_xpath -p "module/size" | awk -F ">" '{print $2}' | awk -F "<" '{print $1}' | awk '{print $1/1000000 "M"}' - done if [ $BE -ne 0 ]; then new "Kill backend" # Check if premature kill diff --git a/test/test_yang_schema_mount.sh b/test/test_yang_schema_mount.sh index 7c57981e..62388714 100755 --- a/test/test_yang_schema_mount.sh +++ b/test/test_yang_schema_mount.sh @@ -83,6 +83,10 @@ expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" " new "get yang-lib at mountpoint" expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" ">" "xmountclixon-example2022-11-01urn:example:urnymountclixon-example2022-11-01urn:example:urn" +new "check there is statistics from mountpoint" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" 'mountpoint: /top/mylist\[name="x"\]/root' +#"" + if [ $BE -ne 0 ]; then new "Kill backend" # Check if premature kill diff --git a/yang/clixon/Makefile.in b/yang/clixon/Makefile.in index 964a6526..639ce7fa 100644 --- a/yang/clixon/Makefile.in +++ b/yang/clixon/Makefile.in @@ -43,7 +43,7 @@ YANG_INSTALLDIR = @YANG_INSTALLDIR@ # Note: mirror these to test/config.sh.in YANGSPECS = clixon-config@2023-03-01.yang # 6.2 -YANGSPECS += clixon-lib@2023-03-01.yang # 6.2 +YANGSPECS += clixon-lib@2023-05-01.yang # 6.3 YANGSPECS += clixon-rfc5277@2008-07-01.yang YANGSPECS += clixon-xml-changelog@2019-03-21.yang YANGSPECS += clixon-restconf@2022-08-01.yang # 5.9 diff --git a/yang/clixon/clixon-lib@2022-12-01.yang b/yang/clixon/clixon-lib@2023-05-01.yang similarity index 79% rename from yang/clixon/clixon-lib@2022-12-01.yang rename to yang/clixon/clixon-lib@2023-05-01.yang index 05ceb9b6..d23a56ae 100644 --- a/yang/clixon/clixon-lib@2022-12-01.yang +++ b/yang/clixon/clixon-lib@2023-05-01.yang @@ -9,6 +9,9 @@ module clixon-lib { import ietf-netconf-monitoring { prefix ncm; } + import ietf-yang-metadata { + prefix "md"; + } organization "Clicon / Clixon"; @@ -16,9 +19,9 @@ module clixon-lib { "Olof Hagsand "; description - "***** BEGIN LICENSE BLOCK ***** + "***** BEGIN LICENSE BLOCK ***** Copyright (C) 2009-2019 Olof Hagsand - Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate) + Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate) This file is part of CLIXON @@ -51,7 +54,7 @@ module clixon-lib { - RPCs for debug, stats and process-control - Informal description of attributes - Additionally, Clixon extends NETCONF for internal use with some internal attributes. These + Clixon also extends NETCONF for internal use with some internal attributes. These are not visible for external usage bit belongs to the namespace of this YANG. The internal attributes are: - content (also RESTCONF) @@ -65,6 +68,14 @@ module clixon-lib { - objectexisted "; + revision 2023-05-01 { + description + "Restructured and extended stats rpc to schema mountpoints"; + } + revision 2023-03-01 { + description + "Added creator meta-object"; + } revision 2022-12-01 { description "Added values of RFC6022 transport identityref @@ -113,15 +124,15 @@ module clixon-lib { type enumeration { enum start { description - "Start if not already running"; + "Start if not already running"; } enum stop { description - "Stop if running"; + "Stop if running"; } enum restart { description - "Stop if running, then start"; + "Stop if running, then start"; } enum status { description @@ -138,7 +149,7 @@ module clixon-lib { } identity netconf { description - "Just NETCONF without specitic underlying transport, + "Just NETCONF without specific underlying transport, Clixon uses stdio for its netconf client and therefore does not know whether it is invoked in a script, by a NETCONF/SSH subsystem, etc"; base ncm:transport; @@ -154,8 +165,8 @@ module clixon-lib { base ncm:transport; } extension autocli-op { - description - "Takes an argument an operation defing how to modify the clispec at + description + "Takes an argument an operation defing how to modify the clispec at this point in the YANG tree for the automated generated CLI. Note that this extension is only used in clixon_cli. Operations is expected to be extended, but the following operations are defined: @@ -163,10 +174,21 @@ module clixon-lib { - hide-database This command hides the database - hide-database-auto-completion This command hides the database and the auto completion (meaning, this command acts as both commands above) Obsolete: use clixon-autocli:hide and clixon-autocli:hide-show instead"; - argument cliop; - status obsolete; - } - rpc debug { + argument cliop; + status obsolete; + } + + md:annotation creator { + type string; + description + "This annotation contains the name of a creator of an object. + One application is the clixon controller where multiple services can + create the same object. When such a service is deleted (or changed) one needs to keep + track of which service created what. + Limitations: only objects that are actually added or deleted. + A sub-object wil not be noted"; + } + rpc debug { description "Set debug level of backend."; input { leaf level { @@ -177,8 +199,15 @@ module clixon-lib { rpc ping { description "Check aliveness of backend daemon."; } - rpc stats { - description "Clixon XML statistics."; + rpc stats { /* Could be moved to state */ + description "Clixon yang and datastore statistics."; + input { + leaf modules { + description "If enabled include per-module statistics"; + type boolean; + mandatory false; + } + } output { container global{ description @@ -198,7 +227,8 @@ module clixon-lib { type uint64; } } - list datastore{ + container datastores{ + list datastore{ description "Per datastore statistics for cxobj"; key "name"; leaf name{ @@ -214,9 +244,11 @@ module clixon-lib { description "Size in bytes of internal datastore cache of datastore tree."; type uint64; } + } } - list module{ - description "Per YANG module statistics"; + container module-sets{ + list module-set{ + description "Statistics per group of module, eg top-level and mount-points"; key "name"; leaf name{ description "Name of YANG module."; @@ -224,14 +256,33 @@ module clixon-lib { } leaf nr{ description - "Number of YANG objects. That is number of residing YANG objects"; + "Total number of YANG objects in set"; type uint64; } leaf size{ description - "Size in bytes of internal YANG object representation."; + "Total size in bytes of internal YANG object representation for module set"; type uint64; } + list module{ + description "Statistics per module (if modules set in input)"; + key "name"; + leaf name{ + description "Name of YANG module."; + type string; + } + leaf nr{ + description + "Number of YANG objects. That is number of residing YANG objects"; + type uint64; + } + leaf size{ + description + "Size in bytes of internal YANG object representation."; + type uint64; + } + } + } } } } @@ -244,7 +295,6 @@ module clixon-lib { } } } - rpc process-control { description "Control a specific process or daemon: start/stop, etc.