From aec0a5fc3f801c51154dc5c0e309b5ad54132f1a Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 31 Jul 2024 14:32:48 +0200 Subject: [PATCH] First version of system-only-config A new extension added in clixon-lib.yang A conditional in xmldb_dump to not write system-only data to datastores A test for verifying system-only data is not written at edit-commit --- lib/src/clixon_xml_io.c | 15 +- test/test_datastore_system_only.sh | 221 ++++++++++ yang/clixon/clixon-lib@2024-08-01.yang | 553 +++++++++++++++++++++++++ 3 files changed, 787 insertions(+), 2 deletions(-) create mode 100755 test/test_datastore_system_only.sh create mode 100644 yang/clixon/clixon-lib@2024-08-01.yang diff --git a/lib/src/clixon_xml_io.c b/lib/src/clixon_xml_io.c index c834f16f..2d7d1ba0 100644 --- a/lib/src/clixon_xml_io.c +++ b/lib/src/clixon_xml_io.c @@ -233,7 +233,7 @@ xml2file_recurse(FILE *f, int haselement; char *val; char *encstr = NULL; /* xml encoded string */ - int exist = 0; + int exist; yang_stmt *y; int level1; int tag = 0; @@ -244,11 +244,22 @@ xml2file_recurse(FILE *f, if (x == NULL) goto ok; + y = xml_spec(x); + /* Check if system-only, then do not write to datastore */ + if (y != NULL) { + exist = 0; + if (yang_extension_value(y, "system-only-config", CLIXON_LIB_NS, &exist, NULL) < 0) + goto done; + if (exist){ + goto ok; + } + } level1 = level*PRETTYPRINT_INDENT; if (prefix) level1 -= strlen(prefix); - if ((y = xml_spec(x)) != NULL){ + if (y != NULL){ if (autocliext){ + exist = 0; if (yang_extension_value(y, "hide-show", CLIXON_AUTOCLI_NS, &exist, NULL) < 0) goto done; if (exist) diff --git a/test/test_datastore_system_only.sh b/test/test_datastore_system_only.sh new file mode 100755 index 00000000..944dbb48 --- /dev/null +++ b/test/test_datastore_system_only.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# Datastore system only config test +# see https://github.com/clicon/clixon/pull/534 and extension system-only-config +# Test uses a "standard" yang and a "local" yang which augmanets the standard + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +# include err() and new() functions and creates $dir + +cfg=$dir/conf_yang.xml +clispec=$dir/automode.cli + +fstandard=$dir/clixon-standard.yang +flocal=$dir/clixon-local.yang + +CFD=$dir/conf.d +test -d $CFD || mkdir -p $CFD + +AUTOCLI=$(autocli_config clixon-\* kw-nokey false) + +# Well-known digest of mount-point xpath +subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml + +cat < $cfg + + $cfg + $CFD + ${YANG_INSTALLDIR} + ${dir} + $flocal + true + $dir + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/run/$APPNAME.sock + /usr/local/lib/$APPNAME/backend + /usr/local/var/run/$APPNAME.pidfile + $dir + true + true + true + true + +EOF + +cat < $CFD/autocli.xml + + + false + kw-nokey + true + + include clixon + enable + clixon-* + + + +EOF + +# A "standard" YANG +cat < $fstandard +module clixon-standard{ + yang-version 1.1; + namespace "urn:example:std"; + prefix std; + grouping system-only-group { + description + "A grouping containing a system-only field, corresponding to + a standard module, which gets augmented by a local yang"; + leaf system-only-data { + description + "System-only config data"; + type string; + } + } + grouping store-grouping { + container keys { + list key { + key "name"; + leaf name { + type string; + } + uses system-only-group; + } + } + } + container store { + description "top-level"; + uses store-grouping; + } +} +EOF + +# A "local" YANG +cat < $flocal +module clixon-mount1{ + yang-version 1.1; + namespace "urn:example:local"; + prefix local; + import clixon-lib { + prefix cl; + } + import clixon-standard { + prefix std; + } + augment "/std:store/std:keys/std:key/std:system-only-data" { + cl:system-only-config { + description + "Marks system-only-config data"; + } + } +} +EOF + +cat < $clispec +CLICON_MODE="example"; +CLICON_PROMPT="%U@%H %W> "; +CLICON_PLUGIN="example_cli"; + +# Autocli syntax tree operations +set @datamodel, cli_auto_set(); +merge @datamodel, cli_auto_merge(); +create @datamodel, cli_auto_create(); +delete("Delete a configuration item") @datamodel, cli_auto_del(); +validate("Validate changes"), cli_validate(); +commit("Commit the changes"), cli_commit(); +quit("Quit"), cli_quit(); +show("Show a particular state of the system"){ + configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{ + xml("Show configuration as XML"), cli_show_auto_mode("candidate", "xml", false, false); + cli("Show configuration as CLI commands"), cli_show_auto_mode("candidate", "cli", false, false, "report-all", "set "); + netconf("Show configuration as netconf edit-config operation"), cli_show_auto_mode("candidate", "netconf", false, false); + text("Show configuration as text"), cli_show_auto_mode("candidate", "text", false, false); + json("Show configuration as JSON"), cli_show_auto_mode("candidate", "json", false, false); + } + state("Show configuration and state"), cli_show_auto_mode("running", "xml", false, true); + compare("Compare candidate and running databases"), compare_dbs("running", "candidate", "xml"); +} +EOF + +# Check content of db +# Args: +# 0: dbname +function check_db() +{ + dbname=$1 + + sudo chmod 755 $dir/${dbname}_db + sudo rm -f $dir/x_db + cat < $dir/x_db + + + + + a + + + + +EOF + new "Check ${dbname}_db" + # ret=$(diff $dir/x_db $dir/${dbname}_db) + ret=$(diff $dir/x_db $dir/${dbname}_db) + if [ $? -ne 0 ]; then + # err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)" + err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)" + fi +} + +new "test params: -f $cfg" + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg +fi + +new "wait backend" +wait_backend + +new "Add mydata" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "amydata" "" + +new "Check mydata not in candidate" +check_db candidate + +new "Get mydata from candidate" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "amydata" + +new "Commit" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" + +new "Check mydata not in running" +check_db running + +new "Get mydata from running" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "amydata" + +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg +fi + +sudo rm -rf $dir + +new "endtest" +endtest diff --git a/yang/clixon/clixon-lib@2024-08-01.yang b/yang/clixon/clixon-lib@2024-08-01.yang new file mode 100644 index 00000000..c97adc61 --- /dev/null +++ b/yang/clixon/clixon-lib@2024-08-01.yang @@ -0,0 +1,553 @@ +module clixon-lib { + yang-version 1.1; + namespace "http://clicon.org/lib"; + prefix cl; + + import ietf-yang-types { + prefix yang; + } + import ietf-netconf-monitoring { + prefix ncm; + } + import ietf-yang-metadata { + prefix "md"; + } + organization + "Clicon / Clixon"; + + contact + "Olof Hagsand "; + + description + "***** BEGIN LICENSE BLOCK ***** + Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate) + + This file is part of CLIXON + + Licensed under the Apache License, Version 2.0 (the \"License\"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an \"AS IS\" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the \"GPL\"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + Clixon Netconf extensions for communication between clients and backend. + This scheme adds: + - Added values of RFC6022 transport identityref + - RPCs for debug, stats and process-control + - Informal description of attributes + + Clixon also extends NETCONF for internal use with some internal attributes. These + are not visible for external usage bit belongs to the namespace of this YANG. + The internal attributes are: + - content (also RESTCONF) + - depth (also RESTCONF) + - username + - autocommit + - copystartup + - transport (see RFC6022) + - source-host (see RFC6022) + - objectcreate + - objectexisted + - link # For split multiple XML files + "; + + revision 2024-08-01 { + description + "Added: list-pagination-partial-state + Added: system-only-config extension + Released in Clixon 7.2"; + } + revision 2024-04-01 { + description + "Added: debug bits type + Added: xmldb-split extension + Added: Default format + Released in Clixon 7.1"; + } + revision 2024-01-01 { + description + "Removed container creators from 6.5 + Released in 7.0"; + } + revision 2023-11-01 { + description + "Added ignore-compare extension + Added creator meta configuration + Removed obsolete extension autocli-op + Released in 6.5.0"; + } + revision 2023-05-01 { + description + "Restructured and extended stats rpc to schema mountpoints + Moved datastore-format typedef from clixon-config + "; + } + revision 2023-03-01 { + description + "Added creator meta-object"; + } + revision 2022-12-01 { + description + "Added values of RFC6022 transport identityref + Added description of internal netconf attributes"; + } + revision 2021-12-05 { + description + "Obsoleted: extension autocli-op"; + } + revision 2021-11-11 { + description + "Changed: RPC stats extended with YANG stats"; + } + revision 2021-03-08 { + description + "Changed: RPC process-control output to choice dependent on operation"; + } + revision 2020-12-30 { + description + "Changed: RPC process-control output parameter status to pid"; + } + revision 2020-12-08 { + description + "Added: autocli-op extension. + rpc process-control for process/daemon management + Released in clixon 4.9"; + } + revision 2020-04-23 { + description + "Added: stats RPC for clixon XML and memory statistics. + Added: restart-plugin RPC for restarting individual plugins without restarting backend."; + } + revision 2019-08-13 { + description + "No changes (reverted change)"; + } + revision 2019-06-05 { + description + "ping rpc added for liveness"; + } + revision 2019-01-02 { + description + "Released in Clixon 3.9"; + } + typedef service-operation { + type enumeration { + enum start { + description + "Start if not already running"; + } + enum stop { + description + "Stop if running"; + } + enum restart { + description + "Stop if running, then start"; + } + enum status { + description + "Check status"; + } + } + description + "Common operations that can be performed on a service"; + } + typedef datastore_format{ + description + "Datastore format (only xml and json implemented in actual data."; + type enumeration{ + enum xml{ + description + "Save and load xmldb as XML + More specifically, such a file looks like: ... provided + DATASTORE_TOP_SYMBOL is 'config'"; + } + enum json{ + description "Save and load xmldb as JSON"; + } + enum text{ + description "'Curly' C-like text format"; + } + enum cli{ + description "CLI format"; + } + enum default{ + description "Default format"; + } + } + } + typedef clixon_debug_t { + description + "Debug flags. + Flags are seperated into subject areas and detail + Can also be given directly as -D to clixon commands + Note there are also constants in the code that need to be in sync with these values"; + type bits { + /* Subjects: */ + bit default { + description "Default logs"; + position 0; + } + bit msg { + description "In/out messages"; + position 1; + } + bit init { + description "Initialization"; + position 2; + } + bit xml { + description "XML processing"; + position 3; + } + bit xpath { + description "XPath processing"; + position 4; + } + bit yang { + description "YANG processing"; + position 5; + } + bit backend { + description "Backend-specific"; + position 6; + } + bit cli { + description "CLI frontend"; + position 7; + } + bit netconf { + description "NETCONF frontend"; + position 8; + } + bit restconf { + description "RESTCONF frontend"; + position 9; + } + bit snmp { + description "SNMP frontend"; + position 10; + } + bit nacm { + description "NACM processing"; + position 11; + } + bit proc { + description "Process handling"; + position 12; + } + bit datastore { + description "Datastore xmldb management"; + position 13; + } + bit event { + description "Event processing"; + position 14; + } + bit rpc { + description "RPC handling"; + position 15; + } + bit stream { + description "Notification streams"; + position 16; + } + bit parse { + description "Parser: XML,YANG, etc"; + position 17; + } + bit app { + description "External applications"; + position 20; + } + bit app2 { + description "External application"; + position 21; + } + bit app3 { + description "External application 2"; + position 22; + } + /* Detail level: */ + bit detail { + description "Details: traces, parse trees, etc"; + position 24; + } + bit detail2 { + description "Extra details"; + position 25; + } + bit detail3 { + description "Probably more detail than you want"; + position 26; + } + } + } + identity snmp { + description + "SNMP"; + base ncm:transport; + } + identity netconf { + description + "Just NETCONF without specific underlying transport, + Clixon uses stdio for its netconf client and therefore does not know whether it is + invoked in a script, by a NETCONF/SSH subsystem, etc"; + base ncm:transport; + } + identity restconf { + description + "RESTCONF either as HTTP/1 or /2, TLS or not, reverse proxy (eg fcgi/nginx) or native"; + base ncm:transport; + } + identity cli { + description + "A CLI session"; + base ncm:transport; + } + extension list-pagination-partial-state { + description + "List should be partially read according to the clixon_pagination_cb_register API. + This is a performance enhancement of pagination state data. + This means that a special callback is used for retreiving list state which is aware of + offset/limit attributes. + In this way the non-config data can be partially read by the server, instead of reading + the whole state on every pagination request. + It affects only the server/backend-side + It only handles the offset and limit attributes, all other attributes, + such as where, sort-by, direction, etc, are ignored"; + } + extension ignore-compare { + description + "The object should be ignored when comparing device configs for equality. + The object should never be added, modified, or deleted on target. + Essentially a read-only object + One example is auto-created objects by the controller, such as uid."; + } + extension xmldb-split { + description + "When split configuration stores are used, ie CLICON_XMLDB_MULTI is set, + This extension marks where in the configuration tree, one file terminates + and a new sub-file is written. + A designer adds the 'xmldb-split' extension to a YANG node which should be split. + For example, a split could be made at mountpoints. + See also the 'link 'attribute. + "; + } + extension system-only-config { + description + "This extension marks which fields in the configuration tree should not be + saved to datastore and be removed from memory after commit. + Instead, the application provides a mechanism to save the system-only-config + in the system. + Note that the XML with these values will be remove from the datastore. The remaining XML + still needs to be valid XML wrt YANG. + An example of an invalid marking would be a list key. Because if the list keys are + removed, the remaining XML would no longer be valid wrt the YANG list"; + } + md:annotation creator { + type string; + description + "This annotation contains the name of a creator of an object. + One application is the clixon controller where multiple services can + create the same object. When such a service is deleted (or changed) one needs to keep + track of which service created what. + Limitations: only objects that are actually added or deleted. + A sub-object will not be noted"; + } + rpc debug { + description + "Set debug flags of backend. + Note only numerical values"; + input { + leaf level { + type uint32; + } + } + } + rpc ping { + description "Check aliveness of backend daemon."; + } + rpc stats { /* Could be moved to state */ + description "Clixon yang and datastore statistics."; + input { + leaf modules { + description "If enabled include per-module statistics"; + type boolean; + mandatory false; + } + } + output { + container global{ + description + "Clixon global statistics. + These are global counters incremented by new() and decreased by free() calls. + This number is higher than the sum of all datastore/module residing objects, since + objects may be used for other purposes than datastore/modules"; + leaf xmlnr{ + description + "Number of existing XML objects: number of residing xml/json objects + in the internal 'cxobj' representation."; + type uint64; + } + leaf yangnr{ + description + "Number of resident YANG objects. "; + type uint64; + } + } + container datastores{ + list datastore{ + description "Per datastore statistics for cxobj"; + key "name"; + leaf name{ + description "Name of datastore (eg running)."; + type string; + } + leaf nr{ + description "Number of XML objects. That is number of residing xml/json objects + in the internal 'cxobj' representation."; + type uint64; + } + leaf size{ + description "Size in bytes of internal datastore cache of datastore tree."; + type uint64; + } + } + } + container module-sets{ + list module-set{ + description "Statistics per domain, eg top-level and mount-points"; + key "name"; + leaf name{ + description "Name of YANG domain."; + type string; + } + leaf nr{ + description + "Total number of YANG objects in set"; + type uint64; + } + leaf size{ + description + "Total size in bytes of internal YANG object representation for module set"; + type uint64; + } + list module{ + description "Statistics per module (if modules set in input)"; + key "name"; + leaf name{ + description "Name of YANG module."; + type string; + } + leaf nr{ + description + "Number of YANG objects. That is number of residing YANG objects"; + type uint64; + } + leaf size{ + description + "Size in bytes of internal YANG object representation."; + type uint64; + } + } + } + } + } + } + rpc restart-plugin { + description "Restart specific backend plugins."; + input { + leaf-list plugin { + description "Name of plugin to restart"; + type string; + } + } + } + rpc process-control { + description + "Control a specific process or daemon: start/stop, etc. + This is for direct managing of a process by the backend. + Alternatively one can manage a daemon via systemd, containerd, kubernetes, etc."; + input { + leaf name { + description "Name of process"; + type string; + mandatory true; + } + leaf operation { + type service-operation; + mandatory true; + description + "One of the strings 'start', 'stop', 'restart', or 'status'."; + } + } + output { + choice result { + case status { + description + "Output from status rpc"; + leaf active { + description + "True if process is running, false if not. + More specifically, there is a process-id and it exists (in Linux: kill(pid,0). + Note that this is actual state and status is administrative state, + which means that changing the administrative state, eg stopped->running + may not immediately switch active to true."; + type boolean; + } + leaf description { + type string; + description "Description of process. This is a static string"; + } + leaf command { + type string; + description "Start command with arguments"; + } + leaf status { + description + "Administrative status (except on external kill where it enters stopped + directly from running): + stopped: pid=0, No process running + running: pid set, Process started and believed to be running + exiting: pid set, Process is killed by parent but not waited for"; + type string; + } + leaf starttime { + description "Time of starting process UTC"; + type yang:date-and-time; + } + leaf pid { + description "Process-id of main running process (if active)"; + type uint32; + } + } + case other { + description + "Output from start/stop/restart rpc"; + leaf ok { + type empty; + } + } + } + } + } +}