experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

First version of system-only-config

Browse source 
A new extension added in clixon-lib.yang
A conditional in xmldb_dump to not write system-only data to datastores
A test for verifying system-only data is not written at edit-commit
This commit is contained in:
Olof hagsand 2024-07-31 14:32:48 +02:00
parent 515d30bdd7
commit aec0a5fc3f
3 changed files with 787 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

15
lib/src/clixon_xml_io.c
View file

@ -233,7 +233,7 @@ xml2file_recurse(FILE *f,
int haselement;
char *val;
char *encstr = NULL; /* xml encoded string */
int exist = 0;
int exist;
yang_stmt *y;
int level1;
int tag = 0;
@ -244,11 +244,22 @@ xml2file_recurse(FILE *f,
if (x == NULL)
goto ok;
y = xml_spec(x);
/* Check if system-only, then do not write to datastore */
if (y != NULL) {
exist = 0;
if (yang_extension_value(y, "system-only-config", CLIXON_LIB_NS, &exist, NULL) < 0)
goto done;
if (exist){
goto ok;
}
}
level1 = level*PRETTYPRINT_INDENT;
if (prefix)
level1 -= strlen(prefix);
if ((y = xml_spec(x)) != NULL){
if (y != NULL){
if (autocliext){
exist = 0;
if (yang_extension_value(y, "hide-show", CLIXON_AUTOCLI_NS, &exist, NULL) < 0)
goto done;
if (exist)

221
test/test_datastore_system_only.sh Executable file
View file

@ -0,0 +1,221 @@
#!/usr/bin/env bash
# Datastore system only config test
# see https://github.com/clicon/clixon/pull/534 and extension system-only-config
# Test uses a "standard" yang and a "local" yang which augmanets the standard
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
APPNAME=example
# include err() and new() functions and creates $dir
cfg=$dir/conf_yang.xml
clispec=$dir/automode.cli
fstandard=$dir/clixon-standard.yang
flocal=$dir/clixon-local.yang
CFD=$dir/conf.d
test -d $CFD || mkdir -p $CFD
AUTOCLI=$(autocli_config clixon-\* kw-nokey false)
# Well-known digest of mount-point xpath
subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_CONFIGDIR>$CFD</CLICON_CONFIGDIR>
<CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
<CLICON_YANG_DIR>${dir}</CLICON_YANG_DIR>
<CLICON_YANG_MAIN_FILE>$flocal</CLICON_YANG_MAIN_FILE>
<CLICON_YANG_LIBRARY>true</CLICON_YANG_LIBRARY>
<CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
<CLICON_SOCK>/usr/local/var/run/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
<CLICON_BACKEND_PIDFILE>/usr/local/var/run/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_NETCONF_MONITORING>true</CLICON_NETCONF_MONITORING>
<CLICON_VALIDATE_STATE_XML>true</CLICON_VALIDATE_STATE_XML>
<CLICON_STREAM_DISCOVERY_RFC5277>true</CLICON_STREAM_DISCOVERY_RFC5277>
<CLICON_YANG_SCHEMA_MOUNT>true</CLICON_YANG_SCHEMA_MOUNT>
</clixon-config>
EOF
cat <<EOF > $CFD/autocli.xml
<clixon-config xmlns="http://clicon.org/config">
<autocli>
<module-default>false</module-default>
<list-keyword-default>kw-nokey</list-keyword-default>
<grouping-treeref>true</grouping-treeref>
<rule>
<name>include clixon</name>
<operation>enable</operation>
<module-name>clixon-*</module-name>
</rule>
</autocli>
</clixon-config>
EOF
# A "standard" YANG
cat <<EOF > $fstandard
module clixon-standard{
yang-version 1.1;
namespace "urn:example:std";
prefix std;
grouping system-only-group {
description
"A grouping containing a system-only field, corresponding to
a standard module, which gets augmented by a local yang";
leaf system-only-data {
description
"System-only config data";
type string;
}
}
grouping store-grouping {
container keys {
list key {
key "name";
leaf name {
type string;
}
uses system-only-group;
}
}
}
container store {
description "top-level";
uses store-grouping;
}
}
EOF
# A "local" YANG
cat <<EOF > $flocal
module clixon-mount1{
yang-version 1.1;
namespace "urn:example:local";
prefix local;
import clixon-lib {
prefix cl;
}
import clixon-standard {
prefix std;
}
augment "/std:store/std:keys/std:key/std:system-only-data" {
cl:system-only-config {
description
"Marks system-only-config data";
}
}
}
EOF
cat <<EOF > $clispec
CLICON_MODE="example";
CLICON_PROMPT="%U@%H %W> ";
CLICON_PLUGIN="example_cli";
# Autocli syntax tree operations
set @datamodel, cli_auto_set();
merge @datamodel, cli_auto_merge();
create @datamodel, cli_auto_create();
delete("Delete a configuration item") @datamodel, cli_auto_del();
validate("Validate changes"), cli_validate();
commit("Commit the changes"), cli_commit();
quit("Quit"), cli_quit();
show("Show a particular state of the system"){
configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{
xml("Show configuration as XML"), cli_show_auto_mode("candidate", "xml", false, false);
cli("Show configuration as CLI commands"), cli_show_auto_mode("candidate", "cli", false, false, "report-all", "set ");
netconf("Show configuration as netconf edit-config operation"), cli_show_auto_mode("candidate", "netconf", false, false);
text("Show configuration as text"), cli_show_auto_mode("candidate", "text", false, false);
json("Show configuration as JSON"), cli_show_auto_mode("candidate", "json", false, false);
}
state("Show configuration and state"), cli_show_auto_mode("running", "xml", false, true);
compare("Compare candidate and running databases"), compare_dbs("running", "candidate", "xml");
}
EOF
# Check content of db
# Args:
# 0: dbname
function check_db()
{
dbname=$1
sudo chmod 755 $dir/${dbname}_db
sudo rm -f $dir/x_db
cat <<EOF > $dir/x_db
<config>
<store xmlns="urn:example:std">
<keys>
<key>
<name>a</name>
</key>
</keys>
</store>
</config>
EOF
new "Check ${dbname}_db"
# ret=$(diff $dir/x_db $dir/${dbname}_db)
ret=$(diff $dir/x_db $dir/${dbname}_db)
if [ $? -ne 0 ]; then
# err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
fi
}
new "test params: -f $cfg"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s init -f $cfg"
start_backend -s init -f $cfg
fi
new "wait backend"
wait_backend
new "Add mydata"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></config></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check mydata not in candidate"
check_db candidate
new "Get mydata from candidate"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><candidate/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
new "Commit"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check mydata not in running"
check_db running
new "Get mydata from running"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
sudo rm -rf $dir
new "endtest"
endtest

553
yang/clixon/clixon-lib@2024-08-01.yang Normal file
View file

@ -0,0 +1,553 @@
module clixon-lib {
yang-version 1.1;
namespace "http://clicon.org/lib";
prefix cl;
import ietf-yang-types {
prefix yang;
}
import ietf-netconf-monitoring {
prefix ncm;
}
import ietf-yang-metadata {
prefix "md";
}
organization
"Clicon / Clixon";
contact
"Olof Hagsand <olof@hagsand.se>";
description
"***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC(Netgate)
This file is part of CLIXON
Licensed under the Apache License, Version 2.0 (the \"License\");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an \"AS IS\" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the \"GPL\"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
Clixon Netconf extensions for communication between clients and backend.
This scheme adds:
- Added values of RFC6022 transport identityref
- RPCs for debug, stats and process-control
- Informal description of attributes
Clixon also extends NETCONF for internal use with some internal attributes. These
are not visible for external usage bit belongs to the namespace of this YANG.
The internal attributes are:
- content (also RESTCONF)
- depth (also RESTCONF)
- username
- autocommit
- copystartup
- transport (see RFC6022)
- source-host (see RFC6022)
- objectcreate
- objectexisted
- link # For split multiple XML files
";
revision 2024-08-01 {
description
"Added: list-pagination-partial-state
Added: system-only-config extension
Released in Clixon 7.2";
}
revision 2024-04-01 {
description
"Added: debug bits type
Added: xmldb-split extension
Added: Default format
Released in Clixon 7.1";
}
revision 2024-01-01 {
description
"Removed container creators from 6.5
Released in 7.0";
}
revision 2023-11-01 {
description
"Added ignore-compare extension
Added creator meta configuration
Removed obsolete extension autocli-op
Released in 6.5.0";
}
revision 2023-05-01 {
description
"Restructured and extended stats rpc to schema mountpoints
Moved datastore-format typedef from clixon-config
";
}
revision 2023-03-01 {
description
"Added creator meta-object";
}
revision 2022-12-01 {
description
"Added values of RFC6022 transport identityref
Added description of internal netconf attributes";
}
revision 2021-12-05 {
description
"Obsoleted: extension autocli-op";
}
revision 2021-11-11 {
description
"Changed: RPC stats extended with YANG stats";
}
revision 2021-03-08 {
description
"Changed: RPC process-control output to choice dependent on operation";
}
revision 2020-12-30 {
description
"Changed: RPC process-control output parameter status to pid";
}
revision 2020-12-08 {
description
"Added: autocli-op extension.
rpc process-control for process/daemon management
Released in clixon 4.9";
}
revision 2020-04-23 {
description
"Added: stats RPC for clixon XML and memory statistics.
Added: restart-plugin RPC for restarting individual plugins without restarting backend.";
}
revision 2019-08-13 {
description
"No changes (reverted change)";
}
revision 2019-06-05 {
description
"ping rpc added for liveness";
}
revision 2019-01-02 {
description
"Released in Clixon 3.9";
}
typedef service-operation {
type enumeration {
enum start {
description
"Start if not already running";
}
enum stop {
description
"Stop if running";
}
enum restart {
description
"Stop if running, then start";
}
enum status {
description
"Check status";
}
}
description
"Common operations that can be performed on a service";
}
typedef datastore_format{
description
"Datastore format (only xml and json implemented in actual data.";
type enumeration{
enum xml{
description
"Save and load xmldb as XML
More specifically, such a file looks like: <config>...</config> provided
DATASTORE_TOP_SYMBOL is 'config'";
}
enum json{
description "Save and load xmldb as JSON";
}
enum text{
description "'Curly' C-like text format";
}
enum cli{
description "CLI format";
}
enum default{
description "Default format";
}
}
}
typedef clixon_debug_t {
description
"Debug flags.
Flags are seperated into subject areas and detail
Can also be given directly as -D <flag> to clixon commands
Note there are also constants in the code that need to be in sync with these values";
type bits {
/* Subjects: */
bit default {
description "Default logs";
position 0;
}
bit msg {
description "In/out messages";
position 1;
}
bit init {
description "Initialization";
position 2;
}
bit xml {
description "XML processing";
position 3;
}
bit xpath {
description "XPath processing";
position 4;
}
bit yang {
description "YANG processing";
position 5;
}
bit backend {
description "Backend-specific";
position 6;
}
bit cli {
description "CLI frontend";
position 7;
}
bit netconf {
description "NETCONF frontend";
position 8;
}
bit restconf {
description "RESTCONF frontend";
position 9;
}
bit snmp {
description "SNMP frontend";
position 10;
}
bit nacm {
description "NACM processing";
position 11;
}
bit proc {
description "Process handling";
position 12;
}
bit datastore {
description "Datastore xmldb management";
position 13;
}
bit event {
description "Event processing";
position 14;
}
bit rpc {
description "RPC handling";
position 15;
}
bit stream {
description "Notification streams";
position 16;
}
bit parse {
description "Parser: XML,YANG, etc";
position 17;
}
bit app {
description "External applications";
position 20;
}
bit app2 {
description "External application";
position 21;
}
bit app3 {
description "External application 2";
position 22;
}
/* Detail level: */
bit detail {
description "Details: traces, parse trees, etc";
position 24;
}
bit detail2 {
description "Extra details";
position 25;
}
bit detail3 {
description "Probably more detail than you want";
position 26;
}
}
}
identity snmp {
description
"SNMP";
base ncm:transport;
}
identity netconf {
description
"Just NETCONF without specific underlying transport,
Clixon uses stdio for its netconf client and therefore does not know whether it is
invoked in a script, by a NETCONF/SSH subsystem, etc";
base ncm:transport;
}
identity restconf {
description
"RESTCONF either as HTTP/1 or /2, TLS or not, reverse proxy (eg fcgi/nginx) or native";
base ncm:transport;
}
identity cli {
description
"A CLI session";
base ncm:transport;
}
extension list-pagination-partial-state {
description
"List should be partially read according to the clixon_pagination_cb_register API.
This is a performance enhancement of pagination state data.
This means that a special callback is used for retreiving list state which is aware of
offset/limit attributes.
In this way the non-config data can be partially read by the server, instead of reading
the whole state on every pagination request.
It affects only the server/backend-side
It only handles the offset and limit attributes, all other attributes,
such as where, sort-by, direction, etc, are ignored";
}
extension ignore-compare {
description
"The object should be ignored when comparing device configs for equality.
The object should never be added, modified, or deleted on target.
Essentially a read-only object
One example is auto-created objects by the controller, such as uid.";
}
extension xmldb-split {
description
"When split configuration stores are used, ie CLICON_XMLDB_MULTI is set,
This extension marks where in the configuration tree, one file terminates
and a new sub-file is written.
A designer adds the 'xmldb-split' extension to a YANG node which should be split.
For example, a split could be made at mountpoints.
See also the 'link 'attribute.
";
}
extension system-only-config {
description
"This extension marks which fields in the configuration tree should not be
saved to datastore and be removed from memory after commit.
Instead, the application provides a mechanism to save the system-only-config
in the system.
Note that the XML with these values will be remove from the datastore. The remaining XML
still needs to be valid XML wrt YANG.
An example of an invalid marking would be a list key. Because if the list keys are
removed, the remaining XML would no longer be valid wrt the YANG list";
}
md:annotation creator {
type string;
description
"This annotation contains the name of a creator of an object.
One application is the clixon controller where multiple services can
create the same object. When such a service is deleted (or changed) one needs to keep
track of which service created what.
Limitations: only objects that are actually added or deleted.
A sub-object will not be noted";
}
rpc debug {
description
"Set debug flags of backend.
Note only numerical values";
input {
leaf level {
type uint32;
}
}
}
rpc ping {
description "Check aliveness of backend daemon.";
}
rpc stats { /* Could be moved to state */
description "Clixon yang and datastore statistics.";
input {
leaf modules {
description "If enabled include per-module statistics";
type boolean;
mandatory false;
}
}
output {
container global{
description
"Clixon global statistics.
These are global counters incremented by new() and decreased by free() calls.
This number is higher than the sum of all datastore/module residing objects, since
objects may be used for other purposes than datastore/modules";
leaf xmlnr{
description
"Number of existing XML objects: number of residing xml/json objects
in the internal 'cxobj' representation.";
type uint64;
}
leaf yangnr{
description
"Number of resident YANG objects. ";
type uint64;
}
}
container datastores{
list datastore{
description "Per datastore statistics for cxobj";
key "name";
leaf name{
description "Name of datastore (eg running).";
type string;
}
leaf nr{
description "Number of XML objects. That is number of residing xml/json objects
in the internal 'cxobj' representation.";
type uint64;
}
leaf size{
description "Size in bytes of internal datastore cache of datastore tree.";
type uint64;
}
}
}
container module-sets{
list module-set{
description "Statistics per domain, eg top-level and mount-points";
key "name";
leaf name{
description "Name of YANG domain.";
type string;
}
leaf nr{
description
"Total number of YANG objects in set";
type uint64;
}
leaf size{
description
"Total size in bytes of internal YANG object representation for module set";
type uint64;
}
list module{
description "Statistics per module (if modules set in input)";
key "name";
leaf name{
description "Name of YANG module.";
type string;
}
leaf nr{
description
"Number of YANG objects. That is number of residing YANG objects";
type uint64;
}
leaf size{
description
"Size in bytes of internal YANG object representation.";
type uint64;
}
}
}
}
}
}
rpc restart-plugin {
description "Restart specific backend plugins.";
input {
leaf-list plugin {
description "Name of plugin to restart";
type string;
}
}
}
rpc process-control {
description
"Control a specific process or daemon: start/stop, etc.
This is for direct managing of a process by the backend.
Alternatively one can manage a daemon via systemd, containerd, kubernetes, etc.";
input {
leaf name {
description "Name of process";
type string;
mandatory true;
}
leaf operation {
type service-operation;
mandatory true;
description
"One of the strings 'start', 'stop', 'restart', or 'status'.";
}
}
output {
choice result {
case status {
description
"Output from status rpc";
leaf active {
description
"True if process is running, false if not.
More specifically, there is a process-id and it exists (in Linux: kill(pid,0).
Note that this is actual state and status is administrative state,
which means that changing the administrative state, eg stopped->running
may not immediately switch active to true.";
type boolean;
}
leaf description {
type string;
description "Description of process. This is a static string";
}
leaf command {
type string;
description "Start command with arguments";
}
leaf status {
description
"Administrative status (except on external kill where it enters stopped
directly from running):
stopped: pid=0, No process running
running: pid set, Process started and believed to be running
exiting: pid set, Process is killed by parent but not waited for";
type string;
}
leaf starttime {
description "Time of starting process UTC";
type yang:date-and-time;
}
leaf pid {
description "Process-id of main running process (if active)";
type uint32;
}
}
case other {
description
"Output from start/stop/restart rpc";
leaf ok {
type empty;
}
}
}
}
}
}