experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cleaned up after regression tests. New upgrade running docs.

Browse source
This commit is contained in:
Olof hagsand 2019-03-13 13:26:04 +01:00
parent 29535d5997
commit b3cd48468d
5 changed files with 113 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

52
datastore/clixon_xmldb_text.c
View file

@ -849,19 +849,19 @@ text_get(xmldb_handle xh,
} }
/*! Modify a base tree x0 with x1 with yang spec y according to operation op /*! Modify a base tree x0 with x1 with yang spec y according to operation op
* @param[in] th Datastore text handle * @param[in] th Datastore text handle
* @param[in] x0 Base xml tree (can be NULL in add scenarios) * @param[in] x0 Base xml tree (can be NULL in add scenarios)
* @param[in] y0 Yang spec corresponding to xml-node x0. NULL if x0 is NULL * @param[in] y0 Yang spec corresponding to xml-node x0. NULL if x0 is NULL
* @param[in] x0p Parent of x0 * @param[in] x0p Parent of x0
* @param[in] x1 xml tree which modifies base * @param[in] x1 XML tree which modifies base
* @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc * @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc
* @param[in] username User name of requestor for nacm * @param[in] username User name of requestor for nacm
* @param[in] xnacm NACM XML tree * @param[in] xnacm NACM XML tree (only if !permit)
* @param[in] permit If set, NACM has permitted this tree on an upper level * @param[in] permit If set, no NACM tests using xnacm required
* @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0. * @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0.
* @retval -1 Error * @retval -1 Error
* @retval 0 Failed (cbret set) * @retval 0 Failed (cbret set)
* @retval 1 OK * @retval 1 OK
* Assume x0 and x1 are same on entry and that y is the spec * Assume x0 and x1 are same on entry and that y is the spec
* @see text_modify_top * @see text_modify_top
*/ */
@ -1000,7 +1000,7 @@ text_modify(struct text_handle *th,
goto fail; goto fail;
} }
case OP_REPLACE: /* fall thru */ case OP_REPLACE: /* fall thru */
if (xnacm && !permit){ if (!permit && xnacm){
if ((ret = nacm_datanode_write(NULL, x1, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0) if ((ret = nacm_datanode_write(NULL, x1, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0)
goto done; goto done;
if (ret == 0) if (ret == 0)
@ -1022,7 +1022,7 @@ text_modify(struct text_handle *th,
if (y0->yn_keyword == Y_ANYXML){ if (y0->yn_keyword == Y_ANYXML){
if (op == OP_NONE) if (op == OP_NONE)
break; break;
if (xnacm && op==OP_MERGE && !permit){ if (op==OP_MERGE && !permit && xnacm){
if ((ret = nacm_datanode_write(NULL, x0, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0) if ((ret = nacm_datanode_write(NULL, x0, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0)
goto done; goto done;
if (ret == 0) if (ret == 0)
@ -1039,7 +1039,7 @@ text_modify(struct text_handle *th,
break; break;
} }
if (x0==NULL){ if (x0==NULL){
if (xnacm && op==OP_MERGE && !permit){ if (op==OP_MERGE && !permit && xnacm){
if ((ret = nacm_datanode_write(NULL, x0, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0) if ((ret = nacm_datanode_write(NULL, x0, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0)
goto done; goto done;
if (ret == 0) if (ret == 0)
@ -1143,18 +1143,18 @@ text_modify(struct text_handle *th,
} /* text_modify */ } /* text_modify */
/*! Modify a top-level base tree x0 with modification tree x1 /*! Modify a top-level base tree x0 with modification tree x1
* @param[in] th Datastore text handle * @param[in] th Datastore text handle
* @param[in] x0 Base xml tree (can be NULL in add scenarios) * @param[in] x0 Base xml tree (can be NULL in add scenarios)
* @param[in] x1 xml tree which modifies base * @param[in] x1 XML tree which modifies base
* @param[in] yspec Top-level yang spec (if y is NULL) * @param[in] yspec Top-level yang spec (if y is NULL)
* @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc * @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc
* @param[in] username User name of requestor for nacm * @param[in] username User name of requestor for nacm
* @param[in] permit If set, NACM has permitted this tree on an upper level * @param[in] xnacm NACM XML tree (only if !permit)
* @param[in] xnacm NACM XML tree * @param[in] permit If set, no NACM tests using xnacm required
* @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0. * @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0.
* @retval -1 Error * @retval -1 Error
* @retval 0 Failed (cbret set) * @retval 0 Failed (cbret set)
* @retval 1 OK * @retval 1 OK
* @see text_modify * @see text_modify
*/ */
static int static int

2
doc/FAQ.md
View file

@ -28,7 +28,7 @@
* [I want to program. How do I extend the example?](#i-want-to-program-how-do-i-extend-the-example) * [I want to program. How do I extend the example?](#i-want-to-program-how-do-i-extend-the-example)
* [How is a plugin initiated?](#how-is-a-plugin-initiated) * [How is a plugin initiated?](#how-is-a-plugin-initiated)
* [How do I write a commit function?](#how-do-i-write-a-commit-function) * [How do I write a commit function?](#how-do-i-write-a-commit-function)
* [How do I check what has changed on commit?](#how do i check what has changed on commit) * [How do I check what has changed on commit?](#how-do-i-check-what-has-changed-on-commit)
* [How do I access the XML tree?](#how-do-i-access-the-xml-tree) * [How do I access the XML tree?](#how-do-i-access-the-xml-tree)
* [How do I write a CLI callback function?](#how-do-i-write-a-cli-callback-function) * [How do I write a CLI callback function?](#how-do-i-write-a-cli-callback-function)
* [How do I write a validation function?](#how-do-i-write-a-validation-function) * [How do I write a validation function?](#how-do-i-write-a-validation-function)

9
doc/ROADMAP.md
View file

@ -7,17 +7,8 @@
- Handle revisions to data model. - Handle revisions to data model.
- Possibly draft-wang-netmod-module-revision-management-01 - Possibly draft-wang-netmod-module-revision-management-01
- See (startup.md) - See (startup.md)
- (DONE) NACM (RFC 8341)
- NACM support for create, read, update, delete operations
- ACM support for specifying a module name other than '*'
- (DONE)XML [Namespace handling](https://github.com/clicon/clixon/issues/49) (DONE)
## Medium prio: ## Medium prio:
- (DONE) Register extra callbacks on system Netconf messages. (Was:Support a plugin callback that is invoked when copy-config is called.)
- (DONE)Preserve CLI command history across sessions. The up/down arrows
- (DONE)Support for XML regex's.
- Currently Posix extended regular expressions
- (DONE) Input validation on custom RPCs/
- [Sanity checks](https://github.com/clicon/clixon/issues/47) - [Sanity checks](https://github.com/clicon/clixon/issues/47)
## Low prio: ## Low prio:

104
doc/startup.md
View file

@ -21,33 +21,41 @@ This document describes the configuration startup mechanism of the Clixon backen
* An upgrade callback when in-compatible XML is encountered * An upgrade callback when in-compatible XML is encountered
* A "failsafe" mode allowing a user to repair the startup on errors or failed validation. * A "failsafe" mode allowing a user to repair the startup on errors or failed validation.
Notes on this document:
* "database" and "datastore" are used interchangeably for the same XML or JSON file storing a configuration.
* For some scenarios, such a the "running" startup mode, a "temporary" datastore is used (called tmp_db). This file may have to be accessed out-of-band in failure scenarios.
## Modes ## Modes
When the Clixon backend starts, it can start in one of four modes: When the Clixon backend starts, it can start in one of four modes:
* `startup`: The configuration is loaded from a persistent `startup` database. This database is loaded, validated and committed into the running database. * `startup`: The configuration is loaded from a persistent `startup` datastore. The XML is loaded, parsed, validated and committed into the running database.
* `running`: Similar to `startup`, but instead the `running` database is used as persistent database. * `running`: Similar to `startup`, but instead the `running` datastore is used as a persistent database. The system copies the original running-db to a temporary store(tmp_db), and commits that temporary datastore into the (new) running datastore.
* `none`: No databases are touched - the system starts and loads existing running database without validation or commits. * `none`: No data stores are touched - the system starts and loads existing running datastore without validation or commits.
* `init`: Similar to `none`, but the running database is cleared before loading * `init`: Similar to `none`, but the running database is cleared before loading
`Startup` targets usecases where running db may be in memory and a `Startup` targets usecases where running db may be in memory and a
separate persistent storage (such as flash) is available. `Running` is separate persistent storage (such as flash) is available. `Running` is
for usecases when the running db is located in persistent The `none` for usecases when the running db is located in persistent. The `none`
and `init` modes are mostly for debugging, or restart at crashes or updates. and `init` modes are mostly for debugging, or restart at crashes or updates.
## Startup configuration ## Startup configuration
When the backend daemon is started in `startup` mode, the system loads When the backend daemon is started in `startup` mode, the system loads
the `startup` database. The `running` mode is very similar, the only the `startup` database.
difference is that the running database is copied (overwrites) the
startup database before this phase.
When loading the startup configuration, it is checked for parse The `running` mode is similar, the only difference is that the running
errors, the yang model-state is detected and the XML is validated database is copied into a temporary database which then acts as the
against the backend Yang models. startup store.
When loading the startup/tmp configuration, the following actions are performed by the system:
* It is checked for parse errors,
* the yang model-state is detected (if present)
* the XML is validated against the Yang models loaded in the backend (NB: may be different from the model-state).
If yang-models do not match, an `upgrade` callback is made. If yang-models do not match, an `upgrade` callback is made.
If any errors are detected, the backend tries to enter a `failsafe` mode. If any errors are detected, the backend enters a `failsafe` mode.
## Model-state ## Model-state
@ -123,24 +131,25 @@ Example upgrade callback:
}; };
``` ```
Note that this is simply a template for upgrade. Actual upgrading may Note that the example shown is only a template for an upgrade
be implememted by a user. function. Actual upgrading code may be implemented by a user.
If no action is made, and the XML is not upgraded, the next step of If no action is made by the upgrade calback, and thus the XML is not
the startup is made, which is XML/Yang validation. upgraded, the next step is XML/Yang validation.
An out-dated XML An out-dated XML may still pass validation and the system will go up
may still pass validation and the system will go up in normal state. in normal state.
However, if the validation fails, the backend will try to enter the However, if the validation fails, the backend will try to enter the
failsafe mode so that the user may perform manual upgarding of the failsafe mode so that the user may perform manual upgarding of the
configuration. configuration.
## Extra XML ## Extra XML
If validation succeeds and the startup configuration has been committed to the running database, a user may add "extra" XML. If the Yang validation succeeds and the startup configuration has been committed to the running database, a user may add "extra" XML.
There are two ways to add extra XML to running database after start. Note that this XML is not "committed" into running. There are two ways to add extra XML to running database after start. Note that this XML is "merged" into running, not "committed".
The first way is via a file. Assume you want to add this xml: The first way is via a file. Assume you want to add this xml:
``` ```
@ -154,9 +163,11 @@ clixon_backend ... -c extra.xml
``` ```
The second way is by programming the plugin_reset() in the backend The second way is by programming the plugin_reset() in the backend
plugin. The example code contains an example on how to do this (see plugin_reset() in example_backend.c). plugin. The example code contains an example on how to do this (see
plugin_reset() in example_backend.c).
The extra-xml feature is not available if startup mode is `none`. It will also not occur in failsafe mode. The extra-xml feature is not available if startup mode is `none`. It
will also not occur in failsafe mode.
## Startup status ## Startup status
@ -174,13 +185,18 @@ in `CLICON_XMLDB_DIR/failsafe_db`. If such a config is not found, the
backend terminates. backend terminates.
If the failsafe is found, the failsafe config is loaded and If the failsafe is found, the failsafe config is loaded and
committed into the running db. The `startup` database will contain syntax committed into the running db.
errors or invalidated XML.
A user can repair the `startup` configuration and either restart the If the startup mode was `startup`, the `startup` database will
backend or copy the startup configuration to candidate and the commit. contain syntax errors or invalidated XML.
Note that the if the startup configuration contains syntactic errors If the startup mode was `running`, the the `tmp` database will contain
syntax errors or invalidated XML.
A user can repair a broken configuration and either restart the
backend or copy the repaired configuration to candidate and then commit.
Note that if the broken configuration contains syntactic errors
(eg `STARTUP_ERR`) you cannot access the startup via Restconf or (eg `STARTUP_ERR`) you cannot access the startup via Restconf or
Netconf operations since the XML may be broken. Netconf operations since the XML may be broken.
@ -192,22 +208,25 @@ and then copy/commit it via CLI, Netconf or Restconf.
This section contains "pseudo" flowcharts showing the dynamics of This section contains "pseudo" flowcharts showing the dynamics of
the configuration databases in the startup phase. the configuration databases in the startup phase.
The flowchart starts in one of the the modes (non, init, startup, running): The flowchart starts in one of the modes (none, init, startup, running):
### init mode
Starting in init mode:
``` ```
reset reset
running |--------+------------> GOTO EXTRA XML running |--------+------------> GOTO EXTRA XML
``` ```
Start in running mode: ### running mode
```
running ----+
\ copy
startup +------------> GOTO STARTUP
``` ```
Starting in startup mode: running ----+ |----------+--------> GOTO EXTRA XML
\ copy parse validate OK / commit
tmp ------+-------+------+-----------+
```
### startup mode
``` ```
reset reset
running |--------+------------> GOTO EXTRA XML running |--------+------------> GOTO EXTRA XML
@ -215,29 +234,36 @@ running |--------+------------> GOTO EXTRA XML
startup -------+--+-------+------------+ startup -------+--+-------+------------+
``` ```
If validation of startup fails: ### Failure
``` ```
failsafe ----------------------+ failsafe ----------------------+
reset \ commit reset \ commit
running |-------+---------------> GOTO EXTRA XML running |-------+---------------> GOTO SYSTEM UP
parse validate fail parse validate fail
startup ---+-------------------------------------> INVALID XML tmp/startup --+-----+---------------------------------> INVALID XML
``` ```
Load EXTRA XML: ### Extra XML
``` ```
running -----------------+----+------> GOTO SYSTEM UP running -----------------+----+------> GOTO SYSTEM UP
reset loadfile / merge reset loadfile / merge
tmp |-------+-----+-----+ tmp |-------+-----+-----+
``` ```
SYSTEM UP: ### System UP
``` ```
running ----+-----------------------> RUNNING running ----+-----------------------> RUNNING
\ copy \ copy
candidate +---------------------> CANDIDATE candidate +---------------------> CANDIDATE
``` ```
### Invalid XML
repair restart
tmp/startup --------+---------+----------------------->
## Thanks ## Thanks
Thanks matt smith and dave cornejo for input Thanks matt smith and dave cornejo for input

30
test/test_upgrade.sh
View file

@ -240,14 +240,17 @@ cat <<EOF > $dir/compat-err.xml
</config> </config>
EOF EOF
# Start system in $mode with existing (old) configuration in $db #! Start system in given mode and check database contents
# mode is one of: init, none, running, or startup # Before script is called populate running_db and startup_db
# db is one of: running_db or startup_db # @param[in] modstate Boolean: Tag datastores with RFC 7895 YANG Module Library
# @param[in] mode Startup mode: init, none, running, or startup
# @param[in] exprun Expected content of running-db
# @param[in] expstart Check startup database or not if ""
runtest(){ runtest(){
modstate=$1 modstate=$1
mode=$2 mode=$2
exprun=$3 # Expected running exprun=$3
expstartup=$4 # Expected startup expstart=$4
new "test params: -f $cfg" new "test params: -f $cfg"
# Bring your own backend # Bring your own backend
@ -268,11 +271,14 @@ runtest(){
sleep $BETIMEOUT sleep $BETIMEOUT
fi fi
new "Get running" new "Check running db content"
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$exprun</rpc-reply>]]>]]>$" expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><running/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$exprun</rpc-reply>]]>]]>$"
new "Get startup" # If given check startup db XML
expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>' "^<rpc-reply>$expstartup</rpc-reply>]]>]]>$" if [ -n "$expstart" ]; then
new "Check startup db content"
expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>" "^<rpc-reply>$expstart</rpc-reply>]]>]]>$"
fi
if [ $BE -ne 0 ]; then if [ $BE -ne 0 ]; then
new "Kill backend" new "Kill backend"
@ -315,8 +321,8 @@ fi
new "3. Load compatible running valid running (rest of tests are startup)" new "3. Load compatible running valid running (rest of tests are startup)"
(cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
(cd $dir; cp compat-valid.xml running_db) (cd $dir; cp compat-valid.xml running_db)
(cd $dir; cp compat-valid.xml startup_db) # XXX runtest true running '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' ''
runtest true running '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' #'<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>'
new "4. Load non-compat valid startup" new "4. Load non-compat valid startup"
(cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
@ -332,7 +338,9 @@ new "6. Load non-compat invalid running. Enter failsafe, startup invalid."
(cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
(cd $dir; cp non-compat-invalid.xml running_db) (cd $dir; cp non-compat-invalid.xml running_db)
(cd $dir; cp non-compat-valid.xml startup_db) # XXX tmp (cd $dir; cp non-compat-valid.xml startup_db) # XXX tmp
#runtest true running '<data><a1 xmlns="urn:example:a">always work</a1></data>' '<data><a0 xmlns="urn:example:a">old version</a0><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b><c xmlns="urn:example:c">bla bla</c></data>' runtest true running '<data><a1 xmlns="urn:example:a">always work</a1></data>' ''
#'<data><a0 xmlns="urn:example:a">old version</a0><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b><c xmlns="urn:example:c">bla bla</c></data>'
new "7. Load compatible invalid startup." new "7. Load compatible invalid startup."
(cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases