From b3cd48468d81858c9b611770fd76e7e2c1289288 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 13 Mar 2019 13:26:04 +0100 Subject: [PATCH] Cleaned up after regression tests. New upgrade running docs. --- datastore/clixon_xmldb_text.c | 52 ++++++++--------- doc/FAQ.md | 2 +- doc/ROADMAP.md | 9 --- doc/startup.md | 106 +++++++++++++++++++++------------- test/test_upgrade.sh | 32 ++++++---- 5 files changed, 113 insertions(+), 88 deletions(-) diff --git a/datastore/clixon_xmldb_text.c b/datastore/clixon_xmldb_text.c index 742a5550..cffd6f17 100644 --- a/datastore/clixon_xmldb_text.c +++ b/datastore/clixon_xmldb_text.c @@ -849,19 +849,19 @@ text_get(xmldb_handle xh, } /*! Modify a base tree x0 with x1 with yang spec y according to operation op - * @param[in] th Datastore text handle - * @param[in] x0 Base xml tree (can be NULL in add scenarios) - * @param[in] y0 Yang spec corresponding to xml-node x0. NULL if x0 is NULL - * @param[in] x0p Parent of x0 - * @param[in] x1 xml tree which modifies base - * @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc + * @param[in] th Datastore text handle + * @param[in] x0 Base xml tree (can be NULL in add scenarios) + * @param[in] y0 Yang spec corresponding to xml-node x0. NULL if x0 is NULL + * @param[in] x0p Parent of x0 + * @param[in] x1 XML tree which modifies base + * @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc * @param[in] username User name of requestor for nacm - * @param[in] xnacm NACM XML tree - * @param[in] permit If set, NACM has permitted this tree on an upper level - * @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0. - * @retval -1 Error - * @retval 0 Failed (cbret set) - * @retval 1 OK + * @param[in] xnacm NACM XML tree (only if !permit) + * @param[in] permit If set, no NACM tests using xnacm required + * @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0. + * @retval -1 Error + * @retval 0 Failed (cbret set) + * @retval 1 OK * Assume x0 and x1 are same on entry and that y is the spec * @see text_modify_top */ @@ -1000,7 +1000,7 @@ text_modify(struct text_handle *th, goto fail; } case OP_REPLACE: /* fall thru */ - if (xnacm && !permit){ + if (!permit && xnacm){ if ((ret = nacm_datanode_write(NULL, x1, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0) goto done; if (ret == 0) @@ -1022,7 +1022,7 @@ text_modify(struct text_handle *th, if (y0->yn_keyword == Y_ANYXML){ if (op == OP_NONE) break; - if (xnacm && op==OP_MERGE && !permit){ + if (op==OP_MERGE && !permit && xnacm){ if ((ret = nacm_datanode_write(NULL, x0, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0) goto done; if (ret == 0) @@ -1039,7 +1039,7 @@ text_modify(struct text_handle *th, break; } if (x0==NULL){ - if (xnacm && op==OP_MERGE && !permit){ + if (op==OP_MERGE && !permit && xnacm){ if ((ret = nacm_datanode_write(NULL, x0, x0?NACM_UPDATE:NACM_CREATE, username, xnacm, cbret)) < 0) goto done; if (ret == 0) @@ -1143,18 +1143,18 @@ text_modify(struct text_handle *th, } /* text_modify */ /*! Modify a top-level base tree x0 with modification tree x1 - * @param[in] th Datastore text handle - * @param[in] x0 Base xml tree (can be NULL in add scenarios) - * @param[in] x1 xml tree which modifies base - * @param[in] yspec Top-level yang spec (if y is NULL) - * @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc + * @param[in] th Datastore text handle + * @param[in] x0 Base xml tree (can be NULL in add scenarios) + * @param[in] x1 XML tree which modifies base + * @param[in] yspec Top-level yang spec (if y is NULL) + * @param[in] op OP_MERGE, OP_REPLACE, OP_REMOVE, etc * @param[in] username User name of requestor for nacm - * @param[in] permit If set, NACM has permitted this tree on an upper level - * @param[in] xnacm NACM XML tree - * @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0. - * @retval -1 Error - * @retval 0 Failed (cbret set) - * @retval 1 OK + * @param[in] xnacm NACM XML tree (only if !permit) + * @param[in] permit If set, no NACM tests using xnacm required + * @param[out] cbret Initialized cligen buffer. Contains return XML if retval is 0. + * @retval -1 Error + * @retval 0 Failed (cbret set) + * @retval 1 OK * @see text_modify */ static int diff --git a/doc/FAQ.md b/doc/FAQ.md index 63593ccf..0f2841b2 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -28,7 +28,7 @@ * [I want to program. How do I extend the example?](#i-want-to-program-how-do-i-extend-the-example) * [How is a plugin initiated?](#how-is-a-plugin-initiated) * [How do I write a commit function?](#how-do-i-write-a-commit-function) - * [How do I check what has changed on commit?](#how do i check what has changed on commit) + * [How do I check what has changed on commit?](#how-do-i-check-what-has-changed-on-commit) * [How do I access the XML tree?](#how-do-i-access-the-xml-tree) * [How do I write a CLI callback function?](#how-do-i-write-a-cli-callback-function) * [How do I write a validation function?](#how-do-i-write-a-validation-function) diff --git a/doc/ROADMAP.md b/doc/ROADMAP.md index 13b4f458..e7d2a3d2 100644 --- a/doc/ROADMAP.md +++ b/doc/ROADMAP.md @@ -7,17 +7,8 @@ - Handle revisions to data model. - Possibly draft-wang-netmod-module-revision-management-01 - See (startup.md) -- (DONE) NACM (RFC 8341) - - NACM support for create, read, update, delete operations - - ACM support for specifying a module name other than '*' -- (DONE)XML [Namespace handling](https://github.com/clicon/clixon/issues/49) (DONE) ## Medium prio: -- (DONE) Register extra callbacks on system Netconf messages. (Was:Support a plugin callback that is invoked when copy-config is called.) -- (DONE)Preserve CLI command history across sessions. The up/down arrows -- (DONE)Support for XML regex's. - - Currently Posix extended regular expressions -- (DONE) Input validation on custom RPCs/ - [Sanity checks](https://github.com/clicon/clixon/issues/47) ## Low prio: diff --git a/doc/startup.md b/doc/startup.md index 99f5f191..10ef3b85 100644 --- a/doc/startup.md +++ b/doc/startup.md @@ -21,33 +21,41 @@ This document describes the configuration startup mechanism of the Clixon backen * An upgrade callback when in-compatible XML is encountered * A "failsafe" mode allowing a user to repair the startup on errors or failed validation. +Notes on this document: +* "database" and "datastore" are used interchangeably for the same XML or JSON file storing a configuration. +* For some scenarios, such a the "running" startup mode, a "temporary" datastore is used (called tmp_db). This file may have to be accessed out-of-band in failure scenarios. + ## Modes When the Clixon backend starts, it can start in one of four modes: -* `startup`: The configuration is loaded from a persistent `startup` database. This database is loaded, validated and committed into the running database. -* `running`: Similar to `startup`, but instead the `running` database is used as persistent database. -* `none`: No databases are touched - the system starts and loads existing running database without validation or commits. +* `startup`: The configuration is loaded from a persistent `startup` datastore. The XML is loaded, parsed, validated and committed into the running database. +* `running`: Similar to `startup`, but instead the `running` datastore is used as a persistent database. The system copies the original running-db to a temporary store(tmp_db), and commits that temporary datastore into the (new) running datastore. +* `none`: No data stores are touched - the system starts and loads existing running datastore without validation or commits. * `init`: Similar to `none`, but the running database is cleared before loading `Startup` targets usecases where running db may be in memory and a separate persistent storage (such as flash) is available. `Running` is -for usecases when the running db is located in persistent The `none` +for usecases when the running db is located in persistent. The `none` and `init` modes are mostly for debugging, or restart at crashes or updates. ## Startup configuration When the backend daemon is started in `startup` mode, the system loads -the `startup` database. The `running` mode is very similar, the only -difference is that the running database is copied (overwrites) the -startup database before this phase. +the `startup` database. -When loading the startup configuration, it is checked for parse -errors, the yang model-state is detected and the XML is validated -against the backend Yang models. +The `running` mode is similar, the only difference is that the running +database is copied into a temporary database which then acts as the +startup store. + +When loading the startup/tmp configuration, the following actions are performed by the system: + +* It is checked for parse errors, +* the yang model-state is detected (if present) +* the XML is validated against the Yang models loaded in the backend (NB: may be different from the model-state). If yang-models do not match, an `upgrade` callback is made. -If any errors are detected, the backend tries to enter a `failsafe` mode. +If any errors are detected, the backend enters a `failsafe` mode. ## Model-state @@ -123,24 +131,25 @@ Example upgrade callback: }; ``` -Note that this is simply a template for upgrade. Actual upgrading may -be implememted by a user. +Note that the example shown is only a template for an upgrade +function. Actual upgrading code may be implemented by a user. -If no action is made, and the XML is not upgraded, the next step of -the startup is made, which is XML/Yang validation. +If no action is made by the upgrade calback, and thus the XML is not +upgraded, the next step is XML/Yang validation. -An out-dated XML -may still pass validation and the system will go up in normal state. +An out-dated XML may still pass validation and the system will go up +in normal state. However, if the validation fails, the backend will try to enter the failsafe mode so that the user may perform manual upgarding of the configuration. + ## Extra XML -If validation succeeds and the startup configuration has been committed to the running database, a user may add "extra" XML. +If the Yang validation succeeds and the startup configuration has been committed to the running database, a user may add "extra" XML. -There are two ways to add extra XML to running database after start. Note that this XML is not "committed" into running. +There are two ways to add extra XML to running database after start. Note that this XML is "merged" into running, not "committed". The first way is via a file. Assume you want to add this xml: ``` @@ -154,9 +163,11 @@ clixon_backend ... -c extra.xml ``` The second way is by programming the plugin_reset() in the backend -plugin. The example code contains an example on how to do this (see plugin_reset() in example_backend.c). +plugin. The example code contains an example on how to do this (see +plugin_reset() in example_backend.c). -The extra-xml feature is not available if startup mode is `none`. It will also not occur in failsafe mode. +The extra-xml feature is not available if startup mode is `none`. It +will also not occur in failsafe mode. ## Startup status @@ -174,13 +185,18 @@ in `CLICON_XMLDB_DIR/failsafe_db`. If such a config is not found, the backend terminates. If the failsafe is found, the failsafe config is loaded and -committed into the running db. The `startup` database will contain syntax -errors or invalidated XML. +committed into the running db. -A user can repair the `startup` configuration and either restart the -backend or copy the startup configuration to candidate and the commit. +If the startup mode was `startup`, the `startup` database will +contain syntax errors or invalidated XML. -Note that the if the startup configuration contains syntactic errors +If the startup mode was `running`, the the `tmp` database will contain +syntax errors or invalidated XML. + +A user can repair a broken configuration and either restart the +backend or copy the repaired configuration to candidate and then commit. + +Note that if the broken configuration contains syntactic errors (eg `STARTUP_ERR`) you cannot access the startup via Restconf or Netconf operations since the XML may be broken. @@ -192,22 +208,25 @@ and then copy/commit it via CLI, Netconf or Restconf. This section contains "pseudo" flowcharts showing the dynamics of the configuration databases in the startup phase. -The flowchart starts in one of the the modes (non, init, startup, running): +The flowchart starts in one of the modes (none, init, startup, running): + +### init mode -Starting in init mode: ``` reset running |--------+------------> GOTO EXTRA XML ``` -Start in running mode: -``` -running ----+ - \ copy -startup +------------> GOTO STARTUP +### running mode ``` -Starting in startup mode: +running ----+ |----------+--------> GOTO EXTRA XML + \ copy parse validate OK / commit +tmp ------+-------+------+-----------+ + +``` + +### startup mode ``` reset running |--------+------------> GOTO EXTRA XML @@ -215,29 +234,36 @@ running |--------+------------> GOTO EXTRA XML startup -------+--+-------+------------+ ``` -If validation of startup fails: +### Failure ``` failsafe ----------------------+ reset \ commit -running |-------+---------------> GOTO EXTRA XML +running |-------+---------------> GOTO SYSTEM UP parse validate fail -startup ---+-------------------------------------> INVALID XML +tmp/startup --+-----+---------------------------------> INVALID XML ``` -Load EXTRA XML: +### Extra XML ``` running -----------------+----+------> GOTO SYSTEM UP reset loadfile / merge tmp |-------+-----+-----+ ``` -SYSTEM UP: +### System UP ``` running ----+-----------------------> RUNNING \ copy candidate +---------------------> CANDIDATE + ``` - + +### Invalid XML + repair restart +tmp/startup --------+---------+-----------------------> + + + ## Thanks Thanks matt smith and dave cornejo for input diff --git a/test/test_upgrade.sh b/test/test_upgrade.sh index 1198a807..44eee28f 100755 --- a/test/test_upgrade.sh +++ b/test/test_upgrade.sh @@ -240,15 +240,18 @@ cat < $dir/compat-err.xml EOF -# Start system in $mode with existing (old) configuration in $db -# mode is one of: init, none, running, or startup -# db is one of: running_db or startup_db +#! Start system in given mode and check database contents +# Before script is called populate running_db and startup_db +# @param[in] modstate Boolean: Tag datastores with RFC 7895 YANG Module Library +# @param[in] mode Startup mode: init, none, running, or startup +# @param[in] exprun Expected content of running-db +# @param[in] expstart Check startup database or not if "" runtest(){ modstate=$1 mode=$2 - exprun=$3 # Expected running - expstartup=$4 # Expected startup - + exprun=$3 + expstart=$4 + new "test params: -f $cfg" # Bring your own backend if [ $BE -ne 0 ]; then @@ -268,11 +271,14 @@ runtest(){ sleep $BETIMEOUT fi - new "Get running" + new "Check running db content" expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' "^$exprun]]>]]>$" - new "Get startup" - expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' "^$expstartup]]>]]>$" + # If given check startup db XML + if [ -n "$expstart" ]; then + new "Check startup db content" + expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^$expstart]]>]]>$" + fi if [ $BE -ne 0 ]; then new "Kill backend" @@ -315,8 +321,8 @@ fi new "3. Load compatible running valid running (rest of tests are startup)" (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; cp compat-valid.xml running_db) -(cd $dir; cp compat-valid.xml startup_db) # XXX -runtest true running 'always workother text' 'always workother text' +runtest true running 'always workother text' '' +#'always workother text' new "4. Load non-compat valid startup" (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases @@ -332,7 +338,9 @@ new "6. Load non-compat invalid running. Enter failsafe, startup invalid." (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; cp non-compat-invalid.xml running_db) (cd $dir; cp non-compat-valid.xml startup_db) # XXX tmp -#runtest true running 'always work' 'old versionalways workother textbla bla' +runtest true running 'always work' '' + +#'old versionalways workother textbla bla' new "7. Load compatible invalid startup." (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases