From 560110b4e8b06802435bb0e8a47b3d8eb57835f8 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 13 Feb 2019 11:35:49 +0100 Subject: [PATCH] * New backend startup and upgrade support, see [doc/startup.md] for details * Datastore files contain RFC7895 module-state information --- CHANGELOG.md | 25 ++ apps/backend/Makefile.in | 1 + apps/backend/backend_client.c | 8 +- apps/backend/backend_commit.c | 92 ++++- apps/backend/backend_commit.h | 1 + apps/backend/backend_main.c | 462 ++++++---------------- apps/backend/backend_plugin.c | 25 ++ apps/backend/backend_plugin.h | 2 +- apps/backend/backend_socket.c | 2 +- apps/backend/backend_startup.c | 340 ++++++++++++++++ apps/backend/backend_startup.h | 49 +++ apps/cli/cli_main.c | 2 +- apps/cli/cli_plugin.c | 28 -- apps/netconf/netconf_main.c | 2 +- datastore/clixon_xmldb_text.c | 289 +++++++++----- datastore/clixon_xmldb_text.h | 2 +- doc/FAQ.md | 2 +- example/README.md | 3 +- example/example_backend.c | 46 ++- example/example_backend_nacm.c | 2 +- include/clixon_custom.h | 8 +- lib/clixon/clixon_err.h | 5 + lib/clixon/clixon_options.h | 4 + lib/clixon/clixon_plugin.h | 30 +- lib/clixon/clixon_xml.h | 4 +- lib/clixon/clixon_xml_db.h | 4 +- lib/clixon/clixon_yang_module.h | 2 +- lib/src/clixon_nacm.c | 2 +- lib/src/clixon_options.c | 30 ++ lib/src/clixon_plugin.c | 2 +- lib/src/clixon_xml.c | 22 +- lib/src/clixon_xml_db.c | 8 +- lib/src/clixon_xml_map.c | 4 +- lib/src/clixon_xml_parse.y | 12 +- lib/src/clixon_yang_module.c | 35 +- test/all.sh | 1 + test/lib.sh | 11 +- test/test_compat.sh | 81 +--- test/test_copy_config.sh | 178 +++++++++ test/test_order.sh | 2 +- test/test_startup.sh | 52 ++- test/test_upgrade.sh | 333 ++++++++++++++++ util/clixon_util_datastore.c | 4 +- yang/clixon/clixon-config@2019-02-06.yang | 9 + 44 files changed, 1595 insertions(+), 631 deletions(-) create mode 100644 apps/backend/backend_startup.c create mode 100644 apps/backend/backend_startup.h create mode 100755 test/test_copy_config.sh create mode 100755 test/test_upgrade.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0718eb21..6284616c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Clixon Changelog +### Major New features +* New backend startup and upgrade support, see [doc/startup.md] for details + * Enable with CLICON_XMLDB_MODSTATE config option + * Tag all datastores with system modules-state as in RFC7895 + * Check modules-state tags when loading a datastore at startup + * Check which modules match, and which do not. + * Loading of a "startup" XML or JSON configuration + * Loading of "extra" XML. + * Detection of in-compatible XML and Yang models in the startup configuration. + * An upgrade callback when in-compatible XML is encountered (`ca_upgrade`) + * A "failsafe" mode allowing a user to repair the startup on errors or failed validation. + * Major rewrite of `backend_main.c` and a new module `backend_startup.c` +* Datastore files contain RFC7895 module-state information + * Added modules-state parameter to xmldb_get datastore function + * Set config option `CLICON_XMLDB_MODSTATE` to true + * Enable this if you wish to use the upgrade feature in the new startup functionality. + * Note that this adds bytes to your configs + +### API changes on existing features (you may need to change your code) +* Removed obsolete `CLICON_CLI_MODEL_TREENAME_PATCH` + +### Minor changes +* Added specific clixon_suberrno code: XMLPARSE_ERRNO to identify XML parse errors. + + ## 3.9.0 (Preliminary Target: February 2019) ### Major New features diff --git a/apps/backend/Makefile.in b/apps/backend/Makefile.in index b3700e33..5ddc0ce3 100644 --- a/apps/backend/Makefile.in +++ b/apps/backend/Makefile.in @@ -74,6 +74,7 @@ APPSRC += backend_socket.c APPSRC += backend_client.c APPSRC += backend_commit.c APPSRC += backend_plugin.c +APPSRC += backend_startup.c APPOBJ = $(APPSRC:.c=.o) # Accessible from plugin diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index c2c370fe..8ec359ea 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -64,8 +64,8 @@ #include #include "clixon_backend_handle.h" -#include "backend_commit.h" #include "backend_plugin.h" +#include "backend_commit.h" #include "backend_client.h" #include "backend_handle.h" @@ -208,7 +208,7 @@ from_client_get_config(clicon_handle h, if ((xfilter = xml_find(xe, "filter")) != NULL) if ((xpath = xml_find_value(xfilter, "select"))==NULL) xpath="/"; - if (xmldb_get(h, db, xpath, 1, &xret) < 0){ + if (xmldb_get(h, db, xpath, 1, &xret, NULL) < 0){ if (netconf_operation_failed(cbret, "application", "read registry")< 0) goto done; goto ok; @@ -335,7 +335,7 @@ client_statedata(clicon_handle h, if ((retval = client_get_streams(h, yspec, xpath, "ietf-restconf-monitoring", "restconf-state", xret)) != 0) goto done; if (clicon_option_bool(h, "CLICON_MODULE_LIBRARY_RFC7895")) - if ((retval = yang_modules_state_get(h, yspec, xret)) != 0) + if ((retval = yang_modules_state_get(h, yspec, 0, xret)) != 0) goto done; if ((retval = clixon_plugin_statedata(h, yspec, xpath, xret)) != 0) goto done; @@ -391,7 +391,7 @@ from_client_get(clicon_handle h, if ((xpath = xml_find_value(xfilter, "select"))==NULL) xpath="/"; /* Get config */ - if (xmldb_get(h, "running", xpath, 0, &xret) < 0){ + if (xmldb_get(h, "running", xpath, 0, &xret, NULL) < 0){ if (netconf_operation_failed(cbret, "application", "read registry")< 0) goto done; goto ok; diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 62a7ea9b..10971817 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -140,7 +140,9 @@ generic_validate(yang_spec *yspec, goto done; } -/*! Common code of candidate_validate and candidate_commit +/*! Validate a candidate db and comnpare to running + * Get both source and dest datastore, validate target, compute diffs + * and call application callback validations. * @param[in] h Clicon handle * @param[in] candidate The candidate database. The wanted backend state * @retval -1 Error - or validation failed (but cbret not set) @@ -167,10 +169,10 @@ validate_common(clicon_handle h, } /* 2. Parse xml trees * This is the state we are going from */ - if (xmldb_get(h, "running", "/", 1, &td->td_src) < 0) + if (xmldb_get(h, "running", "/", 1, &td->td_src, NULL) < 0) goto done; /* This is the state we are going to */ - if (xmldb_get(h, candidate, "/", 1, &td->td_target) < 0) + if (xmldb_get(h, candidate, "/", 1, &td->td_target, NULL) < 0) goto done; /* Validate the target state. It is not completely clear this should be done @@ -244,6 +246,90 @@ validate_common(clicon_handle h, goto done; } +/*! Validate a candidate db and comnpare to running XXX Experimental + * Get both source and dest datastore, validate target, compute diffs + * and call application callback validations. + * @param[in] h Clicon handle + * @param[in] candidate The candidate database. The wanted backend state + * @retval -1 Error - or validation failed (but cbret not set) + * @retval 0 Validation failed (with cbret set) + * @retval 1 Validation OK + * @note Need to differentiate between error and validation fail + * 1. Parse startup XML (or JSON) + * 2. If syntax failure, call startup-cb(ERROR), copy failsafe db to + * candidate and commit. Done + * 3. Check yang module versions between backend and init config XML. (msdiff) + * 4. Validate startup db. (valid) + * 5. If valid fails, call startup-cb(Invalid, msdiff), keep startup in candidate and commit failsafe db. Done. + * 6. Call startup-cb(OK, msdiff) and commit. + */ +int +startup_validate(clicon_handle h, + char *db, + cbuf *cbret) +{ + int retval = -1; + yang_spec *yspec; + int ret; + cxobj *xms = NULL; + transaction_data_t *td = NULL; + + /* Handcraft a transition with only target and add trees */ + if ((td = transaction_new()) == NULL) + goto done; + /* 2. Parse xml trees + * This is the state we are going to + * Note: xmsdiff contains non-matching modules + */ + if (xmldb_get(h, db, "/", 1, &td->td_target, &xms) < 0) + goto done; + if (xms && clixon_plugin_upgrade(h, xms) < 0) + goto done; + + /* Handcraft transition with with only add tree */ + if (cxvec_append(td->td_target, &td->td_avec, &td->td_alen) < 0) + goto done; + + /* 4. Call plugin transaction start callbacks */ + if (plugin_transaction_begin(h, td) < 0) + goto done; + + if ((yspec = clicon_dbspec_yang(h)) == NULL){ + clicon_err(OE_YANG, 0, "Yang spec not set"); + goto done; + } + /* 5. Make generic validation on all new or changed data. + Note this is only call that uses 3-values */ + if ((ret = generic_validate(yspec, td, cbret)) < 0) + goto done; + if (ret == 0) + goto fail; /* STARTUP_INVALID */ + + /* 6. Call plugin transaction validate callbacks */ + if (plugin_transaction_validate(h, td) < 0) + goto done; + + /* 7. Call plugin transaction complete callbacks */ + if (plugin_transaction_complete(h, td) < 0) + goto done; + retval = 1; + done: + if (td) + transaction_free(td); + if (xms) + xml_free(xms); + return retval; + fail: /* cbret should be set */ + if (cbuf_len(cbret)==0){ + clicon_err(OE_CFG, EINVAL, "Validation fail but cbret not set"); + goto done; + } + retval = 0; + goto done; +} + + + /*! Do a diff between candidate and running, then start a commit transaction * * The code reverts changes if the commit fails. But if the revert diff --git a/apps/backend/backend_commit.h b/apps/backend/backend_commit.h index 3af2461a..55a1a205 100644 --- a/apps/backend/backend_commit.h +++ b/apps/backend/backend_commit.h @@ -43,6 +43,7 @@ int from_client_validate(clicon_handle h, char *db, cbuf *cbret); int from_client_commit(clicon_handle h, int pid, cbuf *cbret); int from_client_discard_changes(clicon_handle h, int pid, cbuf *cbret); +int startup_validate(clicon_handle h, char *db, cbuf *cbret); int candidate_commit(clicon_handle h, char *db, cbuf *cbret); #endif /* _BACKEND_COMMIT_H_ */ diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index 0bf2a7f8..cd3a2eac 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -68,9 +68,10 @@ #include "clixon_backend_handle.h" #include "backend_socket.h" #include "backend_client.h" -#include "backend_commit.h" #include "backend_plugin.h" +#include "backend_commit.h" #include "backend_handle.h" +#include "backend_startup.h" /* Command line options to be passed to getopt(3) */ #define BACKEND_OPTS "hD:f:l:d:p:b:Fza:u:P:1s:c:g:y:x:o:" @@ -128,84 +129,6 @@ backend_sig_term(int arg) clicon_exit_set(); /* checked in event_loop() */ } -/*! usage - */ -static void -usage(clicon_handle h, - char *argv0) -{ - char *plgdir = clicon_backend_dir(h); - char *confsock = clicon_sock(h); - char *confpid = clicon_backend_pidfile(h); - char *group = clicon_sock_group(h); - - fprintf(stderr, "usage:%s *\n" - "where options are\n" - "\t-h\t\tHelp\n" - "\t-D \tDebug level\n" - "\t-f \tCLICON config file\n" - "\t-l (s|e|o|f) Log on (s)yslog, std(e)rr or std(o)ut (stderr is default) Only valid if -F, if background syslog is on syslog.\n" - "\t-d \tSpecify backend plugin directory (default: %s)\n" - "\t-p \tYang directory path (see CLICON_YANG_DIR)\n" - "\t-b \tSpecify XMLDB database directory\n" - "\t-F\t\tRun in foreground, do not run as daemon\n" - "\t-z\t\tKill other config daemon and exit\n" - "\t-a UNIX|IPv4|IPv6 Internal backend socket family\n" - "\t-u \tInternal socket domain path or IP addr (see -a)(default: %s)\n" - "\t-P \tPid filename (default: %s)\n" - "\t-1\t\tRun once and then quit (dont wait for events)\n" - "\t-s \tSpecify backend startup mode: none|startup|running|init)\n" - "\t-c \tLoad extra xml configuration, but don't commit.\n" - "\t-g \tClient membership required to this group (default: %s)\n" - - "\t-y \tLoad yang spec file (override yang main module)\n" - "\t-x \tXMLDB plugin\n" - "\t-o \"