diff --git a/CHANGELOG.md b/CHANGELOG.md index 91697a20..79830f07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,10 @@ ``` ### Minor changes +* Added flags to example backend to control its behaviour: + * Start with `-- -r` to run the reset plugin + * Start with `-- -s` to run the state callback +* Rewrote yang dir load algorithm to follow the algorithm in [FAQ](FAQ(doc/FAQ.md#how-are-yang-files-found) with more precise timestamp checks, etc. * Ensured you can add multiple callbacks for any RPC, including basic ones. * Extra RPC:s will be called _after_ the basic ones. * One specific usecase is hook for `copy-config` (see [doc/ROADMAP.md](doc/ROADMAP.md) that can be implemented this way. diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index accc2641..938a6bfb 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -568,11 +568,6 @@ from_client_validate(clicon_handle h, goto done; goto ok; } - if (strcmp(db, "candidate") != 0 && strcmp(db, "tmp") != 0){ - if (netconf_invalid_value(cbret, "protocol", "No such database")< 0) - goto done; - goto ok; - } clicon_debug(1, "Validate %s", db); /* 1. Start transaction */ diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index 3c441039..7c6f4c70 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -596,7 +596,7 @@ main(int argc, if (yang_spec_parse_module(h, str, clicon_yang_module_revision(h), yspec) < 0) goto done; - /* 3. Load all modules in a directory */ + /* 3. Load all modules in a directory (will not overwrite file loaded ^) */ if ((str = clicon_yang_main_dir(h)) != NULL) if (yang_spec_load_dir(h, str, yspec) < 0) goto done; diff --git a/doc/FAQ.md b/doc/FAQ.md index 0f2841b2..3e24a4c7 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -271,6 +271,11 @@ specific. The precedence of the options are as follows: - `CLICON_YANG_MODULE_MAIN` - `CLICON_YANG_MAIN_DIR` +Note that using `CLICON_YANG_MAIN_DIR` Clixon may find several files +containing the same Yang module. Clixon will prefer the one without a +revision date if such a file exists. If no file has a revision date, +Clixon will prefer the newest. + ## How do I enable Yang features? Yang models have features, and parts of a specification can be diff --git a/example/README.md b/example/README.md index 38d0f4ee..0db41ded 100644 --- a/example/README.md +++ b/example/README.md @@ -262,6 +262,8 @@ a real example would poll or get the interface counters via a system call, as well as use the "xpath" argument to identify the requested state data. +The state data is enabled by starting the backend with: `-- -s`. + ## Authentication and NACM The example contains some stubs for authorization according to [RFC8341(NACM)](https://tools.ietf.org/html/rfc8341): * A basic auth HTTP callback, see: example_restconf_credentials() containing three example users: andy, wilma, and guest, according to the examples in Appendix A in [RFC8341](https://tools.ietf.org/html/rfc8341). diff --git a/example/example_backend.c b/example/example_backend.c index 55a2a367..d191a92f 100644 --- a/example/example_backend.c +++ b/example/example_backend.c @@ -51,13 +51,19 @@ /* These include signatures for plugin and transaction callbacks. */ #include -/* Variable to control if reset code is run. +/*! Variable to control if reset code is run. * The reset code inserts "extra XML" which assumes ietf-interfaces is * loaded, and this is not always the case. * Therefore, the backend must be started with -- -r to enable the reset function */ static int _reset = 0; +/*! Variable to control if state code is run + * The state code adds extra non-config data + * Therefore, the backend must be started with -- -s to enable the state function + */ +static int _state = 0; + /* forward */ static int example_stream_timer_setup(clicon_handle h); @@ -210,7 +216,7 @@ example_copy_extra(clicon_handle h, /* Clicon handle */ type string; } } - * + * This yang snippet is present in clixon-example.yang for exampl. */ int example_statedata(clicon_handle h, @@ -220,6 +226,8 @@ example_statedata(clicon_handle h, int retval = -1; cxobj **xvec = NULL; + if (!_state) + goto ok; /* Example of (static) statedata, real code would poll state * Note this state needs to be accomanied by yang snippet * above @@ -230,6 +238,7 @@ example_statedata(clicon_handle h, "43" /* should not be ordered */ "", NULL, &xstate) < 0) goto done; + ok: retval = 0; done: if (xvec) @@ -277,7 +286,8 @@ example_reset(clicon_handle h, int ret; cbuf *cbret = NULL; - goto ok; /* Note not enabled by default */ + if (!_reset) + goto ok; /* Note not enabled by default */ if (xml_parse_string("" "loex:loopback" "", NULL, &xt) < 0) @@ -329,11 +339,14 @@ example_start(clicon_handle h, opterr = 0; optind = 1; - while ((c = getopt(argc, argv, "r")) != -1) + while ((c = getopt(argc, argv, "rs")) != -1) switch (c) { case 'r': _reset = 1; break; + case 's': + _state = 1; + break; } return 0; } diff --git a/lib/src/clixon_netconf_lib.c b/lib/src/clixon_netconf_lib.c index 75a607c4..98ee73f7 100644 --- a/lib/src/clixon_netconf_lib.c +++ b/lib/src/clixon_netconf_lib.c @@ -67,7 +67,7 @@ /*! Create Netconf in-use error XML tree according to RFC 6241 Appendix A * * The request requires a resource that already is in use. - * @param[out] cb CLIgen buf. Error XML is written in this buffer + * @param[out] cb CLIgen buf. Error XML is written in this buffer * @param[in] type Error type: "application" or "protocol" * @param[in] message Error message */ @@ -376,7 +376,6 @@ netconf_missing_element(cbuf *cb, return retval; } - /*! Create Netconf missing-element error XML tree according to RFC 6241 App A * @param[out] xret Error XML tree. Free with xml_free after use * @param[in] type Error type: "application" or "protocol" diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index f8480040..962e3800 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -96,7 +96,9 @@ /* * Local variables */ -/* Mapping between yang keyword string <--> clicon constants */ +/* Mapping between yang keyword string <--> clicon constants + * Here is also the place where doc on some types store variables (cv) + */ static const map_str2int ykmap[] = { {"anydata", Y_ANYDATA}, {"anyxml", Y_ANYXML}, @@ -107,7 +109,7 @@ static const map_str2int ykmap[] = { {"bit", Y_BIT}, {"case", Y_CASE}, {"choice", Y_CHOICE}, - {"config", Y_CONFIG}, + {"config", Y_CONFIG}, /* cv: boolean config flag */ {"contact", Y_CONTACT}, {"container", Y_CONTAINER}, {"default", Y_DEFAULT}, @@ -118,8 +120,8 @@ static const map_str2int ykmap[] = { {"error-app-tag", Y_ERROR_APP_TAG}, {"error_message", Y_ERROR_MESSAGE}, {"extension", Y_EXTENSION}, - {"feature", Y_FEATURE}, - {"fraction-digits", Y_FRACTION_DIGITS}, + {"feature", Y_FEATURE}, /* cv: feature as boolean */ + {"fraction-digits", Y_FRACTION_DIGITS}, /* cv: fraction-digits as uint8 */ {"grouping", Y_GROUPING}, {"identity", Y_IDENTITY}, {"if-feature", Y_IF_FEATURE}, @@ -127,11 +129,11 @@ static const map_str2int ykmap[] = { {"include", Y_INCLUDE}, {"input", Y_INPUT}, {"key", Y_KEY}, - {"leaf", Y_LEAF}, - {"leaf-list", Y_LEAF_LIST}, + {"leaf", Y_LEAF}, /* cv: store default value (if any)*/ + {"leaf-list", Y_LEAF_LIST}, /* cv: store default value (if any)*/ {"length", Y_LENGTH}, {"list", Y_LIST}, - {"mandatory", Y_MANDATORY}, + {"mandatory", Y_MANDATORY}, /* cv: store mandatory boolean */ {"max-elements", Y_MAX_ELEMENTS}, {"min-elements", Y_MIN_ELEMENTS}, {"modifier", Y_MODIFIER}, @@ -151,8 +153,8 @@ static const map_str2int ykmap[] = { {"reference", Y_REFERENCE}, {"refine", Y_REFINE}, {"require-instance", Y_REQUIRE_INSTANCE}, - {"revision", Y_REVISION}, - {"revision-date", Y_REVISION_DATE}, + {"revision", Y_REVISION}, /* cv: YYYY-MM-DD as uint32 */ + {"revision-date", Y_REVISION_DATE}, /* cv: YYYY-MM-DD as uint32 */ {"rpc", Y_RPC}, {"status", Y_STATUS}, {"submodule", Y_SUBMODULE}, @@ -160,7 +162,7 @@ static const map_str2int ykmap[] = { {"typedef", Y_TYPEDEF}, {"unique", Y_UNIQUE}, {"units", Y_UNITS}, - {"unknown", Y_UNKNOWN}, + {"unknown", Y_UNKNOWN}, /* cv: store extra string */ {"uses", Y_USES}, {"value", Y_VALUE}, {"when", Y_WHEN}, @@ -171,6 +173,9 @@ static const map_str2int ykmap[] = { {NULL, -1} }; +/* forward declaration */ +static int ys_parse_date_arg(char *str, uint32_t *date); + /*! Create new yang specification * @retval yspec Free with yspec_free() * @retval NULL Error @@ -426,8 +431,8 @@ yn_each(yang_node *yn, * @see yang_match returns number of matches */ yang_stmt * -yang_find(yang_node *yn, - int keyword, +yang_find(yang_node *yn, + int keyword, const char *argument) { yang_stmt *ys = NULL; @@ -2514,6 +2519,13 @@ yang_spec_parse_file(clicon_handle h, * @retval 0 OK * @retval -1 Error * @see yang_spec_parse_file + * Load all yang files in a directory as primary objects. + * Some details if several same yang module x exists: + * 1) If x is already loaded (eg via direct file loading) skip it + * 2) Prefer x.yang over x@rev.yang (no revision) + * 3) If only x@rev.yang's found, prefer newest (newest revision) + * There is also an extra failsafe which may not be necessary, which removes + * the oldest module if 1-3 for some reason fails. */ int yang_spec_load_dir(clicon_handle h, @@ -2524,13 +2536,26 @@ yang_spec_load_dir(clicon_handle h, int ndp; struct dirent *dp = NULL; int i; + int j; char filename[MAXPATHLEN]; char *base = NULL; /* filename without dir */ - char *b; - int j; int modnr; + yang_stmt *ym; /* yang module */ + yang_stmt *ym0; /* (existing) yang module */ + yang_stmt *yrev; /* yang revision */ + uint32_t revf; /* revision in filename */ + uint32_t revm; /* revision in parsed new module (same as revf) */ + uint32_t rev0; /* revision in existing module */ + char *s; + char *oldbase = NULL; + int taken = 0; - /* Get plugin objects names from plugin directory */ + /* Get yang files names from yang module directory. Note that these + * are sorted alphatetically: + * a.yang, + * a@2000-01-01.yang, + * a@2111-11-11.yang + */ if((ndp = clicon_file_dirent(dir, &dp, "(.yang)$", S_IFREG)) < 0) goto done; if (ndp == 0) @@ -2541,40 +2566,76 @@ yang_spec_load_dir(clicon_handle h, /* Load all yang files in dir */ for (i = 0; i < ndp; i++) { /* base = module name [+ @rev ] + .yang */ - if (base) - free(base); + if (oldbase) + free(oldbase); + oldbase = base; base = NULL; if ((base = strdup(dp[i].d_name)) == NULL){ clicon_err(OE_UNIX, errno, "strdup"); goto done; } + clicon_debug(1, "%s %s", __FUNCTION__, base); *rindex(base, '.') = '\0'; /* strip postfix .yang */ /* base = module name [+ @rev] * if it hasnt @rev then prefer it (dont check other files w @rev) - * Otherwise see if there is a newer */ - if ((b = index(base, '@')) != NULL){ - *b = '\0'; - /* base = module name */ - - /* Entries are sorted, see if later entry exists (include @), if so - * skip this one and take last. - * Assume file without @ is last - */ - for (j = (i+1); j < ndp; j++) - if (strncmp(base, dp[j].d_name, strlen(base)) == 0) - break; - if (j ym0/rev0 */ + rev0 = 0; + if ((ym0 = yang_find((yang_node*)yspec, Y_MODULE, base)) != NULL || + (ym0 = yang_find((yang_node*)yspec, Y_SUBMODULE, base)) != NULL){ + yrev = yang_find((yang_node*)ym0, Y_REVISION, NULL); + rev0 = cv_uint32_get(yrev->ys_cv); + continue; /* skip if already added by specific file or module */ + } + /* Create full filename */ snprintf(filename, MAXPATHLEN-1, "%s/%s", dir, dp[i].d_name); - /* Dont parse if already exists */ - if (yang_find((yang_node*)yspec, Y_MODULE, base) != NULL || - yang_find((yang_node*)yspec, Y_SUBMODULE, base) != NULL) - continue; - if (yang_parse_filename(filename, yspec) == NULL) + if ((ym = yang_parse_filename(filename, yspec)) == NULL) goto done; + revm = 0; + if ((yrev = yang_find((yang_node*)ym, Y_REVISION, NULL)) != NULL) + revm = cv_uint32_get(yrev->ys_cv); + /* Sanity check that file revision does not match internal rev stmt */ + if (revf && revm && revm != revf){ + clicon_err(OE_YANG, EINVAL, "Yang module file revision and in yang does not match: %s vs %u", filename, revm); + goto done; + } + /* If ym0 and ym exists, delete the yang with oldest revision + * This is a failsafe in case anything else fails + */ + if (revm && rev0){ + int size; + if (revm > rev0) /* Loaded module is older or eq -> remove ym */ + ym = ym0; + for (j=0; jyp_len; j++) + if (yspec->yp_stmt[j] == ym) + break; + size = (yspec->yp_len - j - 1)*sizeof(struct yang_stmt *); + memmove(&yspec->yp_stmt[j], + &yspec->yp_stmt[j+1], + size); + ys_free(ym); + yspec->yp_len--; + yspec->yp_stmt[yspec->yp_len] = NULL; + } } if (yang_parse_post(h, yspec, modnr) < 0) goto done; @@ -2584,10 +2645,11 @@ yang_spec_load_dir(clicon_handle h, free(dp); if (base) free(base); + if (oldbase) + free(oldbase); return retval; } - /*! Apply a function call recursively on all yang-stmt s recursively * * Recursively traverse all yang-nodes in a parse-tree and apply fn(arg) for @@ -2843,6 +2905,42 @@ yang_desc_schema_nodeid(yang_node *yn, return retval; } + +/*! parse yang date-arg string + */ +static int +ys_parse_date_arg(char *str, + uint32_t *date) +{ + int retval = -1; + int i; + uint32_t d = 0; + + if (strlen(str) != 10 || str[4] != '-' || str[7] != '-'){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", str); + goto done; + } + if ((i = cligen_tonum(4, str)) < 0){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", str); + goto done; + } + d = i*10000; /* year */ + if ((i = cligen_tonum(2, &str[5])) < 0){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", str); + goto done; + } + d += i*100; /* month */ + if ((i = cligen_tonum(2, &str[8])) < 0){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", str); + goto done; + } + d += i; /* day */ + *date = d; + retval = 0; + done: + return retval; +} + /*! Parse argument as CV and save result in yang cv variable * * Note that some CV:s are parsed directly (eg fraction-digits) while others are parsed @@ -2897,6 +2995,7 @@ ys_parse_sub(yang_stmt *ys, { int retval = -1; uint8_t fd; + uint32_t date = 0; switch (ys->ys_keyword){ case Y_FRACTION_DIGITS: @@ -2908,6 +3007,16 @@ ys_parse_sub(yang_stmt *ys, goto done; } break; + case Y_REVISION: + case Y_REVISION_DATE: /* YYYY-MM-DD encoded as uint32 YYYYMMDD */ + if (ys_parse_date_arg(ys->ys_argument, &date) < 0) + goto done; + if ((ys->ys_cv = cv_new(CGV_UINT32)) == NULL){ + clicon_err(OE_YANG, errno, "cv_new"); + goto done; + } + cv_uint32_set(ys->ys_cv, date); + break; case Y_UNKNOWN: /* XXX This code assumes ymod already loaded but it may not be */ if (extra == NULL) @@ -2998,7 +3107,6 @@ yang_config(yang_stmt *ys) return 1; } - /*! Given a yang node, translate the argument string to a cv vector * * @param[in] ys Yang statement diff --git a/test/all.sh b/test/all.sh index 897ab8a6..e2ff3809 100755 --- a/test/all.sh +++ b/test/all.sh @@ -9,7 +9,7 @@ fi err=0 testnr=0 -for test in test*.sh; do +for test in test_*.sh; do if [ $testnr != 0 ]; then echo; fi testfile=$test . ./$test diff --git a/test/lib.sh b/test/lib.sh index 83325bfe..02728155 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -24,7 +24,6 @@ # Site file, an example of this file in README.md if [ -f ./site.sh ]; then - . ./site.sh if [ $? -ne 0 ]; then return -1 # skip @@ -102,7 +101,8 @@ dir=/var/tmp/$0 if [ ! -d $dir ]; then mkdir $dir fi -# If we bring our own backend BE=0 (it is already started),the backend may + +# If you bring your own backend BE=0 (it is already started),the backend may # have created some files (eg unix socket) in $dir and therefore cannot # be deleted if [ $BE -ne 0 ]; then @@ -440,3 +440,4 @@ expectmatch(){ fi fi } + diff --git a/test/test_copy_config.sh b/test/test_copy_config.sh index 3cde38e9..f9e08e31 100755 --- a/test/test_copy_config.sh +++ b/test/test_copy_config.sh @@ -24,7 +24,6 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi APPNAME=example # include err() and new() functions and creates $dir -. ./lib.sh cfg=$dir/conf_yang.xml diff --git a/test/test_nacm_ext.sh b/test/test_nacm_ext.sh index db9d7c07..042b6488 100755 --- a/test/test_nacm_ext.sh +++ b/test/test_nacm_ext.sh @@ -137,9 +137,9 @@ if [ $BE -ne 0 ]; then err fi sleep 1 - new "start backend -s init -f $cfg" + new "start backend -s init -f $cfg -- -s" # start new backend - start_backend -s init -f $cfg + start_backend -s init -f $cfg -- -s fi new "kill old restconf daemon" diff --git a/test/test_nacm_module_read.sh b/test/test_nacm_module_read.sh index 008f2aef..dbb27a24 100755 --- a/test/test_nacm_module_read.sh +++ b/test/test_nacm_module_read.sh @@ -133,8 +133,8 @@ if [ $BE -ne 0 ]; then if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg" - start_backend -s init -f $cfg + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s fi new "kill old restconf daemon" diff --git a/test/test_netconf.sh b/test/test_netconf.sh index fc8a08f7..266c31a2 100755 --- a/test/test_netconf.sh +++ b/test/test_netconf.sh @@ -42,8 +42,8 @@ if [ $BE -ne 0 ]; then if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg" - start_backend -s init -f $cfg + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s new "waiting" sleep $RCWAIT diff --git a/test/test_order.sh b/test/test_order.sh index 623b06c0..07275b42 100755 --- a/test/test_order.sh +++ b/test/test_order.sh @@ -147,7 +147,7 @@ cat < $dbdir/running_db EOF -new "test params: -s running -f $cfg -y $fyang" +new "test params: -s running -f $cfg -y $fyang -- -s" if [ $BE -ne 0 ]; then new "kill old backend" @@ -156,7 +156,7 @@ if [ $BE -ne 0 ]; then err fi new "start backend" - start_backend -s running -f $cfg -y $fyang + start_backend -s running -f $cfg -y $fyang -- -s new "waiting" sleep $RCWAIT diff --git a/test/test_restconf.sh b/test/test_restconf.sh index 2da7c78b..913216ae 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -43,8 +43,8 @@ if [ $BE -ne 0 ]; then if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg" - start_backend -s init -f $cfg + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s fi new "kill old restconf daemon" diff --git a/test/test_upgrade.sh b/test/test_upgrade.sh index 44eee28f..8029cabb 100755 --- a/test/test_upgrade.sh +++ b/test/test_upgrade.sh @@ -3,14 +3,14 @@ # This relieas on storing RFC7895 YANG Module Library modules-state info # in the datastore (or XML files?) # The test is made with three Yang models A, B and C as follows: -# Yang module A has revisions "814-01-28" and "2019-01-01" +# Yang module A has revisions "0814-01-28" and "2019-01-01" # Yang module B has only revision "2019-01-01" # Yang module C has only revision "2019-01-01" # The system is started YANG modules: # A revision "2019-01-01" # B revision "2019-01-01" # The (startup) configuration XML file has: -# A revision "814-01-28"; +# A revision "0814-01-28"; # B revision "2019-01-01" # C revision "2019-01-01" # Which means the following: @@ -19,24 +19,26 @@ # B has a compatible version # C is not present in the system +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + APPNAME=example -# include err() and new() functions and creates $dir -. ./lib.sh + cfg=$dir/conf_yang.xml -fyangA0=$dir/A@814-01-28.yang +fyangA0=$dir/A@0814-01-28.yang fyangA1=$dir/A@2019-01-01.yang fyangB=$dir/B@2019-01-01.yang -# Yang module A revision "814-01-28" +# Yang module A revision "0814-01-28" # Note that this Yang model will exist in the DIR but will not be loaded # by the system. Just here for reference # XXX: Maybe it should be loaded and used in draft-wu? cat < $fyangA0 module A{ prefix a; - revision 814-01-28; + revision 0814-01-28; namespace "urn:example:a"; leaf a0{ type string; @@ -52,7 +54,7 @@ cat < $fyangA1 module A{ prefix a; revision 2019-01-01; - revision 814-01-28; + revision 0814-01-28; namespace "urn:example:a"; /* leaf a0 has been removed */ leaf a1{ @@ -171,7 +173,7 @@ cat < $dir/non-compat-valid.xml 42 A - 814-01-28 + 0814-01-28 urn:example:a @@ -198,7 +200,7 @@ cat < $dir/non-compat-invalid.xml 42 A - 814-01-28 + 0814-01-28 urn:example:a @@ -332,7 +334,8 @@ runtest true startup 'always workalways work' 'old versionalways workother textbla bla' +#runtest true startup 'always work' 'old versionalways workother textbla bla' # sorted +runtest true startup 'always work' 'always workother textold versionbla bla' # unsorted new "6. Load non-compat invalid running. Enter failsafe, startup invalid." (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases @@ -345,7 +348,8 @@ runtest true running 'always work' ' new "7. Load compatible invalid startup." (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; cp compat-invalid.xml startup_db) -runtest true startup 'always work' 'old versionalways workother textbla bla' +#runtest true startup 'always work' 'old versionalways workother textbla bla' # sorted +runtest true startup 'always work' 'always workother textold versionbla bla' # unsorted # This testcase contains an error/exception of the clixon xml parser, and # I cant track down the memory leakage. @@ -354,8 +358,7 @@ new "8. Load non-compat startup. Syntax fail, enter failsafe, startup invalid" (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases (cd $dir; cp compat-err.xml startup_db) runtest true startup 'always work' 'applicationoperation-failederrorread registry' - -fi +fi # valgrindtest if [ $BE -ne 0 ]; then rm -rf $dir diff --git a/test/test_upgrade_repair.sh b/test/test_upgrade_repair.sh new file mode 100755 index 00000000..e063f19e --- /dev/null +++ b/test/test_upgrade_repair.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Start clixon with module A rec 2019 +# Load startup with non-compatible and invalid module A with rev 0814-01-28 +# Go into fail-safe with invalid startup + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +cfg=$dir/conf_yang.xml +fyangA0=$dir/A@0814-01-28.yang +fyangA1=$dir/A@2019-01-01.yang + +# Yang module A revision "0814-01-28" +# Note that this Yang model will exist in the DIR but will not be loaded +# by the system. Just here for reference +# XXX: Maybe it should be loaded and used in draft-wu? +cat < $fyangA0 +module A{ + prefix a; + revision 0814-01-28; + namespace "urn:example:a"; + leaf a0{ + type string; + } + leaf a1{ + type string; + } +} +EOF + +# Yang module A revision "2019-01-01" +cat < $fyangA1 +module A{ + prefix a; + revision 2019-01-01; + revision 0814-01-28; + namespace "urn:example:a"; + /* leaf a0 has been removed */ + leaf a1{ + description "exists in both versions"; + type string; + } + leaf a2{ + description "has been added"; + type string; + } +} +EOF + +# Create configuration +cat < $cfg + + $cfg + /usr/local/share/clixon + $dir + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/lib/example/backend + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + /usr/local/lib/xmldb/text.so + true + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + +EOF + +# Create failsafe db +cat < $dir/failsafe_db + + always work + +EOF + +# Create non-compat startup db +# startup config XML with following (A obsolete, B OK, C lacking) +cat < $dir/non-compat-invalid.xml + + + 42 + + A + 0814-01-28 + urn:example:a + + + old version + always work + +EOF + +(cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases +(cd $dir; cp non-compat-invalid.xml startup_db) + +mode=startup + +new "test params: -f $cfg" +# Bring your own backend +if [ $BE -ne 0 ]; then + # kill old backend (if any) + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s $mode -f $cfg" + start_backend -s $mode -f $cfg +fi +new "waiting" +sleep $RCWAIT + +new "kill old restconf daemon" +sudo pkill -u www-data clixon_restconf + +new "start restconf daemon" +start_restconf -f $cfg + +new "waiting" +sleep $RCWAIT + +new "Check running db content is failsafe" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^always work]]>]]>$' + +#repair + +#exit + +new "Kill restconf daemon" +stop_restconf + +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + + rm -rf $dir +fi + diff --git a/test/test_yang_load.sh b/test/test_yang_load.sh index c315e400..ad772be6 100755 --- a/test/test_yang_load.sh +++ b/test/test_yang_load.sh @@ -10,16 +10,20 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi APPNAME=example +OLDDATE=0814-01-28 # This is alphabeticaly after 2018-12-02 +NEWDATE=2018-12-02 + cfg=$dir/conf_yang.xml fyang1=$dir/$APPNAME@2018-12-02.yang -fyang2=$dir/$APPNAME@2018-01-01.yang +fyang2=$dir/$APPNAME@$OLDDATE.yang fyang3=$dir/other.yang # 1st variant of the example module cat < $fyang1 module example{ prefix ex; - revision 2018-12-02; + revision $NEWDATE; + revision $OLDDATE; namespace "urn:example:clixon"; leaf newex{ type string; @@ -31,7 +35,7 @@ EOF cat < $fyang2 module example{ prefix ex; - revision 2018-01-01; + revision $OLDDATE; namespace "urn:example:clixon"; leaf oldex{ type string; @@ -43,7 +47,7 @@ EOF cat < $fyang3 module other{ prefix oth; - revision 2018-01-01; + revision $NEWDATE; namespace "urn:example:clixon2"; leaf other{ type string; @@ -94,20 +98,18 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^applicationunknown-elementothererrorUnassigned yang spec]]>]]>$' -if [ $BE -eq 0 ]; then - exit # BE +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" -fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend - #-------------------------------------- new "2. Load old module as file" cat < $cfg @@ -128,12 +130,14 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -# start new backend -start_backend -s init -f $cfg - -new "waiting" -sleep $RCWAIT +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + # start new backend + start_backend -s init -f $cfg + + new "waiting" + sleep $RCWAIT +fi new "Set oldex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -144,15 +148,17 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^applicationunknown-elementothererrorUnassigned yang spec]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend #-------------------------------------- new "3. Load module with no revision" @@ -170,11 +176,13 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -start_backend -s init -f $cfg +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg -new "waiting" -sleep $RCWAIT + new "waiting" + sleep $RCWAIT +fi new "Set newex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -185,14 +193,16 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^applicationunknown-elementothererrorUnassigned yang spec]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend #-------------------------------------- new "4. Load module with old revision" @@ -203,7 +213,7 @@ cat < $cfg /usr/local/share/clixon $IETFRFC example - 2018-01-01 + $OLDDATE /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile /usr/local/var/$APPNAME @@ -211,11 +221,13 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -start_backend -s init -f $cfg - -new "waiting" -sleep $RCWAIT +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg + + new "waiting" + sleep $RCWAIT +fi new "Set oldex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -226,15 +238,17 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^applicationunknown-elementothererrorUnassigned yang spec]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend #-------------------------------------- new "5. Load dir" @@ -252,11 +266,13 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -start_backend -s init -f $cfg +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg -new "waiting" -sleep $RCWAIT + new "waiting" + sleep $RCWAIT +fi new "Set newex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -267,15 +283,18 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend #-------------------------------------- new "6. Load dir override with file" @@ -294,12 +313,13 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -start_backend -s init -f $cfg - -new "waiting" -sleep $RCWAIT +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg + new "waiting" + sleep $RCWAIT +fi new "Set oldex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -309,16 +329,17 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend - #-------------------------------------- new "7. Load dir override with module + revision" @@ -330,7 +351,7 @@ cat < $cfg $IETFRFC $dir example - 2018-01-01 + $OLDDATE /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile /usr/local/var/$APPNAME @@ -338,11 +359,13 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -start_backend -s init -f $cfg +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg -new "waiting" -sleep $RCWAIT + new "waiting" + sleep $RCWAIT +fi new "Set oldex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -353,15 +376,17 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend #-------------------------------------- new "8. Load module w new revision overrided by old file" @@ -373,7 +398,7 @@ cat < $cfg $IETFRFC $fyang2 example - 2018-12-02 + $NEWDATE /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile /usr/local/var/$APPNAME @@ -381,11 +406,13 @@ cat < $cfg EOF -new "start backend -s init -f $cfg" -start_backend -s init -f $cfg +if [ $BE -ne 0 ]; then + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg -new "waiting" -sleep $RCWAIT + new "waiting" + sleep $RCWAIT +fi new "Set oldex" expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^]]>]]>$' @@ -396,14 +423,18 @@ expecteof "$clixon_netconf -qf $cfg" 0 'str]]>]]>' '^applicationunknown-elementothererrorUnassigned yang spec]]>]]>$' -new "Kill backend" -# Check if premature kill -pid=`pgrep -u root -f clixon_backend` -if [ -z "$pid" ]; then - err "backend already dead" -fi -# kill backend -stop_backend -f $cfg -sudo pkill -u root -f clixon_backend +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + sudo pkill -u root -f clixon_backend + + rm -rf $dir +fi + -rm -rf $dir diff --git a/test/test_yang_namespace.sh b/test/test_yang_namespace.sh index 2043ac69..13877e2d 100755 --- a/test/test_yang_namespace.sh +++ b/test/test_yang_namespace.sh @@ -99,7 +99,7 @@ new "netconf get config example1 and example2" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^4299]]>]]>$' new "netconf discard-changes" -expecteof "$clixon_netconf -qf $cfg -y $fyang1" 0 "]]>]]>" "^]]>]]>$" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "restconf set x in example1" expecteq "$(curl -s -X POST -d '{"example1:x":42}' http://localhost/restconf/data)" 0 ''