diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ce3d157..2454dfec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Clixon Changelog +## 4.4.0 (Expected: February 2020) + +### Major New features + +* New and updated search functions using xpath, api-path and instance-id + * New search functions using api-path and instance_id: + * C search functions: `clixon_find_instance_id()` and `clixon_find_api_path()` + * Binary search optimization in lists for indexed leafs in all three formats. + * This improves search performance to O(logN) which is drastical improvements for large lists. + * You can also register explicit indexes for making binary search (not only list keys) + * For more info, see docs at [paths](https://clixon-docs.readthedocs.io/en/latest/paths.html) and +[search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml) + +### API changes on existing features (you may need to change your code) +* On failed validation of leafrefs, error message changed from: `No such leaf` to `No leaf matching path `. +* CLI Error message (clicon_rpc_generate_error()) changed when backend returns netconf error to be more descriptive: + * Original: `Config error: Validate failed. Edit and try again or discard changes: Invalid argument` + * New (example): `Netconf error: application operation-failed Identityref validation failed, undefined not derived from acl-base . Validate failed. Edit and try again or discard changes" + +### Minor changes + +* Test framework + * Added `-- -S ` command-line to main example to be able to return any state to main example. + * Added `test/cicd` test scripts for running on a set of other hosts +* C-code restructuring + * clixon_yang.c partitioned and moved code into clixon_yang_parse_lib.c and clixon_yang_module.c and move back some code from clixon_yang_type.c. + * partly to reduce size, but most important to limit code that accesses internal yang structures, only clixon_yang.c does this now. + +### Corrected Bugs + +## 4.3.1 (2 February 2020) + +Patch release based on testing by Dave Cornejo, Netgate + +### Corrected Bugs +* Compile option `VALIDATE_STATE_XML` introduced in `include/custom.h` to control whether code for state data validation is compiled or not. +* Fixed: Validation of user state data led to wrong validation, if state relied on config data, eg leafref/must/when etc. +* Fixed: No revision in yang module led to errors in validation of state data +* Fixed: Leafref validation did not cover case of when the "path" statement is declared within a typedef, only if it was declared in the data part directly under leaf. +* Fixed: Yang `must` xpath statements containing prefixes stopped working due to namespace context updates + ## 4.3.0 (1 January 2020) There were several issues with multiple namespaces with augmented yangs in 4.2 that have been fixed in 4.3. Some other highlights include: several issues with XPaths including "canonical namespace context" support, a reorganization of the YANG files shipped with the release, and a wildchar in the CLICON_MODE variable. @@ -12,6 +53,7 @@ There were several issues with multiple namespaces with augmented yangs in 4.2 t * Optional yang files can be installed in a separate dir with `--with-opt-yang-installdir=DIR` (renamed from `with-std-yang-installdir`) * C-API * Changed `clicon_rpc_generate_error(msg, xerr)` to `clicon_rpc_generate_error(xerr, msg, arg)` + * If you pass NULL as arg it produces the same message as before. * Added namespace-context parameter `nsc` to `xpath_first` and `xpath_vec`, (`xpath_vec_nsc` and xpath_first_nsc` are removed). * Added clicon_handle as parameter to all `clicon_connect_` functions to get better error message diff --git a/LICENSE.md b/LICENSE.md index af683735..d0d73b96 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,6 @@ -Copyright 2009-2019 Olof Hagsand and Benny Holmgren +Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren +Copyright (C) 2017-2020 Olof Hagsand + CLIXON is dual license. diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 943e9510..35cb40bb 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -252,6 +253,7 @@ client_get_streams(clicon_handle h, * @param[in] h Clicon handle * @param[in] xpath Xpath selection, not used but may be to filter early * @param[in] nsc XML Namespace context for xpath + * @param[in] content config/state or both * @param[in,out] xret Existing XML tree, merge x into this * @retval -1 Error (fatal) * @retval 0 Statedata callback failed (clicon_err called) @@ -261,17 +263,15 @@ static int client_statedata(clicon_handle h, char *xpath, cvec *nsc, + netconf_content content, cxobj **xret) { int retval = -1; - cxobj **xvec = NULL; - size_t xlen; - int i; yang_stmt *yspec; yang_stmt *ymod; int ret; char *namespace; - + if ((yspec = clicon_dbspec_yang(h)) == NULL){ clicon_err(OE_YANG, ENOENT, "No yang spec"); goto done; @@ -320,32 +320,9 @@ client_statedata(clicon_handle h, goto done; if (ret == 0) goto fail; - /* Code complex to filter out anything that is outside of xpath - * Actually this is a safety catch, should really be done in plugins - * and modules_state functions. - */ - if (xpath_vec(*xret, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) - goto done; - /* If vectors are specified then mark the nodes found and - * then filter out everything else, - * otherwise return complete tree. - */ - if (xvec != NULL){ - for (i=0; i + * Function reused from both from_client_get() and from_client_get_config + * @param[in] yspec + * @param[in] db + * @param[in] xpath + * @param[in] username + * @param[in] content + * @param[in] depth * @param[out] cbret Return xml tree, eg ..., - * The set of namespace declarations are those in scope on the - * element. - */ - else - if (xml_nsctx_node(xfilter, &nsc) < 0) - goto done; - if (xpath2canonical(xpath0, nsc, yspec, &xpath, &nsc1) < 0) - goto done; - if (nsc) - xml_nsctx_free(nsc); - nsc = nsc1; - } /* Note xret can be pruned by nacm below (and change name), * so zero-copy cant be used * Also, must use external namespace context here due to 0?depth+1:depth) < 0) goto done; } cprintf(cbret, ""); ok: retval = 0; done: - if (xpath) - free(xpath); - if (xnacm) - xml_free(xnacm); if (xvec) free(xvec); + if (xnacm) + xml_free(xnacm); + if (xret) + xml_free(xret); + return retval; +} + +/*! Retrieve all or part of a specified configuration. + * + * @param[in] h Clicon handle + * @param[in] xe Request: + * @param[out] cbret Return xml tree, eg ..., + * The set of namespace declarations are those in scope on the + * element. + */ + else + if (xml_nsctx_node(xfilter, &nsc) < 0) + goto done; + if (xpath2canonical(xpath0, nsc, yspec, &xpath, &nsc1) < 0) + goto done; + if (nsc) + xml_nsctx_free(nsc); + nsc = nsc1; + } + /* Clixon extensions: depth */ + if ((attr = xml_find_value(xe, "depth")) != NULL){ + char *reason = NULL; + if ((ret = parse_int32(attr, &depth, &reason)) < 0){ + clicon_err(OE_XML, errno, "parse_int32"); + goto done; + } + if (ret == 0){ + if (netconf_bad_attribute(cbret, "application", + "depth", "Unrecognized value of depth attribute") < 0) + goto done; + goto ok; + } + } + if ((ret = client_config_only(h, nsc, yspec, db, xpath, username, -1, cbret)) < 0) + goto done; + ok: + retval = 0; + done: + if (xpath) + free(xpath); if (nsc) xml_nsctx_free(nsc); if (cbx) cbuf_free(cbx); - if (xret) - xml_free(xret); return retval; } @@ -493,7 +521,6 @@ from_client_edit_config(clicon_handle h, cxobj *xc; cxobj *x; enum operation_type operation = OP_MERGE; - int non_config = 0; yang_stmt *yspec; cbuf *cbx = NULL; /* Assist cbuf */ @@ -885,20 +912,26 @@ from_client_get(clicon_handle h, void *arg, void *regarg) { - int retval = -1; - cxobj *xfilter; - char *xpath = NULL; - cxobj *xret = NULL; - int ret; - cxobj **xvec = NULL; - size_t xlen; - cxobj *xnacm = NULL; - char *username; - cvec *nsc = NULL; /* Create a netconf namespace context from filter */ - char *attr; + int retval = -1; + cxobj *xfilter; + char *xpath = NULL; + cxobj *xret = NULL; + int ret; + cxobj **xvec = NULL; + size_t xlen; + cxobj *xnacm = NULL; + char *username; + cvec *nsc = NULL; /* Create a netconf namespace context from filter */ + char *attr; netconf_content content = CONTENT_ALL; - int32_t depth = -1; /* Nr of levels to print, -1 is all, 0 is none */ - yang_stmt *yspec; + int32_t depth = -1; /* Nr of levels to print, -1 is all, 0 is none */ + yang_stmt *yspec; + int i; +#ifdef VALIDATE_STATE_XML + cxobj *xerr = NULL; + cxobj *xr; + cxobj *xb; +#endif username = clicon_username_get(h); if ((yspec = clicon_dbspec_yang(h)) == NULL){ @@ -923,10 +956,10 @@ from_client_get(clicon_handle h, xml_nsctx_free(nsc); nsc = nsc1; } - /* Clixon extensions: depth and content */ + /* Clixon extensions: content */ if ((attr = xml_find_value(xe, "content")) != NULL) content = netconf_content_str2int(attr); - + /* Clixon extensions: depth */ if ((attr = xml_find_value(xe, "depth")) != NULL){ char *reason = NULL; if ((ret = parse_int32(attr, &depth, &reason)) < 0){ @@ -940,29 +973,105 @@ from_client_get(clicon_handle h, goto ok; } } - if (content != CONTENT_NONCONFIG){ - /* Get config - * Note xret can be pruned by nacm below and change name and - * metrged with state data, so zero-copy cant be used - * Also, must use external namespace context here due to 0 && (ret = xml_yang_validate_add(h, xret, &xerr)) < 0) + goto done; + if (ret == 0){ +#if 1 + if (debug){ + cbuf *ccc=cbuf_new(); + if (clicon_xml2cbuf(ccc, xret, 0, 0, -1) < 0) + goto done; + clicon_debug(1, "%s FAIL: %s", __FUNCTION__, cbuf_get(ccc)); + cbuf_free(ccc); + } +#endif + if ((xr = xpath_first(xerr, NULL, "//error-tag")) != NULL && + (xb = xml_body_get(xr))){ + if (xml_value_set(xb, "operation-failed") < 0) + goto done; + } + if ((xr = xpath_first(xerr, NULL, "//error-message")) != NULL && + (xb = xml_body_get(xr))){ + if (xml_value_append(xb, " Internal error, state callback returned invalid XML") < 0) + goto done; + } + if (clicon_xml2cbuf(cbret, xerr, 0, 0, -1) < 0) + goto done; + goto ok; + } +#endif /* VALIDATE_STATE_XML */ + if (content == CONTENT_NONCONFIG){ /* state only, all config should be removed now */ + /* Keep state data only, remove everything that is not config. Note that state data + * may be a sub-part in a config tree, we need to traverse to find all + */ + if (xml_apply(xret, CX_ELMNT, xml_non_config_data, NULL) < 0) + goto done; + if (xml_tree_prune_flagged_sub(xret, XML_FLAG_MARK, 1, NULL) < 0) + goto done; + if (xml_apply(xret, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)XML_FLAG_MARK) < 0) + goto done; + } + /* Code complex to filter out anything that is outside of xpath + * Actually this is a safety catch, should really be done in plugins + * and modules_state functions. + */ + if (xpath_vec(xret, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) + goto done; + /* If vectors are specified then mark the nodes found and + * then filter out everything else, + * otherwise return complete tree. + */ + if (xvec != NULL){ + for (i=0; i 0 && (ret = xml_yang_validate_add(h, x, &xerr)) < 0) - goto done; - if (ret == 0){ - cxobj *xe; - cxobj *xb; - - if ((xe = xpath_first(xerr, NULL, "//error-tag")) != NULL && - (xb = xml_body_get(xe))){ - if (xml_value_set(xb, "operation-failed") < 0) - goto done; - } - if ((xe = xpath_first(xerr, NULL, "//error-message")) != NULL && - (xb = xml_body_get(xe))){ - if (xml_value_append(xb, " Internal error, state callback returned invalid XML") < 0) - goto done; - } - if (*xret){ - xml_free(*xret); - *xret = NULL; - } - *xret = xerr; - xerr = NULL; - goto fail; - } #if 1 if (debug){ cbuf *ccc=cbuf_new(); if (clicon_xml2cbuf(ccc, x, 0, 0, -1) < 0) goto done; - clicon_debug(1, "%s MERGE: %s", __FUNCTION__, cbuf_get(ccc)); + clicon_debug(1, "%s STATE: %s", __FUNCTION__, cbuf_get(ccc)); cbuf_free(ccc); } #endif + if (xml_apply(x, CX_ELMNT, xml_spec_populate, yspec) < 0) + goto done; if ((ret = netconf_trymerge(x, yspec, xret)) < 0) goto done; if (ret == 0) @@ -172,15 +143,13 @@ clixon_plugin_statedata(clicon_handle h, xml_free(x); x = NULL; } - } + } /* while plugin */ retval = 1; done: if (cberr) cbuf_free(cberr); if (x) xml_free(x); - if (xerr) - xml_free(xerr); return retval; fail: retval = 0; diff --git a/apps/cli/cli_generate.c b/apps/cli/cli_generate.c index 6849b63c..55909adb 100644 --- a/apps/cli/cli_generate.c +++ b/apps/cli/cli_generate.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. diff --git a/apps/cli/cli_generate.h b/apps/cli/cli_generate.h index 2571fe91..a05c2cf1 100644 --- a/apps/cli/cli_generate.h +++ b/apps/cli/cli_generate.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. diff --git a/apps/cli/cli_main.c b/apps/cli/cli_main.c index 9f0510b3..980988f0 100644 --- a/apps/cli/cli_main.c +++ b/apps/cli/cli_main.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -174,6 +175,7 @@ cli_terminate(clicon_handle h) cvec_free(nsctx); if ((x = clicon_conf_xml(h)) != NULL) xml_free(x); + xpath_optimize_exit(); cli_plugin_finish(h); cli_history_save(h); cli_handle_exit(h); diff --git a/apps/netconf/Makefile.in b/apps/netconf/Makefile.in index 7c761fbb..f3157039 100644 --- a/apps/netconf/Makefile.in +++ b/apps/netconf/Makefile.in @@ -1,7 +1,8 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2020 Olof Hagsand # # This file is part of CLIXON # diff --git a/apps/netconf/netconf_main.c b/apps/netconf/netconf_main.c index 2844a67d..5efd6c77 100644 --- a/apps/netconf/netconf_main.c +++ b/apps/netconf/netconf_main.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -336,6 +337,7 @@ netconf_terminate(clicon_handle h) cvec_free(nsctx); if ((x = clicon_conf_xml(h)) != NULL) xml_free(x); + xpath_optimize_exit(); event_exit(); clicon_handle_exit(h); clicon_log_exit(); diff --git a/apps/restconf/Makefile.in b/apps/restconf/Makefile.in index 688e2648..16c75f23 100644 --- a/apps/restconf/Makefile.in +++ b/apps/restconf/Makefile.in @@ -1,7 +1,7 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2009-2020 Olof Hagsand # # This file is part of CLIXON # diff --git a/apps/restconf/restconf_lib.c b/apps/restconf/restconf_lib.c index 2ab65c2d..c0ce4202 100644 --- a/apps/restconf/restconf_lib.c +++ b/apps/restconf/restconf_lib.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -611,6 +611,7 @@ restconf_terminate(clicon_handle h) cvec_free(nsctx); if ((x = clicon_conf_xml(h)) != NULL) xml_free(x); + xpath_optimize_exit(); clicon_handle_exit(h); clicon_log_exit(); return 0; diff --git a/configure b/configure index c7b411dd..4955df59 100755 --- a/configure +++ b/configure @@ -2172,9 +2172,9 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu : ${INSTALLFLAGS="-s"} CLIXON_VERSION_MAJOR="4" -CLIXON_VERSION_MINOR="3" +CLIXON_VERSION_MINOR="4" CLIXON_VERSION_PATCH="0" -CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\"" +CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\"" # Check CLIgen if test "$prefix" = "NONE"; then @@ -4976,7 +4976,7 @@ _ACEOF -ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile example/hello/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile" +ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile example/hello/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/cicd/Makefile" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure @@ -5696,6 +5696,7 @@ do "yang/optional/Makefile") CONFIG_FILES="$CONFIG_FILES yang/optional/Makefile" ;; "doc/Makefile") CONFIG_FILES="$CONFIG_FILES doc/Makefile" ;; "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;; + "test/cicd/Makefile") CONFIG_FILES="$CONFIG_FILES test/cicd/Makefile" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac diff --git a/configure.ac b/configure.ac index 7b5fbc70..22ba57cd 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,9 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2019 Olof Hagsand +# Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC # # This file is part of CLIXON # @@ -43,9 +45,9 @@ AC_INIT(lib/clixon/clixon.h.in) : ${INSTALLFLAGS="-s"} CLIXON_VERSION_MAJOR="4" -CLIXON_VERSION_MINOR="3" +CLIXON_VERSION_MINOR="4" CLIXON_VERSION_PATCH="0" -CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\"" +CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\"" # Check CLIgen if test "$prefix" = "NONE"; then @@ -283,6 +285,7 @@ AC_OUTPUT(Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile - test/Makefile + test/Makefile + test/cicd/Makefile ) diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index 630dc43b..2616e9cd 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -1,7 +1,7 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2020 Olof Hagsand # # This file is part of CLIXON # diff --git a/docker/main/Makefile.in b/docker/main/Makefile.in index 7f034a9b..b14b23d3 100644 --- a/docker/main/Makefile.in +++ b/docker/main/Makefile.in @@ -1,7 +1,7 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2020 Olof Hagsand # # This file is part of CLIXON # diff --git a/docker/main/start.sh b/docker/main/start.sh index 88995157..365154b7 100755 --- a/docker/main/start.sh +++ b/docker/main/start.sh @@ -1,4 +1,35 @@ #!/bin/bash +# ***** BEGIN LICENSE BLOCK ***** +# +# Copyright (C) 2017-2020 Olof Hagsand +# +# This file is part of CLIXON +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Alternatively, the contents of this file may be used under the terms of +# the GNU General Public License Version 3 or later (the "GPL"), +# in which case the provisions of the GPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of the GPL, and not to allow others to +# use your version of this file under the terms of Apache License version 2, +# indicate your decision by deleting the provisions above and replace them with +# the notice and other provisions required by the GPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the Apache License version 2 or the GPL. +# +# ***** END LICENSE BLOCK ***** + # Usage: ./startup.sh # Debug: DBG=1 ./startup.sh # See also cleanup.sh diff --git a/docker/main/startsystem.sh b/docker/main/startsystem.sh index 5b54824d..0f7cfb89 100755 --- a/docker/main/startsystem.sh +++ b/docker/main/startsystem.sh @@ -1,4 +1,36 @@ #!/bin/sh + +# ***** BEGIN LICENSE BLOCK ***** +# +# Copyright (C) 2017-2020 Olof Hagsand +# +# This file is part of CLIXON +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Alternatively, the contents of this file may be used under the terms of +# the GNU General Public License Version 3 or later (the "GPL"), +# in which case the provisions of the GPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of the GPL, and not to allow others to +# use your version of this file under the terms of Apache License version 2, +# indicate your decision by deleting the provisions above and replace them with +# the notice and other provisions required by the GPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the Apache License version 2 or the GPL. +# +# ***** END LICENSE BLOCK ***** + # This script is copied into the container on build time and runs # _inside_ the container at start in runtime. It gets environment variables # from the start.sh script. @@ -53,7 +85,7 @@ EOF # - test_order.sh XXX this is a bug need debugging cat < /usr/local/bin/test/site.sh # Add your local site specific env variables (or tests) here. -SKIPLIST="test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh" +SKIPLIST="test_api.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh" #IETFRFC= EOF diff --git a/example/main/example_backend.c b/example/main/example_backend.c index b8db088d..ac45dfbc 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -47,6 +48,7 @@ #include #include #include +#include #include /* clicon */ @@ -71,6 +73,11 @@ static int _reset = 0; */ static int _state = 0; +/*! File where state XML is read from, if _state is true + * Primarily for testing + */ +static char *_state_file = NULL; + /*! Variable to control upgrade callbacks. * If set, call test-case for upgrading ietf-interfaces, otherwise call * auto-upgrade @@ -307,55 +314,67 @@ example_statedata(clicon_handle h, cvec *nsc1 = NULL; cvec *nsc2 = NULL; yang_stmt *yspec = NULL; + int fd; if (!_state) goto ok; yspec = clicon_dbspec_yang(h); - /* Example of statedata, in this case merging state data with - * state information. In this case adding dummy interface operation state - * to configured interfaces. - * Get config according to xpath */ - if ((nsc1 = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:yang:ietf-interfaces")) == NULL) - goto done; - if (xmldb_get0(h, "running", nsc1, "/interfaces/interface/name", 1, &xt, NULL) < 0) - goto done; - if (xpath_vec(xt, nsc1, "/interfaces/interface/name", &xvec, &xlen) < 0) - goto done; - if (xlen){ - cprintf(cb, ""); - for (i=0; i%sex:ethup", name); - cprintf(cb, "42foo"); - cprintf(cb, ""); + /* If -S is set, then read state data from file, otherwise construct it programmatically */ + if (_state_file){ + if ((fd = open(_state_file, O_RDONLY)) < 0){ + clicon_err(OE_UNIX, errno, "open(%s)", _state_file); + goto done; } - cprintf(cb, ""); - if (xml_parse_string(cbuf_get(cb), NULL, &xstate) < 0) + if (xml_parse_file(fd, NULL, yspec, &xstate) < 0) goto done; } - /* State in test_yang.sh , test_restconf.sh and test_order.sh */ - if (yang_find_module_by_namespace(yspec, "urn:example:clixon") != NULL){ - if (xml_parse_string("" - "42" - "41" - "43" /* should not be ordered */ - "", NULL, &xstate) < 0) - goto done; /* For the case when urn:example:clixon is not loaded */ - } - /* Event state from RFC8040 Appendix B.3.1 - * Note: (1) order is by-system so is different, - * (2) event-count is XOR on name, so is not 42 and 4 - */ - if (yang_find_module_by_namespace(yspec, "urn:example:events") != NULL){ - cbuf_reset(cb); - cprintf(cb, ""); - cprintf(cb, "interface-down90"); - cprintf(cb, "interface-up77"); - cprintf(cb, ""); - if (xml_parse_string(cbuf_get(cb), NULL, &xstate) < 0) - goto done; - } + else { + /* Example of statedata, in this case merging state data with + * state information. In this case adding dummy interface operation state + * to configured interfaces. + * Get config according to xpath */ + if ((nsc1 = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:yang:ietf-interfaces")) == NULL) + goto done; + if (xmldb_get0(h, "running", nsc1, "/interfaces/interface/name", 1, &xt, NULL) < 0) + goto done; + if (xpath_vec(xt, nsc1, "/interfaces/interface/name", &xvec, &xlen) < 0) + goto done; + if (xlen){ + cprintf(cb, ""); + for (i=0; i%sex:ethup", name); + cprintf(cb, "42foo"); + cprintf(cb, ""); + } + cprintf(cb, ""); + if (xml_parse_string(cbuf_get(cb), NULL, &xstate) < 0) + goto done; + } + /* State in test_yang.sh , test_restconf.sh and test_order.sh */ + if (yang_find_module_by_namespace(yspec, "urn:example:clixon") != NULL){ + if (xml_parse_string("" + "42" + "41" + "43" /* should not be ordered */ + "", NULL, &xstate) < 0) + goto done; /* For the case when urn:example:clixon is not loaded */ + } + /* Event state from RFC8040 Appendix B.3.1 + * Note: (1) order is by-system so is different, + * (2) event-count is XOR on name, so is not 42 and 4 + */ + if (yang_find_module_by_namespace(yspec, "urn:example:events") != NULL){ + cbuf_reset(cb); + cprintf(cb, ""); + cprintf(cb, "interface-down90"); + cprintf(cb, "interface-up77"); + cprintf(cb, ""); + if (xml_parse_string(cbuf_get(cb), NULL, &xstate) < 0) + goto done; + } + } ok: retval = 0; done: @@ -709,7 +728,7 @@ clixon_plugin_init(clicon_handle h) goto done; opterr = 0; optind = 1; - while ((c = getopt(argc, argv, "rsut:")) != -1) + while ((c = getopt(argc, argv, "rsS:ut")) != -1) switch (c) { case 'r': _reset = 1; @@ -717,6 +736,9 @@ clixon_plugin_init(clicon_handle h) case 's': _state = 1; break; + case 'S': /* state file */ + _state_file = optarg; + break; case 'u': _upgrade = 1; break; diff --git a/example/main/example_cli.cli b/example/main/example_cli.cli index 4f793319..1ef4fe6e 100644 --- a/example/main/example_cli.cli +++ b/example/main/example_cli.cli @@ -1,4 +1,36 @@ -# Common CLI syntax for both server and PMNode operatio mode +# ***** BEGIN LICENSE BLOCK ***** +# +# Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2020 Olof Hagsand +# +# This file is part of CLIXON +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Alternatively, the contents of this file may be used under the terms of +# the GNU General Public License Version 3 or later (the "GPL"), +# in which case the provisions of the GPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of the GPL, and not to allow others to +# use your version of this file under the terms of Apache License version 2, +# indicate your decision by deleting the provisions above and replace them with +# the notice and other provisions required by the GPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the Apache License version 2 or the GPL. +# +# ***** END LICENSE BLOCK ***** + +# Clixon example specification CLICON_MODE="example"; CLICON_PROMPT="%U@%H> "; CLICON_PLUGIN="example_cli"; @@ -37,8 +69,8 @@ show("Show a particular state of the system"){ xml("Show comparison in xml"), compare_dbs((int32)0); text("Show comparison in text"), compare_dbs((int32)1); } - state("Show configuration and state"), cli_show_config_state("running", "text", "/"){ - xml("Show configuration and state as XML"), cli_show_config_state("candidate", "xml", "/");{ + state("Show configuration and state"), cli_show_config_state("running", "text", "/");{ + xml("Show configuration and state as XML"), cli_show_config_state("running", "xml", "/");{ @datamodel, cli_show_auto_state("running", "xml"); } } diff --git a/include/clixon_custom.h b/include/clixon_custom.h index 2da8699c..f816dd63 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -1,7 +1,7 @@ /* ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -63,4 +63,18 @@ * Identify xpaths that search for exactly a list key, eg: "y[k=3]" and then call * binary search. This only works if "y" has proper yang binding and is sorted by system */ -#undef XPATH_LIST_OPTIMIZE +#define XPATH_LIST_OPTIMIZE + +/*! Add search indexes, so that binary search can be made for non-key list indexes + * This also applies if there are multiple keys and you want to search on only the second for + * example. + */ +#undef XML_EXTRA_INDEX + +/*! Validate user state callback content + * Use may register state callbacks using ca_statedata callback + * When this option is set, the XML returned from the callback is validated after merging with the running + * db. If it fails, an internal error is returned to the originating user. + * If the option is not set, the XML returned by the user is not validated. + */ +#define VALIDATE_STATE_XML diff --git a/lib/clixon/clixon.h.in b/lib/clixon/clixon.h.in index 5ed5991f..cb81a4e5 100644 --- a/lib/clixon/clixon.h.in +++ b/lib/clixon/clixon.h.in @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -73,6 +74,7 @@ #include #include #include +#include #include #include #include @@ -82,7 +84,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/clixon/clixon_err.h b/lib/clixon/clixon_err.h index b9daf309..3931b48b 100644 --- a/lib/clixon/clixon_err.h +++ b/lib/clixon/clixon_err.h @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -58,9 +59,10 @@ enum clicon_err{ /* 0 means error not set) */ OE_DB = 1, /* database registries */ - OE_DAEMON, /* demons: pidfiles, etc */ + OE_DAEMON, /* demons: pidfiles, etc */ OE_EVENTS, /* events, filedescriptors, timeouts */ OE_CFG, /* configuration */ + OE_NETCONF, /* Netconf error */ OE_PROTO, /* config/client communication */ OE_REGEX, /* Regexp error */ OE_UNIX, /* unix/linux syscall error */ diff --git a/lib/clixon/clixon_api_path.h b/lib/clixon/clixon_path.h similarity index 58% rename from lib/clixon/clixon_api_path.h rename to lib/clixon/clixon_path.h index b1fba9d9..bc45a145 100644 --- a/lib/clixon/clixon_api_path.h +++ b/lib/clixon/clixon_path.h @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -31,6 +32,8 @@ ***** END LICENSE BLOCK ***** + * "Instance-identifier" is a subset of XML Xpaths and defined in Yang, used in NACM for example. + * and defined in RF7950 Sections 9.13 and 14. * * "api-path" is "URI-encoded path expression" definition in RFC8040 3.5.3 * BNF: @@ -44,8 +47,29 @@ * := ( | "_") ( | | "_" | "-" | ".") */ -#ifndef _CLIXON_API_PATH_H_ -#define _CLIXON_API_PATH_H_ +#ifndef _CLIXON_PATH_H_ +#define _CLIXON_PATH_H_ + +/* + * Types + */ +/* Internal path structure. Somewhat more general than api-path, much less than xpath + * about the same as yang instance-identifier + * Not that cp_cvk api-paths do not specifiy key-names, so cp_cvk is just a list of + * (NULL:value)*, which means that names must be added using api_path_check() based on + * yang. + * Other formats (eg xpath) have the names given in the format. + */ +typedef struct { + qelem_t cp_qelem; /* List header */ + char *cp_prefix; /* Prefix or module name, should be resolved + id to cp_yang */ + char *cp_id; /* Identifier */ + cvec *cp_cvk; /* Key values: list of (name:value) pairs alt (NULL:value) + * Can also be single uint32, if so positional eg x/y[42] + * This seems kludgy but follows RFC 7950 Sec 9.13 + */ + yang_stmt *cp_yang; /* Corresponding yang spec (after XML match - ie resolved) */ +} clixon_path; /* * Prototypes @@ -60,5 +84,14 @@ int api_path2xml(char *api_path, yang_stmt *yspec, cxobj *xtop, yang_class nodeclass, int strict, cxobj **xpathp, yang_stmt **ypathp, cxobj **xerr); int xml2api_path_1(cxobj *x, cbuf *cb); +#if defined(__GNUC__) && __GNUC__ >= 3 +int clixon_xml_find_api_path(cxobj *xcur, yang_stmt *yt, cxobj ***vec, size_t *veclen, char *format, + ...) __attribute__ ((format (printf, 5, 6)));; +int clixon_xml_find_instance_id(cxobj *xcur, yang_stmt *yt, cxobj ***vec, size_t *veclen, char *format, + ...) __attribute__ ((format (printf, 5, 6)));; +#else +int clixon_xml_find_api_path(cxobj *xcur, yang_stmt *yt, cxobj ***vec, size_t *veclen, char *format,o ...); +int clixon_xml_find_instance_id(cxobj *xcur, yang_stmt *yt, cxobj ***vec, size_t *veclen, char *format, ...); +#endif -#endif /* _CLIXON_API_PATH_H_ */ +#endif /* _CLIXON_PATH_H_ */ diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 429fb191..4fa1d273 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -151,7 +152,6 @@ int xml_spec_set(cxobj *x, yang_stmt *spec); cg_var *xml_cv(cxobj *x); int xml_cv_set(cxobj *x, cg_var *cv); cxobj *xml_find(cxobj *xn_parent, char *name); - int xml_addsub(cxobj *xp, cxobj *xc); cxobj *xml_wrap_all(cxobj *xp, char *tag); cxobj *xml_wrap(cxobj *xc, char *tag); @@ -193,6 +193,7 @@ cxobj *xml_dup(cxobj *x0); int cxvec_dup(cxobj **vec0, size_t len0, cxobj ***vec1, size_t *len1); int cxvec_append(cxobj *x, cxobj ***vec, size_t *len); +int cxvec_prepend(cxobj *x, cxobj ***vec, size_t *len); int xml_apply(cxobj *xn, enum cxobj_type type, xml_applyfn_t fn, void *arg); int xml_apply0(cxobj *xn, enum cxobj_type type, xml_applyfn_t fn, void *arg); int xml_apply_ancestor(cxobj *xn, xml_applyfn_t fn, void *arg); diff --git a/lib/clixon/clixon_xml_sort.h b/lib/clixon/clixon_xml_sort.h index d8da6fd8..21c14e32 100644 --- a/lib/clixon/clixon_xml_sort.h +++ b/lib/clixon/clixon_xml_sort.h @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -40,11 +41,13 @@ * Prototypes */ int xml_child_spec(cxobj *x, cxobj *xp, yang_stmt *yspec, yang_stmt **yp); -int xml_cmp(cxobj *x1, cxobj *x2, int enm); +int xml_cmp(cxobj *x1, cxobj *x2, int same, int skip1); int xml_sort(cxobj *x0, void *arg); int xml_insert(cxobj *xp, cxobj *xc, enum insert_type ins, char *key_val, cvec *nsckey); int xml_sort_verify(cxobj *x, void *arg); int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp); -int xml_binsearch(cxobj *xp, yang_stmt *yc, cvec *cvk, cxobj **xretp); +int clixon_xml_find_index(cxobj *xp, yang_stmt *yp, char *namespace, char *name, + cvec *cvk, cxobj ***xvec, size_t *xlen); +int clixon_xml_find_pos(cxobj *xp, yang_stmt *yc, uint32_t pos, cxobj ***xvec, size_t *xlen); #endif /* _CLIXON_XML_SORT_H */ diff --git a/lib/clixon/clixon_xpath_ctx.h b/lib/clixon/clixon_xpath_ctx.h index b1199af4..3a2d2af4 100644 --- a/lib/clixon/clixon_xpath_ctx.h +++ b/lib/clixon/clixon_xpath_ctx.h @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -55,8 +56,9 @@ enum xp_objtype{ XT_STRING }; -/* Expression evaluation occurs with respect to a context. XSLT and XPointer specify how the context is - * determined for XPath expressions used in XSLT and XPointer respectively. The context consists of: +/* Expression evaluation occurs with respect to a context. XSLT and XPointer specify how the + * context is determined for XPath expressions used in XSLT and XPointer respectively. The + * context consists of: * a node (the context node) * a pair of non-zero positive integers (the context position and the context size) * a set of variable bindings diff --git a/lib/clixon/clixon_yang.h b/lib/clixon/clixon_yang.h index 410c79a1..07dfe548 100644 --- a/lib/clixon/clixon_yang.h +++ b/lib/clixon/clixon_yang.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -39,7 +39,22 @@ #ifndef _CLIXON_YANG_H_ #define _CLIXON_YANG_H_ +/* + * Clixon-specific cligen variable (cv) flags + * CLIgen flags defined are in the range 0x01 -0x0f + * An application can use any flags above that + * @see cv_flag + */ +#define V_UNSET 0x10 /* Used by XML code to denote a value is not default */ +/* + * Yang flags used in + */ +#define YANG_FLAG_MARK 0x01 /* (Dynamic) marker for dynamic algorithms, eg expand */ +#ifdef XML_EXTRA_INDEX +#define YANG_FLAG_INDEX 0x02 /* This yang node under list is (extra) index. --> you can access + * list elements using this index with binary search */ +#endif /* * Types @@ -146,12 +161,20 @@ typedef int (yang_applyfn_t)(yang_stmt *ys, void *arg); * Prototypes */ /* Access functions */ +int yang_len_get(yang_stmt *ys); +yang_stmt *yang_child_i(yang_stmt *ys, int i); + yang_stmt *yang_parent_get(yang_stmt *ys); enum rfc_6020 yang_keyword_get(yang_stmt *ys); char *yang_argument_get(yang_stmt *ys); +int yang_argument_set(yang_stmt *ys, char *arg); + cg_var *yang_cv_get(yang_stmt *ys); cvec *yang_cvec_get(yang_stmt *ys); int yang_cvec_set(yang_stmt *ys, cvec *cvv); +uint16_t yang_flag_get(yang_stmt *ys, uint16_t flag); +int yang_flag_set(yang_stmt *ys, uint16_t flag); +int yang_flag_reset(yang_stmt *ys, uint16_t flag); /* Other functions */ yang_stmt *yspec_new(void); @@ -169,10 +192,6 @@ int ys_module_by_xml(yang_stmt *ysp, struct xml *xt, yang_stmt **ymodp); yang_stmt *ys_module(yang_stmt *ys); yang_stmt *ys_real_module(yang_stmt *ys); yang_stmt *ys_spec(yang_stmt *ys); -yang_stmt *yang_find_module_by_prefix(yang_stmt *ys, char *prefix); -yang_stmt *yang_find_module_by_prefix_yspec(yang_stmt *yspec, char *prefix); -yang_stmt *yang_find_module_by_namespace(yang_stmt *yspec, char *namespace); -yang_stmt *yang_find_module_by_name(yang_stmt *yspec, char *name); yang_stmt *yang_find(yang_stmt *yn, int keyword, const char *argument); int yang_match(yang_stmt *yn, int keyword, char *argument); yang_stmt *yang_find_datanode(yang_stmt *yn, char *argument); @@ -186,8 +205,7 @@ int yang_print(FILE *f, yang_stmt *yn); int yang_print_cbuf(cbuf *cb, yang_stmt *yn, int marginal); int if_feature(yang_stmt *yspec, char *module, char *feature); int ys_populate(yang_stmt *ys, void *arg); -yang_stmt *yang_parse_file(int fd, const char *name, yang_stmt *ysp); -yang_stmt *yang_parse_filename(const char *filename, yang_stmt *ysp); +int ys_populate2(yang_stmt *ys, void *arg); int yang_apply(yang_stmt *yn, enum rfc_6020 key, yang_applyfn_t fn, void *arg); int yang_datanode(yang_stmt *ys); @@ -196,17 +214,16 @@ int yang_abs_schema_nodeid(yang_stmt *yspec, yang_stmt *ys, enum rfc_6020 keyword, yang_stmt **yres); int yang_desc_schema_nodeid(yang_stmt *yn, char *schema_nodeid, enum rfc_6020 keyword, yang_stmt **yres); -int ys_parse_date_arg(char *datearg, uint32_t *dateint); - -cg_var *ys_parse(yang_stmt *ys, enum cv_type cvtype); -int ys_parse_sub(yang_stmt *ys, char *extra); int yang_mandatory(yang_stmt *ys); int yang_config(yang_stmt *ys); -int yang_spec_parse_module(clicon_handle h, const char *module, - const char *revision, yang_stmt *yspec); -int yang_spec_parse_file(clicon_handle h, char *filename, yang_stmt *yspec); -int yang_spec_load_dir(clicon_handle h, char *dir, yang_stmt *yspec); +int yang_features(clicon_handle h, yang_stmt *yt); cvec *yang_arg2cvec(yang_stmt *ys, char *delimi); int yang_key_match(yang_stmt *yn, char *name); +int yang_type_cache_regexp_set(yang_stmt *ytype, int rxmode, cvec *regexps); +int yang_type_cache_get(yang_stmt *ytype, yang_stmt **resolved, int *options, + cvec **cvv, cvec *patterns, int *rxmode, cvec *regexps, uint8_t *fraction); +int yang_type_cache_set(yang_stmt *ys, yang_stmt *resolved, int options, cvec *cvv, + cvec *patterns, uint8_t fraction); + #endif /* _CLIXON_YANG_H_ */ diff --git a/lib/clixon/clixon_yang_module.h b/lib/clixon/clixon_yang_module.h index 92186811..341e765b 100644 --- a/lib/clixon/clixon_yang_module.h +++ b/lib/clixon/clixon_yang_module.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -66,7 +66,10 @@ char *yang_modules_revision(clicon_handle h); int yang_modules_state_get(clicon_handle h, yang_stmt *yspec, char *xpath, cvec *nsc, int brief, cxobj **xret); - int clixon_module_upgrade(clicon_handle h, cxobj *xt, modstate_diff_t *msd, cbuf *cb); +yang_stmt *yang_find_module_by_prefix(yang_stmt *ys, char *prefix); +yang_stmt *yang_find_module_by_prefix_yspec(yang_stmt *yspec, char *prefix); +yang_stmt *yang_find_module_by_namespace(yang_stmt *yspec, char *namespace); +yang_stmt *yang_find_module_by_name(yang_stmt *yspec, char *name); #endif /* _CLIXON_YANG_MODULE_H_ */ diff --git a/lib/clixon/clixon_yang_parse_lib.h b/lib/clixon/clixon_yang_parse_lib.h new file mode 100644 index 00000000..48c05efd --- /dev/null +++ b/lib/clixon/clixon_yang_parse_lib.h @@ -0,0 +1,64 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2020 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * CALLING ORDER OF YANG PARSE FILES + * ================================= + * yang_spec_parse_module + * | | + * v v v + * yang_spec_parse_file-> yang_parse_post->yang_parse_recurse->yang_parse_module + * \ / v + * yang_spec_load_dir ------------------------------------> yang_parse_filename + * v + * yang_parse_file + * v + * yang_parse_str + */ + +#ifndef _CLIXON_YANG_PARSE_LIB_H_ +#define _CLIXON_YANG_PARSE_LIB_H_ + +/* + * Prototypes + */ +yang_stmt *yang_parse_file(int fd, const char *name, yang_stmt *ysp); +yang_stmt *yang_parse_filename(const char *filename, yang_stmt *ysp); +int yang_spec_parse_module(clicon_handle h, const char *module, + const char *revision, yang_stmt *yspec); +int yang_spec_parse_file(clicon_handle h, char *filename, yang_stmt *yspec); +int yang_spec_load_dir(clicon_handle h, char *dir, yang_stmt *yspec); +int ys_parse_date_arg(char *datearg, uint32_t *dateint); +cg_var *ys_parse(yang_stmt *ys, enum cv_type cvtype); +int ys_parse_sub(yang_stmt *ys, char *extra); + +#endif /* _CLIXON_YANG_LIB_H_ */ diff --git a/lib/clixon/clixon_yang_type.h b/lib/clixon/clixon_yang_type.h index ecf2c174..289a93ae 100644 --- a/lib/clixon/clixon_yang_type.h +++ b/lib/clixon/clixon_yang_type.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -54,8 +54,6 @@ typedef struct yang_type_cache yang_type_cache; /* * Prototypes */ -int yang_type_cache_cp(yang_type_cache **ycnew, yang_type_cache *ycold); -int yang_type_cache_free(yang_type_cache *ycache); int ys_resolve_type(yang_stmt *ys, void *arg); int yang2cv_type(char *ytype, enum cv_type *cv_type); char *cv2yang_type(enum cv_type cv_type); diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index 053addce..7cf3a045 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -1,7 +1,8 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2020 Olof Hagsand # # This file is part of CLIXON # @@ -68,10 +69,10 @@ INCLUDES = -I. @INCLUDES@ -I$(top_srcdir)/lib/clixon -I$(top_srcdir)/include -I$ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_err.c clixon_event.c \ clixon_string.c clixon_regex.c clixon_handle.c clixon_file.c \ - clixon_xml.c clixon_xml_sort.c clixon_xml_map.c \ - clixon_json.c clixon_yang.c clixon_yang_type.c clixon_yang_module.c \ + clixon_xml.c clixon_xml_sort.c clixon_xml_map.c clixon_json.c \ + clixon_yang.c clixon_yang_type.c clixon_yang_module.c clixon_yang_parse_lib.c \ clixon_yang_cardinality.c clixon_xml_changelog.c clixon_xml_nsctx.c \ - clixon_api_path.c clixon_validate.c \ + clixon_path.c clixon_validate.c \ clixon_hash.c clixon_options.c clixon_data.c clixon_plugin.c \ clixon_proto.c clixon_proto_client.c \ clixon_xpath.c clixon_xpath_ctx.c clixon_xpath_eval.c clixon_xpath_optimize.c \ @@ -82,7 +83,9 @@ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_err.c clixon_event.c \ YACCOBJS := lex.clixon_xml_parse.o clixon_xml_parse.tab.o \ lex.clixon_yang_parse.o clixon_yang_parse.tab.o \ lex.clixon_json_parse.o clixon_json_parse.tab.o \ - lex.clixon_xpath_parse.o clixon_xpath_parse.tab.o + lex.clixon_xpath_parse.o clixon_xpath_parse.tab.o \ + lex.clixon_api_path_parse.o clixon_api_path_parse.tab.o \ + lex.clixon_instance_id_parse.o clixon_instance_id_parse.tab.o # Generated src GENSRC = build.c @@ -105,10 +108,14 @@ clean: rm -f clixon_yang_parse.tab.[ch] clixon_yang_parse.[co] rm -f clixon_json_parse.tab.[ch] clixon_json_parse.[co] rm -f clixon_xpath_parse.tab.[ch] clixon_xpath_parse.[co] + rm -f clixon_api_path_parse.tab.[ch] clixon_api_path_parse.[co] + rm -f clixon_instance_id_parse.tab.[ch] clixon_instance_id_parse.[co] rm -f lex.clixon_xml_parse.c rm -f lex.clixon_yang_parse.c rm -f lex.clixon_json_parse.c rm -f lex.clixon_xpath_parse.c + rm -f lex.clixon_api_path_parse.c + rm -f lex.clixon_instance_id_parse.c ############################################################################# # Implicit rules for lex and yacc. @@ -174,6 +181,32 @@ clixon_xpath_parse.tab.c: clixon_xpath_parse.tab.h lex.clixon_xpath_parse.o : lex.clixon_xpath_parse.c clixon_xpath_parse.tab.h $(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -Wno-error -c $< +# api-path parser +lex.clixon_api_path_parse.c : clixon_api_path_parse.l clixon_api_path_parse.tab.h + $(LEX) -Pclixon_api_path_parse clixon_api_path_parse.l # -d is debug + +clixon_api_path_parse.tab.h: clixon_api_path_parse.y + $(YACC) -l -d -b clixon_api_path_parse -p clixon_api_path_parse clixon_api_path_parse.y # -t is debug + +# extra rule to avoid parallell yaccs +clixon_api_path_parse.tab.c: clixon_api_path_parse.tab.h + +lex.clixon_api_path_parse.o : lex.clixon_api_path_parse.c clixon_api_path_parse.tab.h + $(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -Wno-error -c $< + +# instance-identifier parser +lex.clixon_instance_id_parse.c : clixon_instance_id_parse.l clixon_instance_id_parse.tab.h + $(LEX) -Pclixon_instance_id_parse clixon_instance_id_parse.l # -d is debug + +clixon_instance_id_parse.tab.h: clixon_instance_id_parse.y + $(YACC) -l -d -b clixon_instance_id_parse -p clixon_instance_id_parse clixon_instance_id_parse.y # -t is debug + +# extra rule to avoid parallell yaccs +clixon_instance_id_parse.tab.c: clixon_instance_id_parse.tab.h + +lex.clixon_instance_id_parse.o : lex.clixon_instance_id_parse.c clixon_instance_id_parse.tab.h + $(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -Wno-error -c $< + distclean: clean rm -f Makefile *~ .depend diff --git a/lib/src/clixon_api_path_parse.h b/lib/src/clixon_api_path_parse.h new file mode 100644 index 00000000..67b3e47a --- /dev/null +++ b/lib/src/clixon_api_path_parse.h @@ -0,0 +1,68 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2020 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * "api-path" is "URI-encoded path expression" definition in RFC8040 3.5.3 + */ +#ifndef _CLIXON_API_PATH_PARSE_H_ +#define _CLIXON_API_PATH_PARSE_H_ + +/* + * Types + */ +struct clicon_api_path_yacc_arg{ + const char *ay_name; /* Name of syntax (for error string) */ + int ay_linenum; /* Number of \n in parsed buffer */ + char *ay_parse_string; /* original (copy of) parse string */ + void *ay_lexbuf; /* internal parse buffer from lex */ + clixon_path *ay_top; +}; + +/* + * Variables + */ +extern char *clixon_api_path_parsetext; + +/* + * Prototypes + */ +int api_path_scan_init(struct clicon_api_path_yacc_arg *); +int api_path_scan_exit(struct clicon_api_path_yacc_arg *); + +int api_path_parse_init(struct clicon_api_path_yacc_arg *); +int api_path_parse_exit(struct clicon_api_path_yacc_arg *); + +int clixon_api_path_parselex(void *); +int clixon_api_path_parseparse(void *); +void clixon_api_path_parseerror(void *, char*); + +#endif /* _CLIXON_API_PATH_PARSE_H_ */ diff --git a/lib/src/clixon_api_path_parse.l b/lib/src/clixon_api_path_parse.l new file mode 100644 index 00000000..dcdf08f1 --- /dev/null +++ b/lib/src/clixon_api_path_parse.l @@ -0,0 +1,150 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2020 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + * + * "api-path" is "URI-encoded path expression" definition in RFC8040 3.53. + * BNF: + * := ("/" ( | ))* + * := + * := [ ":"] + * := + * := "=" key-value *("," key-value) + * := + * := + * := ( | "_") ( | | "_" | "-" | ".") + * @note 1. is the RESTCONF root resource (Sec 3.3) omitted in all calls below, it is + * assumed to be stripped from api-path before calling these functions. + * @note 2. characters in a key value string are constrained, and some characters need to be + * percent-encoded, + * XXX For some reason, I cant use "return *string" in these rules, so I resort to symbols + * (eg slash) + */ + +%{ + +#include "clixon_config.h" + +#include +#include +#include +#include + +#include "clixon_api_path_parse.tab.h" /* generated */ + +#include + +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_yang.h" +#include "clixon_log.h" +#include "clixon_string.h" +#include "clixon_xml.h" +#include "clixon_path.h" +#include "clixon_api_path_parse.h" + +/* Redefine main lex function so that you can send arguments to it: _yy is added to arg list */ +#define YY_DECL int clixon_api_path_parselex(void *_ay) + +/* Dont use input function (use user-buffer) */ +#define YY_NO_INPUT + +/* typecast macro */ +#define _AY ((struct clicon_api_path_yacc_arg *)_ay) + +#define MAXBUF 4*4*64*1024 + +#undef clixon_api_path_parsewrap +int +clixon_api_path_parsewrap(void) +{ + return 1; +} + +%} + +namestart [A-Z_a-z] +namechar [A-Z_a-z\-\.0-9] +identifier {namestart}{namechar}* + +%x INIT +%s KEYV + +%% + +[ \t] +\n { _AY->ay_linenum++; } +\r +<> { return X_EOF; } + +\/ { return SLASH;} +\= { BEGIN(KEYV); return EQUAL; } +\: { return COLON; } +{identifier} { clixon_api_path_parselval.string = strdup(yytext); + return IDENTIFIER; } +<. { clixon_api_path_parseerror(_AY, "LEXICAL ERROR\n"); return -1; } + +\, { return COMMA; } +\/ { BEGIN(INIT); return SLASH; } +[^:/?#\[\]@,]+ { clixon_api_path_parselval.string = strdup(yytext); + return STRING;} +. { clixon_api_path_parseerror(_AY, "LEXICAL ERROR\n"); return -1; } + +%% + +/*! Initialize scanner. + */ +int +api_path_scan_init(struct clicon_api_path_yacc_arg *ay) +{ + BEGIN(INIT); + ay->ay_lexbuf = yy_scan_string(ay->ay_parse_string); +#if 1 /* XXX: just to use unput to avoid warning */ + if (0) + yyunput(0, ""); +#endif + + return 0; +} + +/* + * free buffers + * Even within Flex version 2.5 (this is assumed), freeing buffers is different. + */ +int +api_path_scan_exit(struct clicon_api_path_yacc_arg *ay) +{ + yy_delete_buffer(ay->ay_lexbuf); + clixon_api_path_parselex_destroy(); /* modern */ + return 0; +} + diff --git a/lib/src/clixon_api_path_parse.y b/lib/src/clixon_api_path_parse.y new file mode 100644 index 00000000..cff9ee66 --- /dev/null +++ b/lib/src/clixon_api_path_parse.y @@ -0,0 +1,298 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2020 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * "api-path" is "URI-encoded path expression" definition in RFC8040 3.5.3 + * BNF: + * := ("/" ( | ))* + * := # See note 1 below + * := + * := [ ":"] + * := + * := "=" key-value *("," key-value) + * := + * := + * := ( | "_") ( | | "_" | "-" | ".") + * @note 1. is the RESTCONF root resource (Sec 3.3) omitted in all calls below, it is + * assumed to be stripped from api-path before calling these functions. + * @note 2. characters in a key value string are constrained, and some characters need to be + * percent-encoded, + */ + +%start start + + /* Must be here to define YYSTYPE */ +%union { + char *string; + void *stack; /* cv / cvec */ +} + +%token IDENTIFIER +%token STRING +%token SLASH +%token COLON +%token COMMA +%token EQUAL +%token X_EOF + +%type list +%type element +%type api_identifier +%type module_name +%type list_instance +%type key_values +%type key_value + + +%lex-param {void *_ay} /* Add this argument to parse() and lex() function */ +%parse-param {void *_ay} + +%{ +/* Here starts user C-code */ + +/* typecast macro */ +#define _AY ((struct clicon_api_path_yacc_arg *)_ay) + +#define _YYERROR(msg) {clicon_err(OE_XML, 0, "YYERROR %s '%s' %d", (msg), clixon_api_path_parsetext, _AY->ay_linenum); YYERROR;} + +/* add _yy to error parameters */ +#define YY_(msgid) msgid + +#include "clixon_config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "clixon_err.h" +#include "clixon_log.h" +#include "clixon_queue.h" +#include "clixon_string.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_yang.h" +#include "clixon_xml.h" +#include "clixon_path.h" +#include "clixon_api_path_parse.h" + +/* + also called from yacc generated code * +*/ + +void +clixon_api_path_parseerror(void *_ay, + char *s) +{ + clicon_err(OE_XML, 0, "%s on line %d: %s at or before: '%s'", + _AY->ay_name, + _AY->ay_linenum , + s, + clixon_api_path_parsetext); + return; +} + +int +api_path_parse_init(struct clicon_api_path_yacc_arg *ay) +{ + return 0; +} + +int +api_path_parse_exit(struct clicon_api_path_yacc_arg *ay) +{ + return 0; +} + +/*! Append new path structure to clixon path list + */ +static clixon_path * +path_append(clixon_path *list, + clixon_path *new) +{ + clicon_debug(1, "%s()", __FUNCTION__); + if (new == NULL) + return NULL; + ADDQ(new, list); + return list; +} + +/*! Add keyvalue to existing clixon path + */ +static clixon_path * +path_add_keyvalue(clixon_path *cp, + cvec *cvk) +{ + clicon_debug(1, "%s()", __FUNCTION__); + if (cp) + cp->cp_cvk = cvk; + return cp; +} + +static clixon_path * +path_new(char *module_name, + char *id) +{ + clixon_path *cp = NULL; + + clicon_debug(1, "%s(%s,%s)", __FUNCTION__, module_name, id); + if ((cp = malloc(sizeof(*cp))) == NULL){ + clicon_err(OE_UNIX, errno, "malloc"); + goto done; + } + memset(cp, 0, sizeof(*cp)); + if (module_name) + if ((cp->cp_prefix = strdup(module_name)) == NULL){ + clicon_err(OE_UNIX, errno, "strdup"); + goto done; + } + if ((cp->cp_id = strdup(id)) == NULL){ + clicon_err(OE_UNIX, errno, "strdup"); + goto done; + } + return cp; + done: + return NULL; +} + +/*! Append a key-value cv to a cvec, create the cvec if not exist + * @param[in] cvv Either created cvv or NULL, in whihc case it is created + * @param[in] cv Is consumed by thius function (if appended) + * @retval NULL Error + * @retval cvv Cvec + */ +static cvec * +keyval_add(cvec *cvv, + cg_var *cv) +{ + clicon_debug(1, "%s()", __FUNCTION__); + if (cv == NULL) + goto done; + if (cvv == NULL && + (cvv = cvec_new(0)) == NULL) { + clicon_err(OE_UNIX, errno, "cvec_new"); + goto done; + } + if (cvec_append_var(cvv, cv) == NULL){ + clicon_err(OE_UNIX, errno, "cvec_append_var"); + cvv = NULL; + goto done; + } + cv_free(cv); + done: + return cvv; +} + +/*! Create a single key-value as cv and return it + */ +static cg_var * +keyval_set(char *name, + char *val) +{ + cg_var *cv = NULL; + + clicon_debug(1, "%s(%s=%s)", __FUNCTION__, name, val); + if ((cv = cv_new(CGV_STRING)) == NULL){ + clicon_err(OE_UNIX, errno, "cv_new"); + goto done; + } + if (name && cv_name_set(cv, name) == NULL){ + clicon_err(OE_UNIX, errno, "cv_string_set"); + cv = NULL; + goto done; + } + if (cv_string_set(cv, val) == NULL){ + clicon_err(OE_UNIX, errno, "cv_string_set"); + cv = NULL; + goto done; + } + done: + return cv; +} + +%} + + +%% + +/* +*/ + +start : list X_EOF {clicon_debug(2,"top");_AY->ay_top=$1; YYACCEPT; } + ; + +list : list SLASH element { if (($$ = path_append($1, $3)) == NULL) YYABORT; + clicon_debug(2,"list = list / element");} + | { $$ = NULL; + clicon_debug(2,"list = ");} + ; + +element : api_identifier { $$=$1; + clicon_debug(2,"element = api_identifier");} + | list_instance { $$=$1; + clicon_debug(2,"element = list_instance");} + ; + +api_identifier : module_name COLON IDENTIFIER { $$ = path_new($1, $3); free($1); free($3); + clicon_debug(2,"api_identifier = module_name : IDENTIFIER");} + | IDENTIFIER { $$ = path_new(NULL, $1); free($1); + clicon_debug(2,"api_identifier = IDENTIFIER");} + ; + +module_name : IDENTIFIER { $$ = $1; + clicon_debug(2,"module_name = IDENTIFIER");} + ; + +list_instance : api_identifier EQUAL key_values { $$ = path_add_keyvalue($1, $3); + clicon_debug(2,"list_instance->api_identifier = key_values");} + ; + +key_values : key_values COMMA key_value { if (($$ = keyval_add($1, $3)) == NULL) YYABORT; + clicon_debug(2,"key_values->key_values , key_value");} + | key_value { if (($$ = keyval_add(NULL, $1)) == NULL) YYABORT; + clicon_debug(2,"key_values->key_value");} + ; + +key_value : STRING { $$ = keyval_set(NULL, $1); free($1); clicon_debug(2,"keyvalue->STRING"); } + | { $$ = keyval_set(NULL, ""); clicon_debug(2,"keyvalue->"); } + ; + +%% + diff --git a/lib/src/clixon_datastore_write.c b/lib/src/clixon_datastore_write.c index 006fe45e..52fea2d4 100644 --- a/lib/src/clixon_datastore_write.c +++ b/lib/src/clixon_datastore_write.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -853,7 +853,6 @@ xmldb_put(clicon_handle h, xml_name(x1)); goto done; } - if ((de = clicon_db_elmnt_get(h, db)) != NULL){ if (clicon_datastore_cache(h) != DATASTORE_NOCACHE) x0 = de->de_xml; diff --git a/lib/src/clixon_err.c b/lib/src/clixon_err.c index 89a133f1..34c68d7e 100644 --- a/lib/src/clixon_err.c +++ b/lib/src/clixon_err.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -88,6 +89,7 @@ static struct errvec EV[] = { {"Daemon error", OE_DAEMON}, {"Event error", OE_EVENTS}, {"Config error", OE_CFG}, + {"Netconf error", OE_NETCONF}, {"Protocol error", OE_PROTO}, {"Regexp error", OE_REGEX}, {"UNIX error", OE_UNIX}, diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index 5c63e608..df7d4926 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -60,10 +60,11 @@ #include "clixon_string.h" #include "clixon_hash.h" #include "clixon_handle.h" -#include "clixon_yang.h" -#include "clixon_yang_type.h" #include "clixon_options.h" +#include "clixon_yang.h" #include "clixon_xml.h" +#include "clixon_yang_type.h" +#include "clixon_yang_module.h" #include "clixon_xml_sort.h" #include "clixon_xml_map.h" #include "clixon_xml_nsctx.h" /* namespace context */ @@ -331,7 +332,7 @@ json2xml_decode_identityref(cxobj *x, if (yang_find_prefix_by_namespace(y, namespace, &prefix2) == 0){ #ifndef IDENTITYREF_KLUDGE /* Just get the prefix from the module's own namespace */ - if (netconf_unknown_namespace_xml(xerr, "application", + if (xerr && netconf_unknown_namespace_xml(xerr, "application", namespace, "No local prefix corresponding to namespace") < 0) goto done; @@ -365,7 +366,7 @@ json2xml_decode_identityref(cxobj *x, goto done; } else{ - if (netconf_unknown_namespace_xml(xerr, "application", + if (xerr && netconf_unknown_namespace_xml(xerr, "application", prefix, "No module corresponding to prefix") < 0) goto done; @@ -1187,7 +1188,7 @@ json_parse(char *str, * * @code * cxobj *cx = NULL; - * if (json_parse_str(str, &cx) < 0) + * if (json_parse_str(str, yspec, &cx, &xerr) < 0) * err; * xml_free(cx); * @endcode diff --git a/lib/src/clixon_json_parse.h b/lib/src/clixon_json_parse.h index 0f088a32..92dffb92 100644 --- a/lib/src/clixon_json_parse.h +++ b/lib/src/clixon_json_parse.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. diff --git a/lib/src/clixon_json_parse.l b/lib/src/clixon_json_parse.l index 6700a5c5..68b41311 100644 --- a/lib/src/clixon_json_parse.l +++ b/lib/src/clixon_json_parse.l @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -82,7 +82,7 @@ integer {digit}+ real ({digit}+[.]{digit}*)|({digit}*[.]{digit}+) exp ({integer}|{real})[eE][+-]{integer} -%x START +%x START§ %s STRING %s ESCAPE diff --git a/lib/src/clixon_json_parse.y b/lib/src/clixon_json_parse.y index 2994c679..c3cf8850 100644 --- a/lib/src/clixon_json_parse.y +++ b/lib/src/clixon_json_parse.y @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -245,7 +245,7 @@ json_current_body(struct clicon_json_yacc_arg *jy, */ /* top: json -> value is also possible */ -json : value J_EOF { clicon_debug(2,"json->object"); YYACCEPT; } +§json : value J_EOF { clicon_debug(2,"json->object"); YYACCEPT; } ; value : J_TRUE { json_current_body(_JY, "true");} diff --git a/lib/src/clixon_nacm.c b/lib/src/clixon_nacm.c index 51a83c51..b64799be 100644 --- a/lib/src/clixon_nacm.c +++ b/lib/src/clixon_nacm.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -498,7 +498,7 @@ nacm_data_read_xr(cxobj *xt, * For NETCONF filtering purposes, the selection criteria are applied to the * subset of nodes that the user is authorized to read, not the entire datastore. * @note assume mode is internal or external, not disabled - * @node There is unclarity on what "a data node" means wrt a read operation. + * @note There is unclarity on what "a data node" means wrt a read operation. * Suppose a tree is accessed. Is "the data node" just the top of the tree? * (1) Or is it all nodes, recursively, in the data-tree? * (2) Or is the datanode only the requested tree, NOT the whole datatree? diff --git a/lib/src/clixon_netconf_lib.c b/lib/src/clixon_netconf_lib.c index 040d65eb..2e4c8cfc 100644 --- a/lib/src/clixon_netconf_lib.c +++ b/lib/src/clixon_netconf_lib.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -66,6 +67,7 @@ #include "clixon_xpath_ctx.h" #include "clixon_xpath.h" #include "clixon_yang_module.h" +#include "clixon_yang_parse_lib.h" #include "clixon_netconf_lib.h" /*! Create Netconf in-use error XML tree according to RFC 6241 Appendix A diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index 046df602..44a3ed61 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -72,6 +73,7 @@ #include "clixon_data.h" #include "clixon_xpath_ctx.h" #include "clixon_xpath.h" +#include "clixon_yang_parse_lib.h" #include "clixon_netconf_lib.h" #include "clixon_xml_nsctx.h" #include "clixon_validate.h" diff --git a/lib/src/clixon_api_path.c b/lib/src/clixon_path.c similarity index 63% rename from lib/src/clixon_api_path.c rename to lib/src/clixon_path.c index 708c65cd..e2ec092b 100644 --- a/lib/src/clixon_api_path.c +++ b/lib/src/clixon_path.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -31,17 +31,30 @@ ***** END LICENSE BLOCK ***** + * This file has code for several variants of paths in cxobj trees: + * - api-path as defined by RESTCONF + * - instance-identifier as defined by YANG + * - clixon-path is an internal format which both ^ use as internal representation + * + * "Instance-identifier" is a subset of XML Xpaths and defined in Yang, used in NACM for example. + * and defined in RF7950 Sections 9.13 and 14. + * To note: prefixes depend on the XML context in which the value occurs, + * * "api-path" is "URI-encoded path expression" definition in RFC8040 3.5.3 * BNF: * := ("/" ( | ))* - * := + * := # See note 1 below * := [ ":"] * := * := "=" key-value *("," key-value) * := * := * := ( | "_") ( | | "_" | "-" | ".") - + * + * @note 1. is the RESTCONF root resource (Sec 3.3) omitted in all calls below, it is + * assumed to be stripped from api-path before calling these functions. + * @note 2. characters in a key value string are constrained, and some characters need to be + * percent-encoded, */ #ifdef HAVE_CONFIG_H #include "clixon_config.h" /* generated by config & autoconf */ @@ -75,9 +88,158 @@ #include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_xml_nsctx.h" +#include "clixon_xml_sort.h" #include "clixon_netconf_lib.h" #include "clixon_xml_map.h" -#include "clixon_api_path.h" +#include "clixon_yang_module.h" +#include "clixon_path.h" +#include "clixon_api_path_parse.h" +#include "clixon_instance_id_parse.h" + +/*! Given api-path, parse it, and return a clixon-path struct + * + * @param[in] api_path String with api-path syntax according to RESTCONF RFC8040 + * @param[out] cplist Structured internal clixon-path + * @retval 0 OK + * @retval -1 Error + * @code + * clixon_path *cplist = NULL; + * if (api_path_parse(api_path, &cplist) < 0) + * err; + * if (api_path_resolve(cplist, yt) < 0) + * err; + * ... + * if (cplist) + * clixon_path_free(cplist); + * @endcode + * @see clixon_path_free + */ +static int +api_path_parse(char *api_path, + clixon_path **cplist) +{ + int retval = -1; + struct clicon_api_path_yacc_arg ay = {0,}; + + clicon_debug(1, "%s api_path:%s", __FUNCTION__, api_path); + ay.ay_parse_string = api_path; + ay.ay_name = "api-path parser"; + ay.ay_linenum = 1; + if (api_path_scan_init(&ay) < 0) + goto done; + if (api_path_parse_init(&ay) < 0) + goto done; + if (clixon_api_path_parseparse(&ay) != 0) { /* yacc returns 1 on error */ + clicon_log(LOG_NOTICE, "API-PATH error: on line %d", ay.ay_linenum); + if (clicon_errno == 0) + clicon_err(OE_XML, 0, "API-PATH parser error with no error code (should not happen)"); + goto done; + } + api_path_parse_exit(&ay); + api_path_scan_exit(&ay); + *cplist = ay.ay_top; + retval = 0; + done: + return retval; +} + +/*! Given instance-id path, parse it, and return an clixon-path struct + * + * @param[in] api_path String with syntax according to YANG RFC7980 + * @param[out] cplist Structured internal clixon-path + * @retval 0 OK + * @retval -1 Error + * @code + * clixon_path *cplist = NULL; + * if (instance_id_parse(api_path, &cplist) < 0) + * err; + * ... + * if (cplist) + * clixon_path_free(cplist); + * @endcode + * @see clixon_path_free + */ +static int +instance_id_parse(char *path, + clixon_path **cplist) +{ + int retval = -1; + struct clicon_instance_id_yacc_arg iy = {0,}; + + clicon_debug(1, "%s path:%s", __FUNCTION__, path); + iy.iy_parse_string = path; + iy.iy_name = "instance-id parser"; + iy.iy_linenum = 1; + if (instance_id_scan_init(&iy) < 0) + goto done; + if (instance_id_parse_init(&iy) < 0) + goto done; + if (clixon_instance_id_parseparse(&iy) != 0) { /* yacc returns 1 on error */ + clicon_log(LOG_NOTICE, "Instance-id error: on line %d", iy.iy_linenum); + if (clicon_errno == 0) + clicon_err(OE_XML, 0, "Instance-id parser error with no error code (should not happen)"); + goto done; + } + instance_id_parse_exit(&iy); + instance_id_scan_exit(&iy); + *cplist = iy.iy_top; + retval = 0; + done: + return retval; +} + +static int +clixon_path_free(clixon_path *cplist) +{ + clixon_path *cp; + + while ((cp = cplist) != NULL){ + DELQ(cp, cplist, clixon_path *); + if (cp->cp_prefix) + free(cp->cp_prefix); + if (cp->cp_id) + free(cp->cp_id); + if (cp->cp_cvk) + cvec_free(cp->cp_cvk); + free(cp); + } + return 0; +} + +/*! Print path on instance-id/xpath form + */ +static int +clixon_path_print(FILE *f, + clixon_path *cplist) +{ + clixon_path *cp; + cg_var *cv; + + if ((cp = cplist) != NULL){ + do { + fprintf(f, "/"); + if (cp->cp_prefix) + fprintf(f, "%s:", cp->cp_prefix); + fprintf(f, "%s", cp->cp_id); + if (cp->cp_cvk){ + fprintf(f, "="); + cv = NULL; + while ((cv = cvec_each(cp->cp_cvk, cv)) != NULL){ + fprintf(f, "["); + /* If cvk has one integer argument, interpret as position, eg x/y[42] */ + if (cvec_len(cp->cp_cvk) == 1 && (cv_type_get(cv) == CGV_UINT32)) + fprintf(f, "%u", cv_uint32_get(cv)); + else + fprintf(f, "%s=\"%s\"", cv_name_get(cv), cv_string_get(cv)); + fprintf(f, "]"); + } + } + cp = NEXTQ(clixon_path *, cp); + } while (cp && cp != cplist); + } + fprintf(f, "\n"); + return 0; +} /*! Given an XML node, return root node * A root node is an ancestor xr of x with one or both of the following properties @@ -936,7 +1098,7 @@ api_path2xml(char *api_path, clicon_err(OE_UNIX, errno, "cbuf_new"); goto done; } - if (*api_path!='/'){ + if (*api_path != '/'){ cprintf(cberr, "Invalid api-path: %s (must start with '/')", api_path); if (netconf_invalid_value_xml(xerr, "application", cbuf_get(cberr)) < 0) goto done; @@ -955,8 +1117,8 @@ api_path2xml(char *api_path, } nvec--; /* NULL-terminated */ if ((retval = api_path2xml_vec(vec+1, nvec, - xtop, yspec, nodeclass, strict, - xbotp, ybotp, xerr)) < 1) + xtop, yspec, nodeclass, strict, + xbotp, ybotp, xerr)) < 1) goto done; xml_yang_root(*xbotp, &xroot); if (xmlns_assign(xroot) < 0) @@ -1062,3 +1224,409 @@ xml2api_path_1(cxobj *x, done: return retval; } + +/*! Resolve api-path module:names to yang statements + * @param[in] cplist Lisp of clixon-path + * @param[in] yt Yang statement of top symbol (can be yang-spec if top-level) + * @retval -1 Error + * @retval 0 Fail + * @retval 1 OK + * Reasons for fail (retval = 0) are: XXX + * - Modulename in api-path does not correspond to existing module + * - Modulename not defined for top-level id. + * - Corresponding yang node for id not found + * - Number of keys in key-value list does not match Yang list + * - Key-values only defined for list or leaf-list + * @see instance_id_resolve + */ +static int +api_path_resolve(clixon_path *cplist, + yang_stmt *yt) +{ + int retval = -1; + clixon_path *cp; + yang_stmt *yc; + int i; + cg_var *cva; + cg_var *cvy; + + if ((cp = cplist) != NULL){ + do { + if (yang_keyword_get(yt) == Y_SPEC){ + if (cp->cp_prefix == NULL){ + clicon_err(OE_YANG, ENOENT, "Modulename not defined for top-level id."); + goto fail; + } + if ((yt = yang_find_module_by_name(yt, cp->cp_prefix)) == NULL){ + clicon_err(OE_YANG, ENOENT, "Modulename in api-path does not correspond to existing module"); + goto fail; + } + } + if ((yc = yang_find_datanode(yt, cp->cp_id)) == NULL){ + clicon_err(OE_YANG, ENOENT, "Corresponding yang node for id not found"); + goto fail; + } + cp->cp_yang = yc; + if (cp->cp_cvk){ + /* Iterate over yang list keys and assign as names (or "." for leaf-list) in cvk */ + if (yang_keyword_get(yc) == Y_LEAF_LIST){ + cva = NULL; + while ((cva = cvec_each(cp->cp_cvk, cva)) != NULL) { + if (cv_name_get(cva) == NULL) + cv_name_set(cva, "."); + } + } + else if (yang_keyword_get(yc) == Y_LIST){ + if (cvec_len(cp->cp_cvk) > cvec_len(yang_cvec_get(yc))){ + clicon_err(OE_YANG, ENOENT, "Number of keys in key-value list does not match Yang list"); + goto fail; + } + i = 0; + cva = NULL; + while ((cva = cvec_each(cp->cp_cvk, cva)) != NULL) { + if (cv_name_get(cva) == NULL){ + cvy = cvec_i(yang_cvec_get(yc), i); + cv_name_set(cva, cv_string_get(cvy)); + } + i++; + } + } + else{ + clicon_err(OE_YANG, ENOENT, "key-values only defined for list or leaf-list"); + goto fail; + } + } + yt = yc; + cp = NEXTQ(clixon_path *, cp); + } while (cp && cp != cplist); + } + retval = 1; + done: + return retval; + fail: + retval = 0; + goto done; +} + +/*! Resolve instance-id prefix:names to yang statements + * @param[in] cplist Lisp of clixon-path + * @param[in] yt Yang statement of top symbol (can be yang-spec if top-level) + * @param[in] xt XML statement for prefix context + * @retval -1 Error + * @retval 0 Fail + * @retval 1 OK + * @note: The spec says: prefixes depend on the XML context in which the value occurs. However, + * canonical prefixes/namespaces are used based on loaded yang modules. + * This means that prefix=NULL is not allowed. + * Reasons for fail (retval = 0) are: + * - No prefix of identifier (keynames may omit prefix) + * - Prefix does not correspond to existing module. + * - Corresponding yang node for id not found + * - Number of keys in key-value list does not match Yang list + * - Key-values only defined for list or leaf-list + * @see api_path_resolve + */ +static int +instance_id_resolve(clixon_path *cplist, + yang_stmt *yt) +{ + int retval = -1; + clixon_path *cp; + yang_stmt *yc; + int i; + cg_var *cva; + cg_var *cvy; + yang_stmt *yspec; + + yspec = ys_spec(yt); + if ((cp = cplist) != NULL){ + do { + if (cp->cp_prefix == NULL){ + clicon_err(OE_YANG, ENOENT, "No prefix of identifier (keynames may omit prefix)"); + goto fail; + } + if (yang_keyword_get(yt) == Y_SPEC){ + if ((yt = yang_find_module_by_prefix_yspec(yspec, cp->cp_prefix)) == NULL){ + clicon_err(OE_YANG, ENOENT, "Prefix does not correspond to existing module."); + goto fail; + } + } + if ((yc = yang_find_datanode(yt, cp->cp_id)) == NULL){ + clicon_err(OE_YANG, ENOENT, "Corresponding yang node for id not found"); + goto fail; + } + cp->cp_yang = yc; + if (cp->cp_cvk){ + /* Iterate over yang list keys and assign as names in cvk */ + if (yang_keyword_get(yc) == Y_LEAF_LIST) + ; + else if (yang_keyword_get(yc) == Y_LIST){ +#ifdef notused + if (cvec_len(cp->cp_cvk) > cvec_len(yang_cvec_get(yc))){ + clicon_err(OE_YANG, ENOENT, "Number of keys in key-value list does not match Yang list "); + goto fail; + } +#endif + i = 0; + cva = NULL; + while ((cva = cvec_each(cp->cp_cvk, cva)) != NULL) { + if (cv_name_get(cva) == NULL){ + cvy = cvec_i(yang_cvec_get(yc), i); + cv_name_set(cva, cv_string_get(cvy)); + } + i++; + } + } + else{ + clicon_err(OE_YANG, ENOENT, "key-values only defined for list or leaf-list"); + goto fail; + } + } + yt = yc; + cp = NEXTQ(clixon_path *, cp); + } while (cp && cp != cplist); + } + retval = 1; + done: + return retval; + fail: + retval = 0; + goto done; +} + +/*! Search XML tree using Clixon path struct + * + * @param[in] xt Top xml-tree where to search + * @param[in] yt Yang statement of top symbol (can be yang-spec if top-level) + * @param[in] cplist Lisp of clixon-path + * @param[out] xvec Vector of xml-trees. Vector must be free():d after use + * @param[out] xlen Returns length of vector in return value + * @retval -1 Error + * @retval 0 Fail fail: eg no yang + * @retval 1 OK with found xml nodes in xvec (if any) + */ +static int +clixon_path_search(cxobj *xt, + yang_stmt *yt, + clixon_path *cplist, + cxobj ***xvec0, + size_t *xlen0) +{ + int retval = -1; + char *modns; /* Module namespace of api-path */ + clixon_path *cp; + cxobj **xvecp = NULL; /* parent set */ + size_t xlenp = 0; + cxobj **xvecc = NULL; /* child set */ + size_t xlenc = 0; + yang_stmt *yc; + cxobj *xp; + int i; + cg_var *cv; + + modns = NULL; + if ((cp = cplist) != NULL){ + cxvec_append(xt, &xvecp, &xlenp); /* Initialize parent xml set */ + do { + yc = cp->cp_yang; + if ((modns = yang_find_mynamespace(yc)) == NULL) + goto fail; + for (i=0; icp_cvk && /* cornercase for instance-id [] special case */ + (yang_keyword_get(yc) == Y_LIST || yang_keyword_get(yc) == Y_LEAF_LIST) && + cvec_len(cp->cp_cvk) == 1 && (cv = cvec_i(cp->cp_cvk,0)) && + (cv_type_get(cv) == CGV_UINT32)){ + if (clixon_xml_find_pos(xp, yc, cv_uint32_get(cv), &xvecc, &xlenc) < 0) + goto done; + } + else if (clixon_xml_find_index(xp, yang_parent_get(yc), modns, yang_argument_get(yc), + cp->cp_cvk, &xvecc, &xlenc) < 0) + goto done; + } + if (xvecp) + free(xvecp); + xvecp = xvecc; xlenp = xlenc; + xvecc = NULL, xlenc = 0; + cp = NEXTQ(clixon_path *, cp); + } while (cp && cp != cplist); + *xvec0 = xvecp; xvecp = NULL; + *xlen0 = xlenp; + } + retval = 1; + done: + if (xvecp) + free(xvecp); + if (xvecc) + free(xvecc); + return retval; + fail: /* eg no yang for api-path, no match */ + *xvec0 = NULL; + *xlen0 = 0; + retval = 0; + goto done; +} + +/*! Given RESTCONF api-path and an XML tree, return matching xml node vector using stdarg + * + * @param[in] xt Top xml-tree where to search + * @param[in] yt Yang statement of top symbol (can be yang-spec if top-level) + * @param[out] xvec Vector of xml-trees. Vector must be free():d after use + * @param[out] xlen Returns length of vector in return value + * @param[in] format Format string for api-path syntax + * @retval -1 Error + * @retval 0 Nomatch + * @retval 1 OK with found xml nodes in xvec (if any) + * Reasons for nomatch (retval = 0) are: + * - Modulename in api-path does not correspond to existing module + * - Modulename not defined for top-level id. + * - Number of keys in key-value list does not match Yang list + * @code + * cxobj **xvec = NULL; + * size_t xlen; + * if (clixon_xml_find_api_path(x, &xvec, &xlen, "/symbol/%s", "foo") < 0) + * goto err; + * for (i=0; i", yspec, &xt); + * int fd; + * fd = open(filename, O_RDONLY); + * xml_parse_file(fd, "", yspec, &xt); * xml_free(xt); * @endcode * @see xml_parse_string @@ -2246,10 +2255,23 @@ cxvec_dup(cxobj **vec0, return retval; } -/*! Append a new xml tree to an existing xml vector +/*! Append a new xml tree to an existing xml vector last in the list * @param[in] x XML tree (append this to vector) * @param[in,out] vec XML tree vector * @param[in,out] len Length of XML tree vector + * @retval 0 OK + * @retval -1 Error + * @code + * cxobj **xvec = NULL; + * size_t xlen = 0; + * cxobj *x; + * + * if (cxvec_append(x, &xvec, &xlen) < 0) + * err; + * if (xvec) + * free(xvec); + * @endcode + * @see cxvec_prepend */ int cxvec_append(cxobj *x, @@ -2268,6 +2290,44 @@ cxvec_append(cxobj *x, return retval; } +/*! Prepend a new xml tree to an existing xml vector first in the list + * @param[in] x XML tree (append this to vector) + * @param[in,out] vec XML tree vector + * @param[in,out] len Length of XML tree vector + * @retval 0 OK + * @retval -1 Error + * @code + * cxobj **xvec = NULL; + * size_t xlen = 0; + * cxobj *x; + * + * if (cxvec_append(x, &xvec, &xlen) < 0) + * err; + * if (xvec) + * free(xvec); + * @endcode + * @see cxvec_prepend + */ +int +cxvec_prepend(cxobj *x, + cxobj ***vec, + size_t *len) +{ + int retval = -1; + + if ((*vec = realloc(*vec, sizeof(cxobj *) * (*len+1))) == NULL){ + clicon_err(OE_XML, errno, "realloc"); + goto done; + } + memmove(&(*vec)[1], &(*vec)[0], sizeof(cxobj *) * (*len)); + (*vec)[0] = x; + (*len)++; + retval = 0; + done: + return retval; +} + + /*! Apply a function call recursively on all xml node children recursively * Recursively traverse all xml nodes in a parse-tree and apply fn(arg) for * each object found. The function is called with the xml node and an @@ -2548,8 +2608,6 @@ xml_operation(char *opstr, return 0; } - - /*! Map xml operation from enumeration to string * @param[in] op enumeration operation, eg OP_MERGE,... * @retval str String, eg "merge". Static string, no free necessary diff --git a/lib/src/clixon_xml_changelog.c b/lib/src/clixon_xml_changelog.c index 89085cd1..0f2a103a 100644 --- a/lib/src/clixon_xml_changelog.c +++ b/lib/src/clixon_xml_changelog.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -66,6 +67,7 @@ #include "clixon_options.h" #include "clixon_data.h" #include "clixon_yang_module.h" +#include "clixon_yang_parse_lib.h" #include "clixon_netconf_lib.h" #include "clixon_xml_nsctx.h" #include "clixon_xml_map.h" diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 0be80b45..504cf685 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -72,7 +73,6 @@ #include "clixon_err.h" #include "clixon_netconf_lib.h" #include "clixon_xml_sort.h" -#include "clixon_yang_internal.h" /* internal */ #include "clixon_yang_type.h" #include "clixon_xml_map.h" @@ -296,7 +296,7 @@ xml2cvec(cxobj *xt, name = xml_name(xc); if ((ys = yang_find_datanode(yt, name)) == NULL){ clicon_debug(0, "%s: yang sanity problem: %s in xml but not present in yang under %s", - __FUNCTION__, name, yt->ys_argument); + __FUNCTION__, name, yang_argument_get(yt)); if ((body = xml_body(xc)) != NULL){ if ((cv = cv_new(CGV_STRING)) == NULL){ clicon_err(OE_PLUGIN, errno, "cv_new"); @@ -476,7 +476,7 @@ xml_diff1(yang_stmt *ys, continue; } /* Both x0c and x1c exists, check if they are equal. */ - eq = xml_cmp(x0c, x1c, 0); + eq = xml_cmp(x0c, x1c, 0, 0); if (eq < 0){ if (cxvec_append(x0c, x0vec, x0veclen) < 0) goto done; @@ -502,7 +502,7 @@ xml_diff1(yang_stmt *ys, if (cxvec_append(x1c, changed_x1, changedlen) < 0) goto done; } - else if (yc->ys_keyword == Y_LEAF){ + else if (yang_keyword_get(yc) == Y_LEAF){ /* if x0c and x1c are leafs w bodies, then they are changed */ if ((b1 = xml_body(x0c)) == NULL) /* empty type */ break; @@ -761,7 +761,7 @@ xml_default(cxobj *xt, int retval = -1; yang_stmt *ys; yang_stmt *y; - int i; + // int i; // XXX cxobj *xc; cxobj *xb; char *str; @@ -775,15 +775,15 @@ xml_default(cxobj *xt, goto done; } /* Check leaf defaults */ - if (ys->ys_keyword == Y_CONTAINER || ys->ys_keyword == Y_LIST || - ys->ys_keyword == Y_INPUT){ - for (i=0; iys_len; i++){ - y = ys->ys_stmt[i]; - if (y->ys_keyword != Y_LEAF) + if (yang_keyword_get(ys) == Y_CONTAINER || yang_keyword_get(ys) == Y_LIST || + yang_keyword_get(ys) == Y_INPUT){ + y = NULL; + while ((y = yn_each(ys, y)) != NULL) { + if (yang_keyword_get(y) != Y_LEAF) continue; if (!cv_flag(yang_cv_get(y), V_UNSET)){ /* Default value exists */ - if (!xml_find(xt, y->ys_argument)){ - if ((xc = xml_new(y->ys_argument, NULL, y)) == NULL) + if (!xml_find(xt, yang_argument_get(y))){ + if ((xc = xml_new(yang_argument_get(y), NULL, y)) == NULL) goto done; /* assign right prefix */ if ((namespace = yang_find_mynamespace(y)) != NULL){ @@ -843,9 +843,9 @@ xml_sanity(cxobj *xt, goto done; } name = xml_name(xt); - if (strstr(ys->ys_argument, name)==NULL){ + if (strstr(yang_argument_get(ys), name)==NULL){ clicon_err(OE_XML, 0, "xml node name '%s' does not match yang spec arg '%s'", - name, ys->ys_argument); + name, yang_argument_get(ys)); goto done; } retval = 0; @@ -1126,7 +1126,7 @@ xmlns_assign(cxobj *x) } /* 1. Check which namespace x should have (via yang). This is correct namespace. */ if ((ns_correct = yang_find_mynamespace(y)) == NULL){ - clicon_err(OE_YANG, ENOENT, "yang node %s does not have namespace", y->ys_argument); + clicon_err(OE_YANG, ENOENT, "yang node %s does not have namespace", yang_argument_get(y)); goto done; } /* 2. Check which namespace x has via its XML tree */ @@ -1293,7 +1293,7 @@ xml_merge1(cxobj *x0, /* the target */ assert(y0); x1name = xml_name(x1); - if (y0->ys_keyword == Y_LEAF_LIST || y0->ys_keyword == Y_LEAF){ + if (yang_keyword_get(y0) == Y_LEAF_LIST || yang_keyword_get(y0) == Y_LEAF){ x1bstr = xml_body(x1); if (x0==NULL){ if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) @@ -1479,7 +1479,7 @@ yang_enum_int_value(cxobj *node, if (yang_type_resolve(ys, ys, ytype, &yrestype, NULL, NULL, NULL, NULL, NULL) < 0) goto done; - if (yrestype==NULL || strcmp(yrestype->ys_argument, "enumeration")) + if (yrestype==NULL || strcmp(yang_argument_get(yrestype), "enumeration")) goto done; if ((yenum = yang_find(yrestype, Y_ENUM, xml_body(node))) == NULL) goto done; @@ -1487,7 +1487,7 @@ yang_enum_int_value(cxobj *node, if ((yval = yang_find(yenum, Y_VALUE, NULL)) == NULL) goto done; /* reason is string containing why int could not be parsed */ - if (parse_int32(yval->ys_argument, val, &reason) < 0) + if (parse_int32(yang_argument_get(yval), val, &reason) < 0) goto done; retval = 0; done: diff --git a/lib/src/clixon_xml_parse.h b/lib/src/clixon_xml_parse.h index d93754d3..6be1607f 100644 --- a/lib/src/clixon_xml_parse.h +++ b/lib/src/clixon_xml_parse.h @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -51,7 +52,7 @@ struct xml_parse_yacc_arg{ cxobj *ya_xparent; /* xml parent element*/ int ya_skipspace; /* If set, remove all non-terminal bodies (strip pretty-print) */ yang_stmt *ya_yspec; /* If set, top-level yang-spec */ - int ya_lex_state; /* lex start condition (AMPERSAND) */ + int ya_lex_state; /* lex return state */ }; extern char *clixon_xml_parsetext; diff --git a/lib/src/clixon_xml_sort.c b/lib/src/clixon_xml_sort.c index 291d3118..2230ddcb 100644 --- a/lib/src/clixon_xml_sort.c +++ b/lib/src/clixon_xml_sort.c @@ -2,7 +2,8 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2020 Olof Hagsand This file is part of CLIXON. @@ -31,7 +32,7 @@ ***** END LICENSE BLOCK ***** - * XML search functions when used with YANG + * XML sort and search functions when used with YANG */ #ifdef HAVE_CONFIG_H @@ -65,6 +66,7 @@ #include "clixon_options.h" #include "clixon_xml_map.h" #include "clixon_yang_type.h" +#include "clixon_yang_module.h" #include "clixon_xml_sort.h" /*! Get xml body value as cligen variable @@ -193,10 +195,10 @@ xml_child_spec(cxobj *x, * @param[in] x2 object 2 * @param[in] same If set, x1 and x2 are member of same parent & enumeration * is used (see explanation below) + * @param[in] skip1 Key matching skipped for keys not in x1 (see explanation) * @retval 0 If equal * @retval <0 If x1 is less than x2 * @retval >0 If x1 is greater than x2 - * @see xml_cmp1 Similar, but for one object * * There are distinct calls for this function: * 1. For sorting in an existing list of XML children @@ -205,11 +207,19 @@ xml_child_spec(cxobj *x, * if they have the same yang-spec, the existing order is used as tie-breaker. * In other words, if order-by-system, or if the case (2) above, the existing * order is ignored and the actual xml element contents is examined. + * + * Also, if pattern is set, list matching is not made so that x1 and x2 must have same keys. + * instead, x1 and x2 are considered equal if the keys that x1 have match. Keys that x2 but not + * x1 has are ignored. + * Example: x1: 1 and x2: 12 are considered equal. + * This is useful in searching for indexes in trees, where x1 is a search index and not a + * complete tree. + * * @note empty value/NULL is smallest value * @note some error cases return as -1 (qsort cant handle errors) * @note some error cases return as -1 (qsort cant handle errors) * - * NOTE: "comparing" x1 and x2 here judges equality from a structural (model) + * @note "comparing" x1 and x2 here judges equality from a structural (model) * perspective, ie both have the same yang spec, if they are lists, they have the * the same keys. NOT that the values are equal! * In other words, if x is a leaf with the same yang spec, 1 and 2 are @@ -221,7 +231,8 @@ xml_child_spec(cxobj *x, int xml_cmp(cxobj *x1, cxobj *x2, - int same) + int same, + int skip1) { yang_stmt *y1; yang_stmt *y2; @@ -281,9 +292,13 @@ xml_cmp(cxobj *x1, } switch (yang_keyword_get(y1)){ case Y_LEAF_LIST: /* Match with name and value */ - if ((b1 = xml_body(x1)) == NULL) + b1 = xml_body(x1); + b2 = xml_body(x2); + if (b1 == NULL && b2 == NULL) + ; + else if (b1 == NULL) equal = -1; - else if ((b2 = xml_body(x2)) == NULL) + else if (b2 == NULL) equal = 1; else{ if (xml_cv_cache(x1, &cv1) < 0) /* error case */ @@ -301,33 +316,47 @@ xml_cmp(cxobj *x1, } break; case Y_LIST: /* Match with key values - * Use Y_LIST cache (see struct yang_stmt) - */ + * Use Y_LIST cache (see struct yang_stmt) */ cvk = yang_cvec_get(y1); /* Use Y_LIST cache, see ys_populate_list() */ cvi = NULL; while ((cvi = cvec_each(cvk, cvi)) != NULL) { keyname = cv_string_get(cvi); /* operational data may have NULL keys*/ - if ((x1b = xml_find(x1, keyname)) == NULL || - xml_body(x1b) == NULL) + x1b = xml_find(x1, keyname); + /* match1: key matching skipped for keys not in x1 (see explanation) */ + if (skip1 && x1b == NULL) + continue; + x2b = xml_find(x2, keyname); + if (x1b == NULL && x2b == NULL) + ; + else if (x1b == NULL) equal = -1; - else if ((x2b = xml_find(x2, keyname)) == NULL || - xml_body(x2b) == NULL) + else if (x2b == NULL) equal = 1; else{ - if (xml_cv_cache(x1b, &cv1) < 0) /* error case */ - goto done; - if (xml_cv_cache(x2b, &cv2) < 0) /* error case */ - goto done; - assert(cv1 && cv2); - if ((equal = cv_cmp(cv1, cv2)) != 0) - goto done; + b1 = xml_body(x1b); + b2 = xml_body(x2b); + if (b1 == NULL && b2 == NULL) + ; + else if (b1 == NULL) + equal = -1; + else if (b2 == NULL) + equal = 1; + else{ + if (xml_cv_cache(x1b, &cv1) < 0) /* error case */ + goto done; + if (xml_cv_cache(x2b, &cv2) < 0) /* error case */ + goto done; + assert(cv1 && cv2); + equal = cv_cmp(cv1, cv2); + } } - } - equal = 0; + if (equal) + break; + } /* while cvi */ break; default: break; - } + } /* switch */ done: clicon_debug(2, "%s %s %s %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, nr1, nr2, yi1, yi2); return equal; @@ -340,7 +369,7 @@ static int xml_cmp_qsort(const void* arg1, const void* arg2) { - return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1); + return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1, 0); } /*! Sort children of an XML node @@ -366,21 +395,28 @@ xml_sort(cxobj *x, return 0; } -/*! Special case search for ordered-by user where linear sort is used +/*! Special case search for ordered-by user or state data where linear sort is used + * * @param[in] xp Parent XML node (go through its childre) * @param[in] x1 XML node to match * @param[in] yangi Yang order number (according to spec) * @param[in] mid Where to start from (may be in middle of interval) - * @param[out] xretp XML return object, or NULL - * @retval 0 OK, see xretp + * @param[out] xvec Vector of matching XML return objects (can be empty) + * @param[out] xlen Length of xvec + * @retval 0 OK, see xvec (may be empty) + * @retval -1 Error + * XXX: only first match */ static int -xml_search_userorder(cxobj *xp, - cxobj *x1, - int yangi, - int mid, - cxobj **xretp) +xml_find_keys_notsorted(cxobj *xp, + cxobj *x1, + int yangi, + int mid, + int skip1, + cxobj ***xvec, + size_t *xlen) { + int retval = -1; int i; cxobj *xc; yang_stmt *yc; @@ -388,113 +424,186 @@ xml_search_userorder(cxobj *xp, for (i=mid+1; i=0; i--){ /* Then decrement */ xc = xml_child_i(xp, i); yc = xml_spec(xc); - if (yangi!=yang_order(yc)) + if (yangi != yang_order(yc)) /* wrong yang */ break; - if (xml_cmp(xc, x1, 0) == 0){ - *xretp = xc; - goto done; + if (xml_cmp(xc, x1, 0, skip1) == 0){ + if (cxvec_append(xc, xvec, xlen) < 0) + goto done; + goto ok; /* found */ } } - *xretp = NULL; + ok: + retval = 0; done: - return 0; + return retval; +} + +/*! Find more equal objects in a vector up and down in the array of the present + * @param[in] xp Parent XML node (go through its childre) + * @param[in] x1 XML node to match + * @param[in] yangi Yang order number (according to spec) + * @param[in] mid Where to start from (may be in middle of interval) + * @param[out] xvec Vector of matching XML return objects (can be empty) + * @param[out] xlen Length of xvec + * @retval 0 OK, see xvec (may be empty) + * @retval -1 Error + */ +static int +more_equals(cxobj *xp, + cxobj *x1, + int yangi, + int mid, + int skip1, + cxobj ***xvec, + size_t *xlen) +{ + int retval = -1; + int i; + cxobj *xc; + yang_stmt *yc; + + for (i=mid-1; i>=0; i--){ /* First decrement */ + xc = xml_child_i(xp, i); + yc = xml_spec(xc); + if (yangi != yang_order(yc)) /* wrong yang */ + break; + if (xml_cmp(x1, xc, 0, skip1) != 0) + break; + if (cxvec_prepend(xc, xvec, xlen) < 0) + goto done; + } + for (i=mid+1; i= xml_child_nr(xp)) /* beyond range */ - goto notfound; + goto ok; xc = xml_child_i(xp, mid); if ((y = xml_spec(xc)) == NULL) - goto notfound; + goto ok; cmp = yangi-yang_order(y); /* Here is right yang order == same yang? */ if (cmp == 0){ - cmp = xml_cmp(x1, xc, 0); - if (cmp && userorder){ /* Ordered by user (if not equal) */ - xml_search_userorder(xp, x1, yangi, mid, xretp); + cmp = xml_cmp(x1, xc, 0, skip1); + if (cmp && !sorted){ /* Ordered by user (if not equal) */ + retval = xml_find_keys_notsorted(xp, x1, yangi, mid, skip1, xvec, xlen); goto done; } } - if (cmp == 0) - *xretp = xc; + if (cmp == 0){ + if (cxvec_append(xc, xvec, xlen) < 0) + goto done; + /* there may be more? */ + if (more_equals(xp, x1, yangi, mid, skip1, xvec, xlen) < 0) + goto done; + } else if (cmp < 0) - xml_search1(xp, x1, userorder, yangi, low, mid-1, xretp); + xml_find_keys1(xp, x1, sorted, yangi, low, mid-1, skip1, xvec, xlen); else - xml_search1(xp, x1, userorder, yangi, mid+1, upper, xretp); + xml_find_keys1(xp, x1, sorted, yangi, mid+1, upper, skip1, xvec, xlen); + ok: + retval = 0; done: - return 0; - notfound: - *xretp = NULL; - goto done; + return retval; } -/*! Find XML child under xp matching x1 using binary search +/*! Find XML child under xp matching x1 using binary search for list/leaf-list keys + * + * Match is tried xp with x1 with either name only (container/leaf) or using keys (list/leaf-lists) + * Any non-key leafs or other structure of x1 is not matched. + * * @param[in] xp Parent xml node. * @param[in] x1 Find this object among xp:s children * @param[in] yc Yang spec of x1 - * @param[out] xretp XML return object, or NULL - * @retval 0 OK, see xretp + * @param[in] skip1 Key matching skipped for keys not in x1 + * @param[out] xvec Vector of matching XML return objects (can be empty) + * @param[out] xlen Length of xvec + * @retval 0 OK, see xvec (may be empty) + * @retval -1 Error + * @see xml_find_index for a generic search function */ static int -xml_search(cxobj *xp, - cxobj *x1, - yang_stmt *yc, - cxobj **xretp) +xml_find_keys(cxobj *xp, + cxobj *x1, + yang_stmt *yc, + int skip1, + cxobj ***xvec, + size_t *xlen) { - cxobj *xa; - int low = 0; - int upper = xml_child_nr(xp); - int userorder=0; - int yangi; + cxobj *xa; + int low = 0; + int upper = xml_child_nr(xp); + int sorted = 1; + int yangi; /* Assume if there are any attributes, they are first in the list, mask them by raising low to skip them */ for (low=0; low 0) + if (xml_cmp(xprev, x, 1, 0) > 0) goto done; } xprev = x; @@ -779,13 +888,14 @@ xml_sort_verify(cxobj *x0, return retval; } -/*! Given child tree x1c, find matching child in base tree x0 and return as x0cp +/*! Given child tree x1c, find (first) matching child in base tree x0 and return as x0cp * @param[in] x0 Base tree node * @param[in] x1c Modification tree child * @param[in] yc Yang spec of tree child * @param[out] x0cp Matching base tree child (if any) * @retval 0 OK * @retval -1 Error + * XXX: only handles first match */ int match_base_child(cxobj *x0, @@ -802,6 +912,8 @@ match_base_child(cxobj *x0, yang_stmt *y0c; yang_stmt *y0p; yang_stmt *yp; /* yang parent */ + cxobj **xvec = NULL; + size_t xlen = 0; *x0cp = NULL; /* init return value */ /* Special case is if yc parent (yp) is choice/case @@ -816,6 +928,7 @@ match_base_child(cxobj *x0, y0p == yp) break; /* x0c will have a value */ } + *x0cp = x0c; goto ok; /* What to do if not found? */ } switch (yang_keyword_get(yc)){ @@ -823,58 +936,146 @@ match_base_child(cxobj *x0, case Y_LEAF: /* Equal regardless */ break; case Y_LEAF_LIST: /* Match with name and value */ - if (xml_body(x1c) == NULL){ /* Treat as empty string */ - // assert(0); + if (xml_body(x1c) == NULL) /* Treat as empty string */ goto ok; - } break; case Y_LIST: /* Match with key values */ cvk = yang_cvec_get(yc); /* Use Y_LIST cache, see ys_populate_list() */ - /* Count number of key indexes - * Then create two vectors one with names and one with values of x1c, - * ec: keyvec: [a,b,c] keyval: [1,2,3] - */ cvi = NULL; while ((cvi = cvec_each(cvk, cvi)) != NULL) { keyname = cv_string_get(cvi); - // keyvec[i] = keyname; - if ((xb = xml_find(x1c, keyname)) == NULL){ + if ((xb = xml_find(x1c, keyname)) == NULL) goto ok; - } } default: break; } /* Get match. */ - xml_search(x0, x1c, yc, &x0c); + if (xml_find_keys(x0, x1c, yc, 0, &xvec, &xlen) < 0) + goto done; + if (xlen) + *x0cp = xvec[0]; ok: - *x0cp = x0c; retval = 0; + done: + if (xvec) + free(xvec); return retval; } - -/*! Experimental API for binary search - * - * Create a temporary search object: a list (xc) with a key (xk) and call the binary search. - * @param[in] xp Parent xml node. - * @param[in] yc Yang spec of list child - * @param[in] cvk List of keys and values as CLIgen vector - * @param[out] xret Found XML object, NULL if not founs - * @retval 0 OK, see xret - * @code - * cvec *cvk = NULL; vector of index keys - * ... Populate cvk with key/values eg a:5 b:6 - * if (xml_binsearch(xp, yc, cvk, xp) < 0) - * err; - * @endcode - * @retval -1 Error - * Can extend to leaf-list? + +/*! API for search in XML child list with non-indexed variables */ -int -xml_binsearch(cxobj *xp, - yang_stmt *yc, - cvec *cvk, - cxobj **xretp) +static int +xml_find_noyang_cvk(char *ns0, + cxobj *xc, + cvec *cvk, + cxobj ***xvec, + size_t *xlen) +{ + int retval = -1; + cg_var *cvi; + char *ns; + cxobj *xcc; + char *keyname; + char *keyval; + char *body; + + cvi = NULL; + /* Loop through index variables. xc should match all, on exit if cvi=NULL it macthes */ + while ((cvi = cvec_each(cvk, cvi)) != NULL) { + keyname = cv_name_get(cvi); + keyval = cv_string_get(cvi); + if (keyname==NULL || strcmp(keyname, ".")==0){ /* Index variable on form .= (self) */ + body = xml_body(xc); + if ((body==NULL && (keyval==NULL || strlen(keyval) == 0)) || /* both null, break */ + (body != NULL && strcmp(body, keyval) == 0)) /* Name match, break */ + continue; /* match */ + break; /* nomatch */ + } + else{ + /* Index variable on form = + * Loop through children of the matched x (to match keyname and value) */ + xcc = NULL; + while ((xcc = xml_child_each(xc, xcc, CX_ELMNT)) != NULL) { + if (xml2ns(xcc, xml_prefix(xcc), &ns) < 0) + goto done; + if (strcmp(ns0, ns) != 0) /* Namespace does not match, skip */ + continue; + if (strcmp(keyname, xml_name(xcc)) != 0) /* Name does not match, skip */ + continue; + body = xml_body(xcc); + if (body==NULL && (keyval==NULL || strlen(keyval) == 0)) /* both null, break */ + break; + if (body != NULL && strcmp(body, keyval) == 0) /* Name match, break */ + break; + } + if (xcc == NULL) + break; /* No match found */ + } + } + if (cvi == NULL) /* means we iterated through all indexes without breaks */ + if (cxvec_append(xc, xvec, xlen) < 0) + goto done; + retval = 0; + done: + return retval; +} + +/*! API for search in XML child list with no yang available + * Fallback if no yang available. Only linear search for matching name, and eventual index match + */ +static int +xml_find_noyang_name(cxobj *xp, + char *ns0, + char *name, + cvec *cvk, + cxobj ***xvec, + size_t *xlen) +{ + int retval = -1; + cxobj *xc; + char *ns; + + if (name == NULL || ns0 == NULL){ + clicon_err(OE_XML, EINVAL, "name and namespace required"); + goto done; + } + /* Go through children linearly */ + xc = NULL; + while ((xc = xml_child_each(xp, xc, CX_ELMNT)) != NULL) { + ns = NULL; + if (xml2ns(xc, xml_prefix(xc), &ns) < 0) + goto done; + if (ns == NULL) + continue; + if (strcmp(ns0, ns) != 0) /* Namespace does not match, skip */ + continue; + if (strcmp(name, xml_name(xc)) != 0) /* Namespace does not match, skip */ + continue; + if (cvk){ /* Check indexes */ + if (xml_find_noyang_cvk(ns0, xc, cvk, xvec, xlen) < 0) + goto done; + } + else /* No index variables: just add node to found list */ + if (cxvec_append(xc, xvec, xlen) < 0) + goto done; + } + retval = 0; + done: + return retval; +} + +/*! API for search in XML child list with yang available + * @retval 1 OK + * @retval 0 Revert, try again with no-yang search + * @retval -1 Error + */ +static int +xml_find_index_yang(cxobj *xp, + yang_stmt *yc, + cvec *cvk, + cxobj ***xvec, + size_t *xlen) { int retval = -1; cxobj *xc = NULL; @@ -882,30 +1083,58 @@ xml_binsearch(cxobj *xp, cg_var *cvi = NULL; cbuf *cb = NULL; yang_stmt *yk; + char *kname; + cvec *ycvk; + cg_var *ycv = NULL; + int i; char *name; - - if (yc == NULL || xml_spec(xp) == NULL){ - clicon_err(OE_YANG, ENOENT, "yang spec not found"); - goto done; - } + name = yang_argument_get(yc); if ((cb = cbuf_new()) == NULL){ clicon_err(OE_UNIX, errno, "cbuf_new"); goto done; } - cprintf(cb, "<%s>", name); - cvi = NULL; - while ((cvi = cvec_each(cvk, cvi)) != NULL) { - cprintf(cb, "<%s>%s", - cv_name_get(cvi), - cv_string_get(cvi), - cv_name_get(cvi)); + switch (yang_keyword_get(yc)){ + case Y_LIST: + cprintf(cb, "<%s>", name); + ycvk = yang_cvec_get(yc); /* Check that those are proper index keys */ + cvi = NULL; + i = 0; + while ((cvi = cvec_each(cvk, cvi)) != NULL) { + if ((kname = cv_name_get(cvi)) == NULL){ + clicon_err(OE_YANG, ENOENT, "missing yang key name in cvk"); + goto done; + } + /* Parameter in cvk is not key or not in right key order, then we cannot call + * xml_find_keys and we need to revert to noyang + */ + if ((ycv = cvec_i(ycvk, i)) == NULL || strcmp(kname, cv_string_get(ycv))){ + goto revert; + break; + } + cprintf(cb, "<%s>%s", kname, cv_string_get(cvi), kname); + i++; + } + cprintf(cb, "", name); + break; + case Y_LEAF_LIST: + if (cvec_len(cvk) != 1){ + clicon_err(OE_YANG, ENOENT, "expected exactly one leaf-list key"); + goto done; + } + cvi = cvec_i(cvk, 0); + cprintf(cb, "<%s>%s", name, cv_string_get(cvi), name); + break; + default: + cprintf(cb, "<%s/>", name); + break; } - cprintf(cb, "", name); + clicon_debug(1, "%s XML:%s", __FUNCTION__, cbuf_get(cb)); if (xml_parse_string(cbuf_get(cb), yc, &xc) < 0) goto done; if (xml_rootchild(xc, 0, &xc) < 0) goto done; + /* Populate created XML tree with yang specs */ if (xml_spec_set(xc, yc) < 0) goto done; xk = NULL; @@ -917,11 +1146,150 @@ xml_binsearch(cxobj *xp, if (xml_spec_set(xk, yk) < 0) goto done; } - retval = xml_search(xp, xc, yc, xretp); + if (xml_find_keys(xp, xc, yc, 1, xvec, xlen) < 0) + goto done; + retval = 1; /* OK */ done: if (cb) cbuf_free(cb); if (xc) xml_free(xc); return retval; + revert: /* means try name-only*/ + retval = 0; + goto done; } + +/*! API for search in XML child list using indexes and binary search if applicable + * + * Generic search function as alternative to xml_find() and others that finds YANG associated + * children of an XML tree node using indexes and optimized lookup. Optimized lookup is used only + * if an associated YANG exists. The optimized lookup is based on binary trees but is used + * The function takes as input: + * 1) An XML tree (parent) node (cxobj). See note(1) below + * 2) A set of parameters to define wanted children by name or yang-spec using yc, name, ns, yp: + * a. yp and ns:name given: YANG spec of parent + ns:name derives yc. Not found: only ns:name + * b. name given: yp is derived from xp spec, then goto 2b above. + * 3) A vector of index expressions on the form = refining the child set: + * a. cvk is NULL, return all children matching according to (B) + * b. cvk contains = entries, return children with leafs matching . + * See "Optimized" section below. + * 4) A return vector of XML nodes matching name and index variables + * + * # Optimized + * The function makes the index matching = above optimized using binary search if: + * - yc is defined AND one of the following + * - if xp is leaf-list and "id" is "." + * - if xp is a yang list and "id" is a registered index key + * - if xp is a yang list and first "id" is first leaf key, second "id" is second leaf key, etc. + * - Otherwise search is made using linear search + * + * @param[in] xp Parent xml node. + * @param[in] yc Yang spec of list child (preferred) See rule (2) above + * @param[in] yp Yang spec of parent node or yang-spec/yang (Alternative if yc not given). + * @param[in] ns Namespace (needed only if name is derived from top-symbol) + * @param[in] name Name of child (not required if yc given) + * @param[in] cvk List of keys and values as CLIgen vector on the form k1=foo, k2=bar + * @param[out] xvec Array of found nodes + * @param[out] xlen Len of xvec + * @retval 0 OK, see xret + * @retval -1 Error + * @code + * cxobj **xvec = NULL; + * size_t xlen = 0; + * cvec *cvk = NULL; vector of index keys + * ... Populate cvk with key/values eg a:5 b:6 + * if (clixon_xml_find_index(xp, yp, "a", ns, cvk, &xvec, &xlen) < 0) + * err; + * @endcode + * Discussion: + * (1) Rule 2 on how to get the child name election seems unecessary complex. First, it would be + * nice to just state name and parent 2c. But xp as top-objects may sometimes be dummies, for + * parsing, copy-buffers, or simply for XML nodes that do not have YANG. + * (2) Short form could be: "first" only returning first object to get rid of vectors. + * (3) Another short form could be: keyname,keyval instead of cvk for single keys. + */ +int +clixon_xml_find_index(cxobj *xp, + yang_stmt *yp, + char *namespace, + char *name, + cvec *cvk, + cxobj ***xvec, + size_t *xlen) +{ + int retval = -1; + int ret; + yang_stmt *yc = NULL; + + if (name == NULL){ + clicon_err(OE_YANG, ENOENT, "name"); + goto done; + } + if (yp == NULL) + yp = xml_spec(xp); + if (yp){/* 2. YANG spec of parent + name derives yc. If not found use just name. */ + if (yang_keyword_get(yp) == Y_SPEC) + yp = yang_find_module_by_namespace(yp, namespace); + if (yp) + yc = yang_find_datanode(yp, name); + } + if (yc){ + if ((ret = xml_find_index_yang(xp, yc, cvk, xvec, xlen)) < 0) + goto done; + if (ret == 0) + if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) + goto done; + } + else + if (xml_find_noyang_name(xp, namespace, name, cvk, xvec, xlen) < 0) + goto done; + retval = 0; + done: + return retval; +} + +/*! Find positional parameter in xml child list, eg x/y[42]. Note, not optimized + * + * Create a temporary search object: a list (xc) with a key (xk) and call the binary search. + * @param[in] xp Parent xml node. + * @param[in] yc Yang spec of list child + * @param[in] pos Position + * @param[out] xvec Array of found nodes + * @param[out] xlen Len of xvec + * @retval 0 OK, see xret + * @retval -1 Error + */ +int +clixon_xml_find_pos(cxobj *xp, + yang_stmt *yc, + uint32_t pos, + cxobj ***xvec, + size_t *xlen) +{ + int retval = -1; + cxobj *xc = NULL; + char *name; + uint32_t u; + + if (yc == NULL){ + clicon_err(OE_YANG, ENOENT, "yang spec not found"); + goto done; + } + name = yang_argument_get(yc); + u = 0; + xc = NULL; + while ((xc = xml_child_each(xp, xc, CX_ELMNT)) != NULL) { + if (strcmp(name, xml_name(xc))) + continue; + if (pos == u++){ /* Found */ + if (cxvec_append(xc, xvec, xlen) < 0) + goto done; + break; + } + } + retval = 0; + done: + return retval; +} + diff --git a/lib/src/clixon_xpath.c b/lib/src/clixon_xpath.c index 9f658c2f..69a35cca 100644 --- a/lib/src/clixon_xpath.c +++ b/lib/src/clixon_xpath.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -83,6 +83,7 @@ #include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_xml_nsctx.h" +#include "clixon_yang_module.h" #include "clixon_xpath_ctx.h" #include "clixon_xpath.h" #include "clixon_xpath_parse.h" @@ -726,12 +727,12 @@ xpath_vec(cxobj *xcur, va_start(ap, veclen); len = vsnprintf(NULL, 0, xpformat, ap); va_end(ap); - /* allocate a message string exactly fitting the message length */ + /* allocate an xpath string exactly fitting the length */ if ((xpath = malloc(len+1)) == NULL){ clicon_err(OE_UNIX, errno, "malloc"); goto done; } - /* second round: compute write message from reason and args */ + /* second round: actually compute xpath string content */ va_start(ap, veclen); if (vsnprintf(xpath, len+1, xpformat, ap) < 0){ clicon_err(OE_UNIX, errno, "vsnprintf"); diff --git a/lib/src/clixon_xpath_ctx.c b/lib/src/clixon_xpath_ctx.c index 6a2fb5ae..37e9e55a 100644 --- a/lib/src/clixon_xpath_ctx.c +++ b/lib/src/clixon_xpath_ctx.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -117,8 +117,6 @@ ctx_dup(xp_ctx *xc0) * @param[in] xc XPATH evaluation context * @param[in] ind Indentation margin * @param[in] str Prefix string in printout - - */ int ctx_print_cb(cbuf *cb, diff --git a/lib/src/clixon_xpath_eval.c b/lib/src/clixon_xpath_eval.c index 60ee0172..1c5aca05 100644 --- a/lib/src/clixon_xpath_eval.c +++ b/lib/src/clixon_xpath_eval.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -339,25 +339,17 @@ xp_eval_step(xp_ctx *xc0, else for (i=0; ixc_size; i++){ xv = xc->xc_nodeset[i]; x = NULL; - if ((ret = xpath_optimize_check(xs, xv, &x)) < 0) + if ((ret = xpath_optimize_check(xs, xv, &vec, &veclen)) < 0) goto done; - switch (ret){ - case 1: /* optimized */ - if (x) /* keep only x */ - if (cxvec_append(x, &vec, &veclen) < 0) - goto done; - break; - case 0: /* regular code */ - if (ret == 0){ /* No optimization made */ - while ((x = xml_child_each(xv, x, CX_ELMNT)) != NULL) { - /* xs->xs_c0 is nodetest */ - if (nodetest == NULL || nodetest_eval(x, nodetest, nsc, localonly) == 1){ - if (cxvec_append(x, &vec, &veclen) < 0) - goto done; - } + if (ret == 0){/* regular code, no optimization made */ + while ((x = xml_child_each(xv, x, CX_ELMNT)) != NULL) { + /* xs->xs_c0 is nodetest */ + if (nodetest == NULL || nodetest_eval(x, nodetest, nsc, localonly) == 1){ + if (cxvec_append(x, &vec, &veclen) < 0) + goto done; } } - } /* switch */ + } } } ctx_nodeset_replace(xc, vec, veclen); @@ -932,7 +924,7 @@ xp_eval(xp_ctx *xc, xp_ctx *xr2 = NULL; int use_xr0 = 0; /* In 2nd child use transitively result of 1st child */ - if (debug) + if (debug > 1) ctx_print(stderr, xc, xpath_tree_int2str(xs->xs_type)); /* Pre-actions before check first child c0 */ @@ -1088,7 +1080,7 @@ xp_eval(xp_ctx *xc, xr0 = NULL; } ok: - if (debug) + if (debug>1) ctx_print(stderr, *xrp, xpath_tree_int2str(xs->xs_type)); retval = 0; done: diff --git a/lib/src/clixon_xpath_optimize.c b/lib/src/clixon_xpath_optimize.c index e0ede3ca..426da62a 100644 --- a/lib/src/clixon_xpath_optimize.c +++ b/lib/src/clixon_xpath_optimize.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -68,7 +68,6 @@ #include "clixon_xpath.h" #include "clixon_xpath_optimize.h" - #ifdef XPATH_LIST_OPTIMIZE static xpath_tree *_xmtop = NULL; /* pattern match tree top */ static xpath_tree *_xm = NULL; @@ -77,7 +76,6 @@ static int _optimize_enable = 1; static int _optimize_hits = 0; #endif /* XPATH_LIST_OPTIMIZE */ - /* XXX development in clixon_xpath_eval */ int xpath_list_optimize_stats(int *hits) @@ -89,6 +87,9 @@ xpath_list_optimize_stats(int *hits) return 0; } +/*! Enable xpath optimize + * Cant replace this with optin since there is no handle in xpath functions,... + */ int xpath_list_optimize_set(int enable) { @@ -213,7 +214,8 @@ loop_preds(xpath_tree *xt, * * @param[in] xt XPath tree * @param[in] xv XML base node - * @param[out] xp XML found node (retval == 1) + * @param[out] xvec Array of found nodes + * @param[out] xlen Len of xvec * @param[out] key * @param[out] keyval * @retval -1 Error @@ -225,7 +227,8 @@ loop_preds(xpath_tree *xt, static int xpath_list_optimize_fn(xpath_tree *xt, cxobj *xv, - cxobj **xp) + cxobj ***xvec, + size_t *xlen) { int retval = -1; xpath_tree *xm = NULL; @@ -262,9 +265,9 @@ xpath_list_optimize_fn(xpath_tree *xt, if (veclen != 2) goto ok; name = vec[0]->xs_s1; - /* Extract variables XXX strdups */ + /* Extract variables */ if ((yc = yang_find(yp, Y_LIST, name)) == NULL) -#ifdef NOTYET +#ifdef NOTYET /* leaf-list is not detected by xpath optimize detection */ if ((yc = yang_find(yp, Y_LEAF_LIST, name)) == NULL) /* XXX */ #endif goto ok; @@ -286,14 +289,12 @@ xpath_list_optimize_fn(xpath_tree *xt, i = 0; cvi = NULL; while ((cvi = cvec_each(cvk, cvi)) != NULL) { - if (strcmp(cv_name_get(cvi), cv_string_get(cvec_i(cvv,i)))){ - fprintf(stderr, "%s key name mismatch %s vs %s\n", - __FUNCTION__, cv_name_get(cvi), cv_string_get(cvec_i(cvv,i))); + if (strcmp(cv_name_get(cvi), cv_string_get(cvec_i(cvv,i)))) goto ok; - } i++; } - if (xml_binsearch(xv, yc, cvk, xp) < 0) + /* Use 2a form since yc allready given to compute cvk */ + if (clixon_xml_find_index(xv, yp, NULL, name, cvk, xvec, xlen) < 0) goto done; retval = 1; /* match */ done: @@ -316,8 +317,9 @@ xpath_list_optimize_fn(xpath_tree *xt, */ int xpath_optimize_check(xpath_tree *xs, - cxobj *xv, - cxobj **xp) + cxobj *xv, + cxobj ***xvec, + size_t *xlen) { #ifdef XPATH_LIST_OPTIMIZE @@ -325,7 +327,7 @@ xpath_optimize_check(xpath_tree *xs, if (!_optimize_enable) return 0; /* use regular code */ - if ((ret = xpath_list_optimize_fn(xs, xv, xp)) < 0) + if ((ret = xpath_list_optimize_fn(xs, xv, xvec, xlen)) < 0) return -1; if (ret == 1){ _optimize_hits++; diff --git a/lib/src/clixon_xpath_parse.h b/lib/src/clixon_xpath_parse.h index 72ba992b..c09c1015 100644 --- a/lib/src/clixon_xpath_parse.h +++ b/lib/src/clixon_xpath_parse.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -39,7 +39,7 @@ /* * Types */ -struct clicon_xpath_yacc_arg{ /* XXX: mostly unrelevant */ +struct clicon_xpath_yacc_arg{ const char *xy_name; /* Name of syntax (for error string) */ int xy_linenum; /* Number of \n in parsed buffer */ char *xy_parse_string; /* original (copy of) parse string */ @@ -55,11 +55,11 @@ extern char *clixon_xpath_parsetext; /* * Prototypes */ -int xpath_scan_init(struct clicon_xpath_yacc_arg *jy); -int xpath_scan_exit(struct clicon_xpath_yacc_arg *jy); +int xpath_scan_init(struct clicon_xpath_yacc_arg *xy); +int xpath_scan_exit(struct clicon_xpath_yacc_arg *xy); -int xpath_parse_init(struct clicon_xpath_yacc_arg *jy); -int xpath_parse_exit(struct clicon_xpath_yacc_arg *jy); +int xpath_parse_init(struct clicon_xpath_yacc_arg *xy); +int xpath_parse_exit(struct clicon_xpath_yacc_arg *xy); int clixon_xpath_parselex(void *); int clixon_xpath_parseparse(void *); diff --git a/lib/src/clixon_xpath_parse.l b/lib/src/clixon_xpath_parse.l index 9ac77880..b7f57b7c 100644 --- a/lib/src/clixon_xpath_parse.l +++ b/lib/src/clixon_xpath_parse.l @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -77,8 +77,6 @@ clixon_xpath_parsewrap(void) return 1; } - - %} digit [0-9] diff --git a/lib/src/clixon_xpath_parse.y b/lib/src/clixon_xpath_parse.y index 5586069c..44504d6b 100644 --- a/lib/src/clixon_xpath_parse.y +++ b/lib/src/clixon_xpath_parse.y @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -124,7 +124,7 @@ #include "clixon_xpath_parse.h" -extern int clixon_xpath_parseget_lineno (void); +extern int clixon_xpath_parseget_lineno (void); /*XXX obsolete ? */ /* also called from yacc generated code * diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 09e6a822..70d2b06c 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -34,19 +34,6 @@ * Yang functions * @see https://tools.ietf.org/html/rfc6020 YANG 1.0 * @see https://tools.ietf.org/html/rfc7950 YANG 1.1 - * - * CALLING ORDER OF YANG PARSE FILES - * ================================= - * yang_spec_parse_module - * | | - * v v v - * yang_spec_parse_file-> yang_parse_post->yang_parse_recurse->yang_parse_module - * \ / v - * yang_spec_load_dir ------------------------------------> yang_parse_filename - * v - * yang_parse_file - * v - * yang_parse_str */ #ifdef HAVE_CONFIG_H @@ -91,12 +78,11 @@ #include "clixon_data.h" #include "clixon_options.h" #include "clixon_yang_parse.h" +#include "clixon_yang_parse_lib.h" #include "clixon_yang_cardinality.h" -#include "clixon_yang_internal.h" /* internal */ +#include "clixon_yang_module.h" #include "clixon_yang_type.h" - -/* Size of json read buffer when reading from file*/ -#define BUFLEN 1024 +#include "clixon_yang_internal.h" /* internal included by this file only, not API*/ /* * Local variables @@ -178,9 +164,33 @@ static const map_str2int ykmap[] = { {NULL, -1} }; +/* Forward static */ +static int yang_type_cache_free(yang_type_cache *ycache); +static int yang_type_cache_cp(yang_stmt *ynew, yang_stmt *yold); + /* Access functions */ +/*! Get Number of children yang statements + * @param[in] ys Yang statement node + */ +int +yang_len_get(yang_stmt *ys) +{ + return ys->ys_len; +} + +/*! Get Specific Yang statement child + * @param[in] ys Yang statement node + * @param[in] i Return this child + */ +yang_stmt * +yang_child_i(yang_stmt *ys, + int i) +{ + return ys->ys_stmt[i]; +} + /*! Get yang statement parent * @param[in] ys Yang statement node */ @@ -208,6 +218,19 @@ yang_argument_get(yang_stmt *ys) return ys->ys_argument; } +/*! Set yang argument, not not copied + * @param[in] ys Yang statement node + * @param[in] arg Argument + * Typically only done at parsing / initiation + */ +int +yang_argument_set(yang_stmt *ys, + char *arg) +{ + ys->ys_argument = arg; /* not strdup/copied */ + return 0; +} + /*! Get yang statement CLIgen variable * @param[in] ys Yang statement node */ @@ -242,6 +265,42 @@ yang_cvec_set(yang_stmt *ys, return 0; } +/*! Get yang stmt flags, used for internal algorithms + * @param[in] ys Yang statement + * @param[in] flag Flags value(s) to get, see YANG_FLAG_* + * @retval value Flags value masked by "flag" parameter, see YANG_FLAG_* + */ +uint16_t +yang_flag_get(yang_stmt *ys, + uint16_t flag) +{ + return ys->ys_flags&flag; +} + +/*! Set yang stmt flags, used for internal algorithms + * @param[in] ys Yang statement + * @param[in] flag Flag value(s) to set, see YANG_FLAG_* + */ +int +yang_flag_set(yang_stmt *ys, + uint16_t flag) +{ + ys->ys_flags |= flag; + return 0; +} + +/*! Reset yang stmt flags, used for internal algorithms + * @param[in] ys Yang statement + * @param[in] flag Flag value(s) to reset, see YANG_FLAG_* + */ +int +yang_flag_reset(yang_stmt *ys, + uint16_t flag) +{ + ys->ys_flags &= ~flag; + return 0; +} + /* End access functions */ /*! Create new yang specification @@ -424,7 +483,7 @@ ys_cp(yang_stmt *ynew, } if (yold->ys_typecache){ ynew->ys_typecache = NULL; - if (yang_type_cache_cp(&ynew->ys_typecache, yold->ys_typecache) < 0) + if (yang_type_cache_cp(ynew, yold) < 0) goto done; } for (i=0; iys_len; i++){ @@ -606,7 +665,7 @@ yang_match(yang_stmt *yn, return match; } -/*! Find child data node with matching argument (container, leaf, etc) +/*! Find child data node with matching argument (container, leaf, list, leaf-list) * * @param[in] yn Yang node, current context node. * @param[in] argument if NULL, match any(first) argument. XXX is that really a case? @@ -1007,17 +1066,6 @@ yang_order(yang_stmt *y) return -1; } -/*! Reset flag in complete tree, arg contains flag */ -static int -ys_flag_reset(yang_stmt *ys, - void *arg) -{ - int flags = (intptr_t)arg; - - ys->ys_flags |= ~flags; - return 0; -} - char * yang_key2str(int keyword) { @@ -1151,127 +1199,6 @@ ys_spec(yang_stmt *ys) return (yang_stmt*)ys; } -/*! Given a yang statement and a prefix, return yang module to that relative prefix - * Note, not the other module but the proxy import statement only - * @param[in] ys A yang statement - * @param[in] prefix prefix - * @retval ymod Yang module statement if found - * @retval NULL not found - * @node Prefixes are relative to the module they are defined - * @see yang_find_module_by_name - * @see yang_find_module_by_namespace - */ -yang_stmt * -yang_find_module_by_prefix(yang_stmt *ys, - char *prefix) -{ - yang_stmt *yimport; - yang_stmt *yprefix; - yang_stmt *my_ymod; - yang_stmt *ymod = NULL; - yang_stmt *yspec; - char *myprefix; - - if ((yspec = ys_spec(ys)) == NULL){ - clicon_err(OE_YANG, 0, "My yang spec not found"); - goto done; - } - if ((my_ymod = ys_module(ys)) == NULL){ - clicon_err(OE_YANG, 0, "My yang module not found"); - goto done; - } - myprefix = yang_find_myprefix(ys); - if (myprefix && strcmp(myprefix, prefix) == 0){ - ymod = my_ymod; - goto done; - } - yimport = NULL; - while ((yimport = yn_each(my_ymod, yimport)) != NULL) { - if (yang_keyword_get(yimport) != Y_IMPORT) - continue; - if ((yprefix = yang_find(yimport, Y_PREFIX, NULL)) != NULL && - strcmp(yang_argument_get(yprefix), prefix) == 0){ - break; - } - } - if (yimport){ - if ((ymod = yang_find(yspec, Y_MODULE, yang_argument_get(yimport))) == NULL){ - clicon_err(OE_YANG, 0, "No module or sub-module found with prefix %s", - prefix); - yimport = NULL; - goto done; /* unresolved */ - } - } - done: - return ymod; -} - -/* Get module from its own prefix - * This is really not a valid usecase, a kludge for the identityref derived - * list workaround (IDENTITYREF_KLUDGE) - */ -yang_stmt * -yang_find_module_by_prefix_yspec(yang_stmt *yspec, - char *prefix) -{ - yang_stmt *ymod = NULL; - yang_stmt *yprefix; - - while ((ymod = yn_each(yspec, ymod)) != NULL) - if (ymod->ys_keyword == Y_MODULE && - (yprefix = yang_find(ymod, Y_PREFIX, NULL)) != NULL && - strcmp(yang_argument_get(yprefix), prefix) == 0) - return ymod; - return NULL; -} - -/*! Given a yang spec and a namespace, return yang module - * - * @param[in] yspec A yang specification - * @param[in] namespace namespace - * @retval ymod Yang module statement if found - * @retval NULL not found - * @see yang_find_module_by_name - * @see yang_find_module_by_prefix module-specific prefix - */ -yang_stmt * -yang_find_module_by_namespace(yang_stmt *yspec, - char *namespace) -{ - yang_stmt *ymod = NULL; - - if (namespace == NULL) - goto done; - while ((ymod = yn_each(yspec, ymod)) != NULL) { - if (yang_find(ymod, Y_NAMESPACE, namespace) != NULL) - break; - } - done: - return ymod; -} - -/*! Given a yang spec and a module name, return yang module or submodule - * - * @param[in] yspec A yang specification - * @param[in] name Name of module - * @retval ymod Yang module statement if found - * @retval NULL not found - * @see yang_find_module_by_namespace - * @see yang_find_module_by_prefix module-specific prefix - */ -yang_stmt * -yang_find_module_by_name(yang_stmt *yspec, - char *name) -{ - yang_stmt *ymod = NULL; - - while ((ymod = yn_each(yspec, ymod)) != NULL) - if ((ymod->ys_keyword == Y_MODULE || ymod->ys_keyword == Y_SUBMODULE) && - strcmp(ymod->ys_argument, name)==0) - return ymod; - return NULL; -} - /*! string is quoted if it contains space or tab, needs double '' */ static int inline quotedstring(char *s) @@ -1966,12 +1893,13 @@ ys_populate(yang_stmt *ys, return retval; } + /*! Run after grouping expand and augment * @see ys_populate run before grouping expand and augment */ -static int +int ys_populate2(yang_stmt *ys, - void *arg) + void *arg) { int retval = -1; clicon_handle h = (clicon_handle)arg; @@ -1995,778 +1923,6 @@ ys_populate2(yang_stmt *ys, return retval; } -/*! Resolve a grouping name from a point in the yang tree - * @param[in] ys Yang statement of "uses" statement doing the lookup - * @param[in] prefix Prefix of grouping to look for - * @param[in] name Name of grouping to look for - * @param[out] ygrouping0 A found grouping yang structure as result - * @retval 0 OK, ygrouping may be NULL - * @retval -1 Error, with clicon_err called - */ -static int -ys_grouping_resolve(yang_stmt *ys, - char *prefix, - char *name, - yang_stmt **ygrouping0) -{ - int retval = -1; - yang_stmt *ymodule; - yang_stmt *ygrouping = NULL; - yang_stmt *yn; - - /* find the grouping associated with argument and expand(?) */ - if (prefix){ /* Go to top and find import that matches */ - if ((ymodule = yang_find_module_by_prefix(ys, prefix)) != NULL) - ygrouping = yang_find(ymodule, Y_GROUPING, name); - } - else - while (1){ - /* Check upwards in hierarchy for matching groupings */ - if ((yn = ys->ys_parent) == NULL || yn->ys_keyword == Y_SPEC) - break; - /* Here find grouping */ - if ((ygrouping = yang_find(yn, Y_GROUPING, name)) != NULL) - break; - /* Proceed to next level */ - ys = (yang_stmt*)yn; - } - *ygrouping0 = ygrouping; - retval = 0; - // done: - return retval; -} - -/*! This is an augment node, augment the original datamodel. - * @param[in] ys The augment statement - * @param[in] yspec Yang specification - * @see RFC7950 Sec 7.17 - * The target node MUST be either a container, list, choice, case, input, - * output, or notification node. - * If the "augment" statement is on the top level the absolute form MUST be - * used. - * All data nodes defined in the "augment" statement are defined as XML - * elements in the XML namespace of the module where the "augment" is - * specified. - */ -static int -yang_augment_node(yang_stmt *ys, - yang_stmt *ysp) -{ - int retval = -1; - char *schema_nodeid; - yang_stmt *ytarget = NULL; - yang_stmt *yc; - int i; - yang_stmt *ymod; - - if ((ymod = ys_module(ys)) == NULL){ - clicon_err(OE_YANG, 0, "My yang module not found"); - goto done; - } - schema_nodeid = ys->ys_argument; - clicon_debug(2, "%s %s", __FUNCTION__, schema_nodeid); - /* Find the target */ - if (yang_abs_schema_nodeid(ysp, ys, schema_nodeid, -1, &ytarget) < 0) - goto done; - if (ytarget == NULL) - goto ok; - /* Extend ytarget with ys' children - * First enlarge ytarget vector - */ - for (i=0; iys_len; i++){ - if ((yc = ys_dup(ys->ys_stmt[i])) == NULL) - goto done; - yc->ys_mymodule = ymod; - if (yn_insert(ytarget, yc) < 0) - goto done; - } - ok: - retval = 0; - done: - return retval; -} - -/*! Find all top-level augments and change original datamodels. */ -static int -yang_augment_spec(yang_stmt *ysp, - int modnr) -{ - int retval = -1; - yang_stmt *ym; - yang_stmt *ys; - int i; - int j; - - i = modnr; - while (iys_len){ /* Loop through modules and sub-modules */ - ym = ysp->ys_stmt[i++]; - j = 0; - while (jys_len){ /* Top-level symbols in modules */ - ys = ym->ys_stmt[j++]; - switch (ys->ys_keyword){ - case Y_AUGMENT: /* top-level */ - if (yang_augment_node(ys, ysp) < 0) - goto done; - break; - default: - break; - } - } - } - retval = 0; - done: - return retval; -} - -/*! Given a refine node, perform the refinement action on the target refine node - * The RFC is somewhat complicate in the rules for refine. - * Most nodes will be replaced, but some are added - * @param[in] yr Refine node - * @param[in] yt Refine target node (will be modified) - * @see RFC7950 Sec 7.13.2 - * There may be some missed cornercases - */ -static int -ys_do_refine(yang_stmt *yr, - yang_stmt *yt) -{ - int retval = -1; - yang_stmt *yrc; /* refine child */ - yang_stmt *yrc1; - yang_stmt *ytc; /* target child */ - enum rfc_6020 keyw; - int i; - - /* Loop through refine node children. First if remove do that first - * In some cases remove a set of nodes. - */ - yrc = NULL; - while ((yrc = yn_each(yr, yrc)) != NULL) { - keyw = yang_keyword_get(yrc); - switch (keyw){ - case Y_DEFAULT: /* remove old, add new */ - case Y_DESCRIPTION: - case Y_REFERENCE: - case Y_CONFIG: - case Y_MANDATORY: - case Y_PRESENCE: - case Y_MIN_ELEMENTS: - case Y_MAX_ELEMENTS: - case Y_EXTENSION: - /* Remove old matching, dont increment due to prune in loop */ - for (i=0; iys_len; ){ - ytc = yt->ys_stmt[i]; - if (keyw != yang_keyword_get(ytc)){ - i++; - continue; - } - ys_prune(yt, i); - ys_free(ytc); - } - /* fall through and add if not found */ - case Y_MUST: /* keep old, add new */ - case Y_IF_FEATURE: - break; - default: - break; - } - } - /* Second, add the node(s) */ - yrc = NULL; - while ((yrc = yn_each(yr, yrc)) != NULL) { - keyw = yang_keyword_get(yrc); - /* Make copy */ - if ((yrc1 = ys_dup(yrc)) == NULL) - goto done; - if (yn_insert(yt, yrc1) < 0) - goto done; - } - retval = 0; - done: - return retval; -} - -/*! Macro expansion of grouping/uses done in step 2 of yang parsing - * RFC7950: - * Identifiers appearing inside the grouping are resolved - * relative to the scope in which the grouping is defined, not where it is - * used. Prefix mappings, type names, grouping names, and extension usage are - * evaluated in the hierarchy where the "grouping" statement appears. - * The identifiers defined in the grouping are not bound to a namespace - * until the contents of the grouping are added to the schema tree via a - * "uses" statement that does not appear inside a "grouping" statement, - * at which point they are bound to the namespace of the current module. - */ -static int -yang_expand_grouping(yang_stmt *yn) -{ - int retval = -1; - yang_stmt *ys = NULL; - yang_stmt *ygrouping; /* grouping original */ - yang_stmt *ygrouping2; /* grouping copy */ - yang_stmt *yg; /* grouping child */ - yang_stmt *yr; /* refinement */ - int glen; - int i; - int j; - char *id = NULL; - char *prefix = NULL; - size_t size; - - /* Cannot use yang_apply here since child-list is modified (is destructive) */ - i = 0; - while (iys_len){ - ys = yn->ys_stmt[i]; - switch(ys->ys_keyword){ - case Y_USES: - /* Split argument into prefix and name */ - if (nodeid_split(yang_argument_get(ys), &prefix, &id) < 0) - goto done; - if (ys_grouping_resolve(ys, prefix, id, &ygrouping) < 0) - goto done; - if (prefix){ - free(prefix); - prefix = NULL; - } - if (id){ - free(id); - id = NULL; - } - if (ygrouping == NULL){ - clicon_log(LOG_NOTICE, "%s: Yang error : grouping \"%s\" not found in module \"%s\"", - __FUNCTION__, ys->ys_argument, ys_module(ys)->ys_argument); - goto done; - break; - } - /* Check mark flag to see if this grouping (itself) has been expanded - If not, this needs to be done before we can insert it into - the 'uses' place */ - if ((ygrouping->ys_flags & YANG_FLAG_MARK) == 0){ - if (yang_expand_grouping(ygrouping) < 0) - goto done; - ygrouping->ys_flags |= YANG_FLAG_MARK; /* Mark as expanded */ - } - /* Make a copy of the grouping, then make refinements to this copy - */ - if ((ygrouping2 = ys_dup(ygrouping)) == NULL) - goto done; - /* Replace ys with ygrouping,... - * First enlarge parent vector - */ - glen = ygrouping2->ys_len; - /* - * yn is parent: the children of ygrouping replaces ys. - * Is there a case when glen == 0? YES AND THIS BREAKS - */ - if (glen != 1){ - size = (yn->ys_len - i - 1)*sizeof(struct yang_stmt *); - yn->ys_len += glen - 1; - if (glen && (yn->ys_stmt = realloc(yn->ys_stmt, (yn->ys_len)*sizeof(yang_stmt *))) == 0){ - clicon_err(OE_YANG, errno, "realloc"); - goto done; - } - /* Then move all existing elements up from i+1 (not uses-stmt) */ - if (size) - memmove(&yn->ys_stmt[i+glen], - &yn->ys_stmt[i+1], - size); - } - - /* Iterate through refinements and modify grouping copy - * See RFC 7950 7.13.2 yrt is the refine target node - */ - yr = NULL; - while ((yr = yn_each(ys, yr)) != NULL) { - yang_stmt *yrt; /* refine target node */ - if (yang_keyword_get(yr) != Y_REFINE) - continue; - /* Find a node */ - if (yang_desc_schema_nodeid(ygrouping2, - yang_argument_get(yr), - -1, - &yrt) < 0) - goto done; - /* Not found, try next */ - if (yrt == NULL) - continue; - /* Do the actual refinement */ - if (ys_do_refine(yr, yrt) < 0) - goto done; - /* RFC: The argument is a string that identifies a node in the - * grouping. I interpret that as only one node --> break */ - break; - } - /* Then copy and insert each child element */ - for (j=0; jys_stmt[j]; /* Child of refined copy */ - yn->ys_stmt[i+j] = yg; - yg->ys_parent = yn; - } - /* Remove 'uses' node */ - ys_free(ys); - /* Remove the grouping copy */ - ygrouping2->ys_len = 0; - ys_free(ygrouping2); - break; /* Note same child is re-iterated since it may be changed */ - default: - i++; - break; - } - } - /* Second pass since length may have changed */ - for (i=0; iys_len; i++){ - ys = yn->ys_stmt[i]; - if (yang_expand_grouping(ys) < 0) - goto done; - } - retval = 0; - done: - if (prefix) - free(prefix); - if (id) - free(id); - return retval; -} - -/*! Parse a string containing a YANG spec into a parse-tree - * - * Syntax parsing. A string is input and a syntax-tree is returned (or error). - * A variable record is also returned containing a list of (global) variable values. - * (cloned from cligen) - * @param[in] str String of yang statements - * @param[in] name Log string, typically filename - * @param[in] yspec Yang specification. - * @retval ymod Top-level yang (sub)module - * @retval NULL Error encountered - * See top of file for diagram of calling order - */ -static yang_stmt * -yang_parse_str(char *str, - const char *name, /* just for errs */ - yang_stmt *yspec) -{ - struct clicon_yang_yacc_arg yy = {0,}; - yang_stmt *ymod = NULL; - - if (yspec == NULL){ - clicon_err(OE_YANG, 0, "Yang parse need top level yang spec"); - goto done; - } - yy.yy_name = (char*)name; - yy.yy_linenum = 1; - yy.yy_parse_string = str; - yy.yy_stack = NULL; - yy.yy_module = NULL; /* this is the return value - the module/sub-module */ - if (ystack_push(&yy, yspec) == NULL) - goto done; - if (strlen(str)){ /* Not empty */ - if (yang_scan_init(&yy) < 0) - goto done; - if (yang_parse_init(&yy) < 0) - goto done; - if (clixon_yang_parseparse(&yy) != 0) { /* yacc returns 1 on error */ - clicon_log(LOG_NOTICE, "Yang error: %s on line %d", name, yy.yy_linenum); - if (clicon_errno == 0) - clicon_err(OE_YANG, 0, "yang parser error with no error code (should not happen)"); - yang_parse_exit(&yy); - yang_scan_exit(&yy); - goto done; - } - if (yang_parse_exit(&yy) < 0) - goto done; - if (yang_scan_exit(&yy) < 0) - goto done; - } - ymod = yy.yy_module; - done: - ystack_pop(&yy); - if (yy.yy_stack) - free (yy.yy_stack); - return ymod; /* top-level (sub)module */ -} - -/*! Parse yang spec from an open file descriptor - * @param[in] fd File descriptor containing the YANG file as ASCII characters - * @param[in] name For debug, eg filename - * @param[in] ysp Yang specification. Should have been created by caller using yspec_new - * @retval ymod Top-level yang (sub)module - * @retval NULL Error - * @note this function simply parse a yang spec, no dependencies or checks - */ -yang_stmt * -yang_parse_file(int fd, - const char *name, - yang_stmt *ysp) -{ - char *buf = NULL; - int i; - int c; - int len; - yang_stmt *ymod = NULL; - int ret; - - len = BUFLEN; /* any number is fine */ - if ((buf = malloc(len)) == NULL){ - perror("pt_file malloc"); - return NULL; - } - memset(buf, 0, len); - i = 0; /* position in buf */ - while (1){ /* read the whole file */ - if ((ret = read(fd, &c, 1)) < 0){ - clicon_err(OE_XML, errno, "read"); - break; - } - if (ret == 0) - break; /* eof */ - if (len==i){ - if ((buf = realloc(buf, 2*len)) == NULL){ - clicon_err(OE_XML, errno, "realloc"); - goto done; - } - memset(buf+len, 0, len); - len *= 2; - } - buf[i++] = (char)(c&0xff); - } /* read a line */ - if ((ymod = yang_parse_str(buf, name, ysp)) < 0) - goto done; - done: - if (buf) - free(buf); - return ymod; /* top-level (sub)module */ -} - -/*! Given a yang filename, extract the revision as an integer as YYYYMMDD - * @param[in] filename Filename on the form: name [+ @rev ] + .yang - * @param[out] basep "Base" filename, stripped: [+ @rev ] + .yang - * @param[out] revp Revision as YYYYMMDD (0 if not found) - */ -static int -filename2revision(const char *filename, - char **basep, - uint32_t *revp) -{ - int retval = -1; - char *base = NULL; - char *p; - - /* base = module name [+ @rev ] + .yang */ - if ((base = strdup(filename)) == NULL){ - clicon_err(OE_UNIX, errno, "strdup"); - goto done; - } - clicon_debug(1, "%s %s", __FUNCTION__, base); - if ((p = rindex(base, '.')) != NULL) /* strip postfix .yang */ - *p = '\0'; - if ((p = index(base, '@')) != NULL){ /* extract revision date */ - *p++ = '\0'; - if (ys_parse_date_arg(p, revp) < 0) - goto done; - } - if (basep){ - *basep = base; - base = NULL; - } - retval = 0; - done: - if (base) - free(base); - return retval; -} - -/*! No specific revision give. Match a yang file given module - * @param[in] h CLICON handle - * @param[in] module Name of main YANG module. - * @param[in] revision Revision or NULL - * @param[out] revactual Actual revision (if retval=1) - * @param[out] fbuf Buffer containing filename (if retval=1) - * @retval 1 Match found, Most recent entry returned in fbuf and revactual - * @retval 0 No matching entry found - * @retval -1 Error - * @note for bootstrapping, dir may have to be set. -*/ -static int -yang_parse_find_match(clicon_handle h, - const char *module, - const char *revision, - uint32_t *revactual, - cbuf *fbuf) -{ - int retval = -1; - struct dirent *dp = NULL; - int ndp; - cbuf *regex = NULL; - cxobj *x; - cxobj *xc; - char *dir; - - /* get clicon config file in xml form */ - if ((x = clicon_conf_xml(h)) == NULL) - goto ok; - if ((regex = cbuf_new()) == NULL){ - clicon_err(OE_YANG, errno, "cbuf_new"); - goto done; - } - /* RFC 6020: The name of the file SHOULD be of the form: - * module-or-submodule-name ['@' revision-date] ( '.yang' / '.yin' ) - * revision-date ::= 4DIGIT "-" 2DIGIT "-" 2DIGIT - */ - if (revision) - cprintf(regex, "^%s@%s(.yang)$", module, revision); - else - cprintf(regex, "^%s(@[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])?(.yang)$", - module); - xc = NULL; - while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) { - if (strcmp(xml_name(xc), "CLICON_YANG_DIR") != 0) - continue; - dir = xml_body(xc); - /* get all matching files in this directory */ - if ((ndp = clicon_file_dirent(dir, - &dp, - cbuf_get(regex), - S_IFREG)) < 0) - goto done; - /* Entries are sorted, last entry should be most recent date - */ - if (ndp != 0){ - cprintf(fbuf, "%s/%s", dir, dp[ndp-1].d_name); - retval = 1; - goto done; - } - } - ok: - retval = 0; - done: - if (regex) - cbuf_free(regex); - if (dp) - free(dp); - return retval; -} - -/*! Open a file, read into a string and invoke yang parsing - * - * Similar to clicon_yang_str(), just read a file first - * @param[in] filename Name of file - * @param[in] ysp Yang specification. Should have been created by caller using yspec_new - * @retval ymod Top-level yang (sub)module - * @retval NULL Error encountered - - * The database symbols are inserted in alphabetical order. - * See top of file for diagram of calling order - */ -yang_stmt * -yang_parse_filename(const char *filename, - yang_stmt *ysp) -{ - yang_stmt *ymod = NULL; - int fd = -1; - struct stat st; - - clicon_debug(1, "%s %s", __FUNCTION__, filename); - if (stat(filename, &st) < 0){ - clicon_err(OE_YANG, errno, "%s not found", filename); - goto done; - } - if ((fd = open(filename, O_RDONLY)) < 0){ - clicon_err(OE_YANG, errno, "open(%s)", filename); - goto done; - } - if ((ymod = yang_parse_file(fd, filename, ysp)) < 0) - goto done; - done: - if (fd != -1) - close(fd); - return ymod; /* top-level (sub)module */ -} - -/*! Given a (sub)module, parse all (sub)modules in turn recursively - * - * Find a yang module file, and then recursively parse all its imported modules. - * @param[in] h CLICON handle - * @param[in] module Module name - * @param[in] revision Revision (or NULL) - * @param[in] ysp Yang statement - * @retval 0 OK - * @retval -1 Error - * - * See top of file for diagram of calling order - */ -static yang_stmt * -yang_parse_module(clicon_handle h, - const char *module, - const char *revision, - yang_stmt *ysp) -{ - cbuf *fbuf = NULL; - char *filename; - int nr; - yang_stmt *ymod = NULL; - yang_stmt *yrev; /* yang revision */ - uint32_t revf = 0; /* revision in filename */ - uint32_t revm = 0; /* revision in parsed new module (should be same as revf) */ - - if ((fbuf = cbuf_new()) == NULL){ - clicon_err(OE_YANG, errno, "cbuf_new"); - goto done; - } - /* Match a yang file with or without revision in yang-dir list */ - if ((nr = yang_parse_find_match(h, module, revision, &revf, fbuf)) < 0) - goto done; - if (nr == 0){ - clicon_err(OE_YANG, errno, "No yang files found matching \"%s\" in the list of CLICON_YANG_DIRs", module); - goto done; - } - filename = cbuf_get(fbuf); - if ((ymod = yang_parse_filename(filename, ysp)) == NULL) - goto done; - if ((yrev = yang_find(ymod, Y_REVISION, NULL)) != NULL) - revm = cv_uint32_get(yrev->ys_cv); - if (filename2revision(filename, NULL, &revf) < 0) - goto done; - /* Sanity check that file revision does not match internal rev stmt */ - if (revf && revm && revm != revf){ - clicon_err(OE_YANG, EINVAL, "Yang module file revision and in yang does not match: %s vs %u", filename, revm); - ymod = NULL; - goto done; - } - done: - if (fbuf) - cbuf_free(fbuf); - return ymod; /* top-level (sub)module */ -} - -/*! Given a (sub)module, parse all (sub)modules in turn recursively - * - * Find a yang module file, and then recursively parse all its imported modules. - * @param[in] h CLICON handle - * @param[in] ymod Yang module. - * @param[in] yspec Yang specification. - * @retval 0 OK - * @retval -1 Error - * - * See top of file for diagram of calling order - */ -static int -yang_parse_recurse(clicon_handle h, - yang_stmt *ymod, - yang_stmt *ysp) -{ - int retval = -1; - yang_stmt *yi = NULL; /* import */ - yang_stmt *yrev; - char *submodule; - char *subrevision; - yang_stmt *subymod; - enum rfc_6020 keyw; - - /* go through all import (modules) and include(submodules) of ysp */ - while ((yi = yn_each(ymod, yi)) != NULL){ - keyw = yi->ys_keyword; - if (keyw != Y_IMPORT && keyw != Y_INCLUDE) - continue; - /* common part */ - submodule = yi->ys_argument; - /* Is there a specific revision (or just latest)? */ - if ((yrev = yang_find(yi, Y_REVISION_DATE, NULL)) != NULL) - subrevision = yrev->ys_argument; - else - subrevision = NULL; - /* if already loaded, ignore, else parse the file */ - if (yang_find(ysp, - keyw==Y_IMPORT?Y_MODULE:Y_SUBMODULE, - submodule) == NULL){ - /* recursive call */ - if ((subymod = yang_parse_module(h, submodule, subrevision, ysp)) == NULL) - goto done; - /* Go through its sub-modules recursively */ - if (yang_parse_recurse(h, subymod, ysp) < 0){ - ymod = NULL; - goto done; - } - } - } - retval = 0; - done: - return retval; /* top-level (sub)module */ -} - -/*! - * @param[in] ys Yang statement - * @param[in] dummy Necessary for called in yang_apply - * @see yang_apply_fn - */ -static int -ys_schemanode_check(yang_stmt *ys, - void *dummy) -{ - int retval = -1; - yang_stmt *yspec; - yang_stmt *yres = NULL; - yang_stmt *yp; - char *arg; - enum rfc_6020 keyword; - char **vec = NULL; - char *v; - int nvec; - int i; - - yp = yang_parent_get(ys); - arg = yang_argument_get(ys); - keyword = yang_keyword_get(ys); - switch (yang_keyword_get(ys)){ - case Y_AUGMENT: - if (yang_keyword_get(yp) == Y_MODULE || /* Not top-level */ - yang_keyword_get(yp) == Y_SUBMODULE) - break; - /* fallthru */ - case Y_REFINE: - if (yang_desc_schema_nodeid(yp, arg, -1, &yres) < 0) - goto done; - if (yres == NULL){ - clicon_err(OE_YANG, 0, "schemanode sanity check of %s %s", - yang_key2str(keyword), arg); - goto done; - } - break; - case Y_UNIQUE:{ - /* Unique: Sec 7.8.3 It takes as an argument a string that contains a space- - separated list of schema node identifiers */ - if ((vec = clicon_strsep(arg, " \t\n", &nvec)) == NULL) - goto done; - for (i=0; iys_len; i++) - if (yang_parse_recurse(h, yspec->ys_stmt[i], yspec) < 0) - goto done; - - /* 2. Check cardinality maybe this should be done after grouping/augment */ - for (i=modnr; iys_len; i++) - if (yang_cardinality(h, yspec->ys_stmt[i], yspec->ys_stmt[i]->ys_argument) < 0) - goto done; - - /* 3: Check features: check if enabled and remove disabled features */ - for (i=modnr; iys_len; i++) /* XXX */ - if (yang_features(h, yspec->ys_stmt[i]) < 0) - goto done; - - /* 4: Go through parse tree and populate it with cv types */ - for (i=modnr; iys_len; i++) - if (yang_apply(yspec->ys_stmt[i], -1, ys_populate, (void*)h) < 0) - goto done; - - /* 5: Resolve all types: populate type caches. Requires eg length/range cvecs - * from ys_populate step. - * Must be done using static binding. - */ - for (i=modnr; iys_len; i++) - if (yang_apply(yspec->ys_stmt[i], Y_TYPE, ys_resolve_type, h) < 0) - goto done; - - /* Up to here resolving is made in the context they are defined, rather - * than the context they are used (except for submodules being merged w - * modules). Like static scoping. - * After this we expand all grouping/uses and unfold all macros into a - * single tree as they are used. - */ - - /* 6: Macro expansion of all grouping/uses pairs. Expansion needs marking */ - for (i=modnr; iys_len; i++){ - if (yang_expand_grouping(yspec->ys_stmt[i]) < 0) - goto done; - yang_apply(yspec->ys_stmt[i], -1, ys_flag_reset, (void*)YANG_FLAG_MARK); - } - - /* 7: Top-level augmentation of all modules. (Augment also in uses) */ - if (yang_augment_spec(yspec, modnr) < 0) - goto done; - - /* 4: Go through parse tree and do 2nd step populate (eg default) */ - for (i=modnr; iys_len; i++) - if (yang_apply(yspec->ys_stmt[i], -1, ys_populate2, (void*)h) < 0) - goto done; - - /* 8: sanity check of schemanode references, need more here */ - for (i=modnr; iys_len; i++) - if (yang_apply(yspec->ys_stmt[i], -1, ys_schemanode_check, NULL) < 0) - goto done; - retval = 0; - done: - return retval; -} - -/*! Parse yang specification and its dependencies recursively given module - * @param[in] h clicon handle - * @param[in] module Module name, or absolute filename (including dir) - * @param[in] revision Revision, or NULL - * @param[in] yspec Modules parse are added to this yangspec - * @retval 0 OK - * @retval -1 Error - * @see yang_spec_parse_file - */ -int -yang_spec_parse_module(clicon_handle h, - const char *module, - const char *revision, - yang_stmt *yspec) -{ - int retval = -1; - int modnr; /* Existing number of modules */ - char *base = NULL;; - - if (yspec == NULL){ - clicon_err(OE_YANG, EINVAL, "yang spec is NULL"); - goto done; - } - if (module == NULL){ - clicon_err(OE_YANG, EINVAL, "yang module not set"); - goto done; - } - /* Apply steps 2.. on new modules, ie ones after modnr. */ - modnr = yspec->ys_len; - /* Do not load module if it already exists */ - if (yang_find(yspec, Y_MODULE, module) != NULL) - goto ok; - if (yang_parse_module(h, module, revision, yspec) == NULL) - goto done; - if (yang_parse_post(h, yspec, modnr) < 0) - goto done; - ok: - retval = 0; - done: - if (base) - free(base); - return retval; -} - -/*! Parse yang specification and its dependencies recursively given filename - * @param[in] h clicon handle - * @param[in] filename Actual filename (including dir and revision) - * @param[in] yspec Modules parse are added to this yangspec - * @retval 0 OK - * @retval -1 Error - * @see yang_spec_parse_module for yang dir,module,revision instead of - * actual filename - * @see yang_spec_load_dir For loading all files in a directory - */ -int -yang_spec_parse_file(clicon_handle h, - char *filename, - yang_stmt *yspec) -{ - int retval = -1; - int modnr; /* Existing number of modules */ - char *base = NULL;; - - /* Apply steps 2.. on new modules, ie ones after modnr. */ - modnr = yspec->ys_len; - /* Find module, and do not load file if module already exists */ - if (basename(filename) == NULL){ - clicon_err(OE_YANG, errno, "No basename"); - goto done; - } - if ((base = strdup(basename(filename))) == NULL){ - clicon_err(OE_YANG, errno, "strdup"); - goto done; - } - if (index(base, '@') != NULL) - *index(base, '@') = '\0'; - if (yang_find(yspec, Y_MODULE, base) != NULL) - goto ok; - if (yang_parse_filename(filename, yspec) == NULL) - goto done; - if (yang_parse_post(h, yspec, modnr) < 0) - goto done; - ok: - retval = 0; - done: - if (base) - free(base); - return retval; -} - -/*! Load all yang modules in directory - * @param[in] h Clicon handle - * @param[in] dir Load all yang modules in this directory - * @param[in] yspec Modules parse are added to this yangspec - * @retval 0 OK - * @retval -1 Error - * @see yang_spec_parse_file - * Load all yang files in a directory as primary objects. - * Some details if several same yang module x exists: - * 1) If x is already loaded (eg via direct file loading) skip it - * 2) Prefer x.yang over x@rev.yang (no revision) - * 3) If only x@rev.yang's found, prefer newest (newest revision) - * There is also an extra failsafe which may not be necessary, which removes - * the oldest module if 1-3 for some reason fails. - */ -int -yang_spec_load_dir(clicon_handle h, - char *dir, - yang_stmt *yspec) -{ - int retval = -1; - int ndp; - struct dirent *dp = NULL; - int i; - int j; - char filename[MAXPATHLEN]; - char *base = NULL; /* filename without dir */ - int modnr; - yang_stmt *ym; /* yang module */ - yang_stmt *ym0; /* (existing) yang module */ - yang_stmt *yrev; /* yang revision */ - uint32_t revf = 0; /* revision in filename */ - uint32_t revm = 0; /* revision in parsed new module (should be same as revf) */ - uint32_t rev0; /* revision in existing module */ - char *oldbase = NULL; - int taken = 0; - - /* Get yang files names from yang module directory. Note that these - * are sorted alphatetically: - * a.yang, - * a@2000-01-01.yang, - * a@2111-11-11.yang - */ - if((ndp = clicon_file_dirent(dir, &dp, "(.yang)$", S_IFREG)) < 0) - goto done; - if (ndp == 0) - clicon_log(LOG_WARNING, "%s: No yang files found in %s", - __FUNCTION__, dir); - /* Apply post steps on new modules, ie ones after modnr. */ - modnr = yspec->ys_len; - /* Load all yang files in dir */ - for (i = 0; i < ndp; i++) { - /* base = module name [+ @rev ] + .yang */ - if (oldbase) - free(oldbase); - oldbase = base; - base = NULL; - revf = 0; - if (filename2revision(dp[i].d_name, &base, &revf) < 0) - goto done; - if (oldbase && strcmp(base, oldbase)) /* new yang file basename */ - taken = 0; - if (revf == 0) /* No revision: a.yang - take that */ - taken = 1; - else{ /* a@xxx.yang */ - if (taken) - continue; /* skip if already taken */ - /* Look forward: is there anyone else later? */ - if (i+1 ym0/rev0 */ - rev0 = 0; - if ((ym0 = yang_find(yspec, Y_MODULE, base)) != NULL || - (ym0 = yang_find(yspec, Y_SUBMODULE, base)) != NULL){ - yrev = yang_find(ym0, Y_REVISION, NULL); - rev0 = cv_uint32_get(yrev->ys_cv); - continue; /* skip if already added by specific file or module */ - } - /* Create full filename */ - snprintf(filename, MAXPATHLEN-1, "%s/%s", dir, dp[i].d_name); - if ((ym = yang_parse_filename(filename, yspec)) == NULL) - goto done; - revm = 0; - if ((yrev = yang_find(ym, Y_REVISION, NULL)) != NULL) - revm = cv_uint32_get(yrev->ys_cv); - /* Sanity check that file revision does not match internal rev stmt */ - if (revf && revm && revm != revf){ /* XXX */ - clicon_err(OE_YANG, EINVAL, "Yang module file revision and in yang does not match: %s(%u) vs %u", filename, revf, revm); - goto done; - } - /* If ym0 and ym exists, delete the yang with oldest revision - * This is a failsafe in case anything else fails - */ - if (revm && rev0){ - if (revm > rev0) /* Loaded module is older or eq -> remove ym */ - ym = ym0; - for (j=0; jys_len; j++) - if (yspec->ys_stmt[j] == ym) - break; - ys_prune(yspec, j); - ys_free(ym); - } - } - if (yang_parse_post(h, yspec, modnr) < 0) - goto done; - retval = 0; - done: - if (dp) - free(dp); - if (base) - free(base); - if (oldbase) - free(oldbase); - return retval; -} - /*! Apply a function call recursively on all yang-stmt s recursively * * Recursively traverse all yang-nodes in a parse-tree and apply fn(arg) for @@ -3344,8 +2202,7 @@ yang_abs_schema_nodeid(yang_stmt *yspec, } /* Assume schema nodeid looks like: "/prefix:id[/prefix:id]*" */ if (nvec < 2){ - clicon_err(OE_YANG, 0, "NULL or truncated path: %s", - schema_nodeid); + clicon_err(OE_YANG, 0, "NULL or truncated path: %s", schema_nodeid); goto done; } /* split : */ @@ -3370,6 +2227,10 @@ yang_abs_schema_nodeid(yang_stmt *yspec, } } } + if (ymod == NULL){ + clicon_err(OE_YANG, 0, "No module with prefix %s found", prefix); + goto done; + } if (schema_nodeid_vec(ymod, vec+1, nvec-1, keyword, yres) < 0) goto done; ok: /* yres may not be set */ @@ -3421,195 +2282,6 @@ yang_desc_schema_nodeid(yang_stmt *yn, return retval; } -/*! parse yang date-arg string and return a uint32 useful for arithmetics - * @param[in] datearg yang revision string as "YYYY-MM-DD" - * @param[out] dateint Integer version as YYYYMMDD - * @retval 0 OK - * @retval -1 Error, eg str is not on the format "YYYY-MM-DD" - */ -int -ys_parse_date_arg(char *datearg, - uint32_t *dateint) -{ - int retval = -1; - int i; - uint32_t d = 0; - - if (strlen(datearg) != 10 || datearg[4] != '-' || datearg[7] != '-'){ - clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); - goto done; - } - if ((i = cligen_tonum(4, datearg)) < 0){ - clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); - goto done; - } - d = i*10000; /* year */ - if ((i = cligen_tonum(2, &datearg[5])) < 0){ - clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); - goto done; - } - d += i*100; /* month */ - if ((i = cligen_tonum(2, &datearg[8])) < 0){ - clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); - goto done; - } - d += i; /* day */ - *dateint = d; - retval = 0; - done: - return retval; -} - -/*! Parse argument as CV and save result in yang cv variable - * - * Note that some CV:s are parsed directly (eg fraction-digits) while others are parsed - * in third pass (ys_populate). The reason being that all information is not - * available in the first pass. Prefer to do stuff in ys_populate - */ -cg_var * -ys_parse(yang_stmt *ys, - enum cv_type cvtype) -{ - int cvret; - char *reason = NULL; - - assert(ys->ys_cv == NULL); /* Cv:s are parsed in different places, difficult to separate */ - if ((ys->ys_cv = cv_new(cvtype)) == NULL){ - clicon_err(OE_YANG, errno, "cv_new"); - goto done; - } - if ((cvret = cv_parse1(ys->ys_argument, ys->ys_cv, &reason)) < 0){ /* error */ - clicon_err(OE_YANG, errno, "parsing cv"); - ys->ys_cv = NULL; - goto done; - } - if (cvret == 0){ /* parsing failed */ - clicon_err(OE_YANG, errno, "Parsing CV: %s", reason); - ys->ys_cv = NULL; - goto done; - } - /* cvret == 1 means parsing is OK */ - done: - if (reason) - free(reason); - return ys->ys_cv; -} - -/*! First round yang syntactic statement specific checks. No context checks. - * - * Specific syntax checks and variable creation for stand-alone yang statements. - * That is, siblings, etc, may not be there. Complete checks are made in - * ys_populate instead. - * @param[in] ys yang statement - * @param[in] extra Yang extra for cornercases (unknown/extension). Is consumed - * - * The cv:s created in parse-tree as follows: - * fraction-digits : Create cv as uint8, check limits [1:8] (must be made in 1st pass) - * revision: cv as uint32 date: Integer version as YYYYMMDD - * min-elements: cv as uint32 - * max-elements: cv as uint32, '0' means unbounded - * @see ys_populate - */ -int -ys_parse_sub(yang_stmt *ys, - char *extra) -{ - int retval = -1; - uint8_t fd; - uint32_t date = 0; - char *arg; - enum rfc_6020 keyword; - char *reason = NULL; - int ret; - uint32_t minmax; - - arg = yang_argument_get(ys); - keyword = yang_keyword_get(ys); - switch (keyword){ - case Y_FRACTION_DIGITS: - if (ys_parse(ys, CGV_UINT8) == NULL) - goto done; - fd = cv_uint8_get(ys->ys_cv); - if (fd < 1 || fd > 18){ - clicon_err(OE_YANG, errno, "%u: Out of range, should be [1:18]", fd); - goto done; - } - break; - case Y_REVISION: - case Y_REVISION_DATE: /* YYYY-MM-DD encoded as uint32 YYYYMMDD */ - if (ys_parse_date_arg(arg, &date) < 0) - goto done; - if ((ys->ys_cv = cv_new(CGV_UINT32)) == NULL){ - clicon_err(OE_YANG, errno, "cv_new"); - goto done; - } - cv_uint32_set(ys->ys_cv, date); - break; - case Y_STATUS: /* RFC7950 7.21.2: "current", "deprecated", or "obsolete". */ - if (strcmp(arg, "current") && - strcmp(arg, "deprecated") && - strcmp(arg, "obsolete")){ - clicon_err(OE_YANG, errno, "Invalid status: \"%s\", expected current, deprecated, or obsolete", arg); - goto done; - - } - break; - case Y_MAX_ELEMENTS: - case Y_MIN_ELEMENTS: - if ((ys->ys_cv = cv_new(CGV_UINT32)) == NULL){ - clicon_err(OE_YANG, errno, "cv_new"); - goto done; - } - if (keyword == Y_MAX_ELEMENTS && - strcmp(arg, "unbounded") == 0) - cv_uint32_set(ys->ys_cv, 0); /* 0 means unbounded for max */ - else{ - if ((ret = parse_uint32(arg, &minmax, &reason)) < 0){ - clicon_err(OE_YANG, errno, "parse_uint32"); - goto done; - } - if (ret == 0){ - clicon_err(OE_YANG, EINVAL, "element-min/max parse error: %s", reason); - if (reason) - free(reason); - goto done; - } - cv_uint32_set(ys->ys_cv, minmax); - } - break; - case Y_MODIFIER: - if (strcmp(yang_argument_get(ys), "invert-match")){ - clicon_err(OE_YANG, EINVAL, "modifier %s, expected invert-match", yang_argument_get(ys)); - goto done; - } - break; - case Y_UNKNOWN:{ /* save (optional) argument in ys_cv */ - if (extra == NULL) - break; - if ((ys->ys_cv = cv_new(CGV_STRING)) == NULL){ - clicon_err(OE_YANG, errno, "cv_new"); - goto done; - } - if ((ret = cv_parse1(extra, ys->ys_cv, &reason)) < 0){ /* error */ - clicon_err(OE_YANG, errno, "parsing cv"); - goto done; - } - if (ret == 0){ /* parsing failed */ - clicon_err(OE_YANG, errno, "Parsing CV: %s", reason); - goto done; - } - break; - } - default: - break; - } - retval = 0; - done: - if (extra) - free(extra); - return retval; -} - /*! Return if this leaf is mandatory or not * Note: one can cache this value in ys_cvec instead of functionally evaluating it. * @retval 1 yang statement is leaf and it has a mandatory sub-stmt with value true @@ -3772,3 +2444,204 @@ yang_key_match(yang_stmt *yn, return retval; } +/*! Set type cache for yang type + * @param[in] rxmode Kludge to know which regexp engine is used + * @see yang_type_cache_regexp_set where cache is extended w compiled regexps + */ +int +yang_type_cache_set(yang_stmt *ys, + yang_stmt *resolved, + int options, + cvec *cvv, + cvec *patterns, + uint8_t fraction) +{ + int retval = -1; + yang_type_cache *ycache; + + if (ys->ys_typecache != NULL){ + clicon_err(OE_YANG, EEXIST, "yang type cache"); + goto done; + } + if ((ys->ys_typecache = (yang_type_cache *)malloc(sizeof(*ycache))) == NULL){ + clicon_err(OE_UNIX, errno, "malloc"); + goto done; + } + ycache = ys->ys_typecache; + memset(ycache, 0, sizeof(*ycache)); + ycache->yc_resolved = resolved; + ycache->yc_options = options; + if (cvv){ + if ((ycache->yc_cvv = cvec_dup(cvv)) == NULL){ + clicon_err(OE_UNIX, errno, "cvec_dup"); + goto done; + } + } + if (patterns && (ycache->yc_patterns = cvec_dup(patterns)) == NULL){ + clicon_err(OE_UNIX, errno, "cvec_dup"); + goto done; + } + ycache->yc_fraction = fraction; + retval = 0; + done: + return retval; +} + +/*! Extend yang type cache with compiled regexps + * Compiled Regexps are computed in validate code - after initial cache set + * @param[in] regexps + */ +int +yang_type_cache_regexp_set(yang_stmt *ytype, + int rxmode, + cvec *regexps) +{ + int retval = -1; + yang_type_cache *ycache; + + assert(regexps); + assert(yang_keyword_get(ytype) == Y_TYPE); + assert((ycache = ytype->ys_typecache) != NULL); + assert(ycache->yc_regexps == NULL); + ycache->yc_rxmode = rxmode; + if ((ycache->yc_regexps = cvec_dup(regexps)) == NULL){ + clicon_err(OE_UNIX, errno, "cvec_dup"); + goto done; + } + retval = 0; + done: + return retval; +} + +/*! Get individual fields (direct/destructively) from yang type cache. + * @param[out] patterns Initialized cvec of regexp patterns strings + * @retval -1 Error + * @retval 0 No cache + * @retval 1 OK + */ +int +yang_type_cache_get(yang_stmt *ytype, + yang_stmt **resolved, + int *options, + cvec **cvv, + cvec *patterns, + int *rxmode, + cvec *regexps, + uint8_t *fraction) +{ + int retval = -1; + cg_var *cv = NULL; + yang_type_cache *ycache; + + ycache = ytype->ys_typecache; + if (ycache == NULL){ /* No cache return 0 */ + retval = 0; + goto done; + } + if (resolved) + *resolved = ycache->yc_resolved; + if (options) + *options = ycache->yc_options; + if (cvv) + *cvv = ycache->yc_cvv; + if (patterns){ + cv = NULL; + while ((cv = cvec_each(ycache->yc_patterns, cv)) != NULL) + cvec_append_var(patterns, cv); + } + if (regexps){ + cv = NULL; + while ((cv = cvec_each(ycache->yc_regexps, cv)) != NULL) + cvec_append_var(regexps, cv); + } + if (rxmode) + *rxmode = ycache->yc_rxmode; + if (fraction) + *fraction = ycache->yc_fraction; + retval = 1; /* cache exists and is returned OK */ + done: + return retval; +} + +/*! Copy yang type cache + */ +static int +yang_type_cache_cp(yang_stmt *ynew, + yang_stmt *yold) +{ + int retval = -1; + int options; + cvec *cvv; + cvec *patterns = NULL; + uint8_t fraction; + yang_stmt *resolved; + int ret; + + if ((patterns = cvec_new(0)) == NULL){ + clicon_err(OE_UNIX, errno, "cvec_new"); + goto done; + } + /* Note, regexps are not copied since they are voids, if they were, they + * could not be freed in a simple way since copies are made at augment/group + */ + if ((ret = yang_type_cache_get(yold, + &resolved, &options, &cvv, patterns, NULL, NULL, &fraction)) < 0) + goto done; + if (ret == 1 && + yang_type_cache_set(ynew, resolved, options, cvv, patterns, fraction) < 0) + goto done; + retval = 0; + done: + if (patterns) + cvec_free(patterns); + return retval; +} + +/*! Free yang type cache + */ +static int +yang_type_cache_free(yang_type_cache *ycache) +{ + cg_var *cv; + void *p; + + if (ycache->yc_cvv) + cvec_free(ycache->yc_cvv); + if (ycache->yc_patterns) + cvec_free(ycache->yc_patterns); + if (ycache->yc_regexps){ + cv = NULL; + while ((cv = cvec_each(ycache->yc_regexps, cv)) != NULL){ + /* need to store mode since clicon_handle is not available */ + switch (ycache->yc_rxmode){ + case REGEXP_POSIX: + cligen_regex_posix_free(cv_void_get(cv)); + break; + case REGEXP_LIBXML2: + cligen_regex_libxml2_free(cv_void_get(cv)); + break; + default: + break; + } + if ((p = cv_void_get(cv)) != NULL){ + free(p); + cv_void_set(cv, NULL); + } + } + cvec_free(ycache->yc_regexps); + } + free(ycache); + return 0; +} + +#ifdef XML_EXTRA_INDEX +/*! Mark element as index in list + */ +int +yang_list_index_add(yang_stmt *yi) +{ + assert(yang_parent_get(yi) && yang_keyword_get(yang_parent_get(yi)) == Y_LIST); + yi->ys_flags |= YANG_FLAG_INDEX; + return 0; +} +#endif /* XML_EXTRA_INDEX */ diff --git a/lib/src/clixon_yang_cardinality.c b/lib/src/clixon_yang_cardinality.c index 7db189d4..24e1f86b 100644 --- a/lib/src/clixon_yang_cardinality.c +++ b/lib/src/clixon_yang_cardinality.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -63,7 +63,6 @@ #include "clixon_handle.h" #include "clixon_err.h" #include "clixon_yang.h" -#include "clixon_yang_internal.h" /* internal */ #include "clixon_yang_cardinality.h" /* @@ -545,8 +544,8 @@ yang_cardinality(clicon_handle h, /* 4) Recurse */ i = 0; - while (iys_len){ /* Note, children may be removed cant use yn_each */ - ys = yt->ys_stmt[i++]; + while (i< yang_len_get(yt)){ /* Note, children may be removed cant use yn_each */ + ys = yang_child_i(yt, i++); if (yang_cardinality(h, ys, modname) < 0) goto done; } diff --git a/lib/src/clixon_yang_internal.h b/lib/src/clixon_yang_internal.h index ffa6cc53..44753f06 100644 --- a/lib/src/clixon_yang_internal.h +++ b/lib/src/clixon_yang_internal.h @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -31,24 +31,15 @@ ***** END LICENSE BLOCK ***** - * Yang functions - * @see https://tools.ietf.org/html/rfc6020 YANG 1.0 - * @see https://tools.ietf.org/html/rfc7950 YANG 1.1 - */ + * + * This file defines the internal YANG data structures used by the Clixon implementation + * This file SHOULD ONLY be included by clixon_yang.h. + * Accesses should be made via the API defined in clixon_yang.h + */ #ifndef _CLIXON_YANG_INTERNAL_H_ #define _CLIXON_YANG_INTERNAL_H_ -/* - * Clixon-specific cligen variable (cv) flags - * CLIgen flags defined are in the range 0x01 -0x0f - * An application can use any flags above that - * @see cv_flag - */ -#define V_UNSET 0x10 /* Used by XML code to denote a value is not default */ - -#define YANG_FLAG_MARK 0x01 /* Marker for dynamic algorithms, eg expand */ - /*! Yang type cache. Yang type statements can cache all typedef info here * @note unions not cached */ @@ -76,7 +67,7 @@ struct yang_stmt{ enum rfc_6020 ys_keyword; /* See clicon_yang_parse.tab.h */ char *ys_argument; /* String / argument depending on keyword */ - int ys_flags; /* Flags according to YANG_FLAG_* above */ + uint16_t ys_flags; /* Flags according to YANG_FLAG_* */ yang_stmt *ys_mymodule; /* Shortcut to "my" module. Augmented nodes can belong to other modules than the ancestor module */ @@ -116,3 +107,4 @@ struct yang_stmt{ #define yang_schemanode(y) (yang_datanode(y) || (y)->ys_keyword == Y_RPC || (y)->ys_keyword == Y_CHOICE || (y)->ys_keyword == Y_CASE || (y)->ys_keyword == Y_INPUT || (y)->ys_keyword == Y_OUTPUT || (y)->ys_keyword == Y_NOTIFICATION) #endif /* _CLIXON_YANG_INTERNAL_H_ */ + diff --git a/lib/src/clixon_yang_module.c b/lib/src/clixon_yang_module.c index 4210fa6a..efada448 100644 --- a/lib/src/clixon_yang_module.c +++ b/lib/src/clixon_yang_module.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -77,8 +77,8 @@ #include "clixon_plugin.h" #include "clixon_netconf_lib.h" #include "clixon_xml_map.h" +#include "clixon_yang_parse_lib.h" #include "clixon_yang_module.h" -#include "clixon_yang_internal.h" /* internal */ modstate_diff_t * modstate_diff_new(void) @@ -155,7 +155,7 @@ yang_modules_revision(clicon_handle h) if ((ymod = yang_find(yspec, Y_MODULE, "ietf-yang-library")) != NULL || (ymod = yang_find(yspec, Y_SUBMODULE, "ietf-yang-library")) != NULL){ if ((yrev = yang_find(ymod, Y_REVISION, NULL)) != NULL){ - revision = yrev->ys_argument; + revision = yang_argument_get(yrev); } } return revision; @@ -189,25 +189,25 @@ yms_build(clicon_handle h, goto done; } - cprintf(cb,"", yns->ys_argument); + cprintf(cb,"", yang_argument_get(yns)); cprintf(cb,"%s", msid); ymod = NULL; while ((ymod = yn_each(yspec, ymod)) != NULL) { - if (ymod->ys_keyword != Y_MODULE && - ymod->ys_keyword != Y_SUBMODULE) + if (yang_keyword_get(ymod) != Y_MODULE && + yang_keyword_get(ymod) != Y_SUBMODULE) continue; cprintf(cb,""); - cprintf(cb,"%s", ymod->ys_argument); + cprintf(cb,"%s", yang_argument_get(ymod)); if ((ys = yang_find(ymod, Y_REVISION, NULL)) != NULL) - cprintf(cb,"%s", ys->ys_argument); + cprintf(cb,"%s", yang_argument_get(ys)); else{ /* RFC7895 1 If no (such) revision statement exists, the module's or submodule's revision is the zero-length string. */ cprintf(cb,""); } if ((ys = yang_find(ymod, Y_NAMESPACE, NULL)) != NULL) - cprintf(cb,"%s", ys->ys_argument); + cprintf(cb,"%s", yang_argument_get(ys)); else cprintf(cb,""); /* This follows order in rfc 7895: feature, conformance-type, @@ -215,10 +215,10 @@ yms_build(clicon_handle h, if (!brief){ yc = NULL; while ((yc = yn_each(ymod, yc)) != NULL) { - switch(yc->ys_keyword){ + switch(yang_keyword_get(yc)){ case Y_FEATURE: - if (yc->ys_cv && cv_bool_get(yc->ys_cv)) - cprintf(cb,"%s", yc->ys_argument); + if (yang_cv_get(yc) && cv_bool_get(yang_cv_get(yc))) + cprintf(cb,"%s", yang_argument_get(yc)); break; default: break; @@ -228,12 +228,12 @@ yms_build(clicon_handle h, } yc = NULL; while ((yc = yn_each(ymod, yc)) != NULL) { - switch(yc->ys_keyword){ + switch(yang_keyword_get(yc)){ case Y_SUBMODULE: cprintf(cb,""); - cprintf(cb,"%s", yc->ys_argument); + cprintf(cb,"%s", yang_argument_get(yc)); if ((ys = yang_find(yc, Y_REVISION, NULL)) != NULL) - cprintf(cb,"%s", ys->ys_argument); + cprintf(cb,"%s", yang_argument_get(ys)); else cprintf(cb,""); cprintf(cb,""); @@ -413,7 +413,7 @@ mod_ns_upgrade(clicon_handle h, goto fail; if ((yrev = yang_find(ymod, Y_REVISION, NULL)) == NULL) goto fail; - if (ys_parse_date_arg(yrev->ys_argument, &to) < 0) + if (ys_parse_date_arg(yang_argument_get(yrev), &to) < 0) goto done; } if ((ret = upgrade_callback_call(h, xt, ns, from, to, cbret)) < 0) @@ -482,3 +482,125 @@ clixon_module_upgrade(clicon_handle h, retval = 0; goto done; } + +/*! Given a yang statement and a prefix, return yang module to that relative prefix + * Note, not the other module but the proxy import statement only + * @param[in] ys A yang statement + * @param[in] prefix prefix + * @retval ymod Yang module statement if found + * @retval NULL not found + * @note Prefixes are relative to the module they are defined + * @see yang_find_module_by_name + * @see yang_find_module_by_namespace + */ +yang_stmt * +yang_find_module_by_prefix(yang_stmt *ys, + char *prefix) +{ + yang_stmt *yimport; + yang_stmt *yprefix; + yang_stmt *my_ymod; + yang_stmt *ymod = NULL; + yang_stmt *yspec; + char *myprefix; + + if ((yspec = ys_spec(ys)) == NULL){ + clicon_err(OE_YANG, 0, "My yang spec not found"); + goto done; + } + if ((my_ymod = ys_module(ys)) == NULL){ + clicon_err(OE_YANG, 0, "My yang module not found"); + goto done; + } + myprefix = yang_find_myprefix(ys); + if (myprefix && strcmp(myprefix, prefix) == 0){ + ymod = my_ymod; + goto done; + } + yimport = NULL; + while ((yimport = yn_each(my_ymod, yimport)) != NULL) { + if (yang_keyword_get(yimport) != Y_IMPORT) + continue; + if ((yprefix = yang_find(yimport, Y_PREFIX, NULL)) != NULL && + strcmp(yang_argument_get(yprefix), prefix) == 0){ + break; + } + } + if (yimport){ + if ((ymod = yang_find(yspec, Y_MODULE, yang_argument_get(yimport))) == NULL){ + clicon_err(OE_YANG, 0, "No module or sub-module found with prefix %s", + prefix); + yimport = NULL; + goto done; /* unresolved */ + } + } + done: + return ymod; +} + +/* Get module from its own prefix + * This is really not a valid usecase, a kludge for the identityref derived + * list workaround (IDENTITYREF_KLUDGE) + * Actually, for canonical prefixes it is! + */ +yang_stmt * +yang_find_module_by_prefix_yspec(yang_stmt *yspec, + char *prefix) +{ + yang_stmt *ymod = NULL; + yang_stmt *yprefix; + + while ((ymod = yn_each(yspec, ymod)) != NULL) + if (yang_keyword_get(ymod) == Y_MODULE && + (yprefix = yang_find(ymod, Y_PREFIX, NULL)) != NULL && + strcmp(yang_argument_get(yprefix), prefix) == 0) + return ymod; + return NULL; +} + +/*! Given a yang spec and a namespace, return yang module + * + * @param[in] yspec A yang specification + * @param[in] namespace namespace + * @retval ymod Yang module statement if found + * @retval NULL not found + * @see yang_find_module_by_name + * @see yang_find_module_by_prefix module-specific prefix + */ +yang_stmt * +yang_find_module_by_namespace(yang_stmt *yspec, + char *namespace) +{ + yang_stmt *ymod = NULL; + + if (namespace == NULL) + goto done; + while ((ymod = yn_each(yspec, ymod)) != NULL) { + if (yang_find(ymod, Y_NAMESPACE, namespace) != NULL) + break; + } + done: + return ymod; +} + +/*! Given a yang spec and a module name, return yang module or submodule + * + * @param[in] yspec A yang specification + * @param[in] name Name of module + * @retval ymod Yang module statement if found + * @retval NULL not found + * @see yang_find_module_by_namespace + * @see yang_find_module_by_prefix module-specific prefix + */ +yang_stmt * +yang_find_module_by_name(yang_stmt *yspec, + char *name) +{ + yang_stmt *ymod = NULL; + + while ((ymod = yn_each(yspec, ymod)) != NULL) + if ((yang_keyword_get(ymod) == Y_MODULE || yang_keyword_get(ymod) == Y_SUBMODULE) && + strcmp(yang_argument_get(ymod), name)==0) + return ymod; + return NULL; +} diff --git a/lib/src/clixon_yang_parse.y b/lib/src/clixon_yang_parse.y index e363ff69..047ca849 100644 --- a/lib/src/clixon_yang_parse.y +++ b/lib/src/clixon_yang_parse.y @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -51,7 +51,6 @@ %start file %union { - int intval; char *string; } @@ -190,8 +189,8 @@ #include "clixon_err.h" #include "clixon_log.h" #include "clixon_yang.h" +#include "clixon_yang_parse_lib.h" #include "clixon_yang_parse.h" -#include "clixon_yang_internal.h" /* internal */ extern int clixon_yang_parseget_lineno (void); @@ -300,7 +299,7 @@ ysp_add(struct clicon_yang_yacc_arg *yy, if ((ys = ys_new(keyword)) == NULL) goto err; /* NOTE: does not make a copy of string, ie argument is 'consumed' here */ - ys->ys_argument = argument; + yang_argument_set(ys, argument); if (yn_insert(yn, ys) < 0) /* Insert into hierarchy */ goto err; if (ys_parse_sub(ys, extra) < 0) /* Check statement-specific syntax */ diff --git a/lib/src/clixon_yang_parse_lib.c b/lib/src/clixon_yang_parse_lib.c new file mode 100644 index 00000000..708ab5b8 --- /dev/null +++ b/lib/src/clixon_yang_parse_lib.c @@ -0,0 +1,1360 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2020 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * Yang functions + * @see https://tools.ietf.org/html/rfc6020 YANG 1.0 + * @see https://tools.ietf.org/html/rfc7950 YANG 1.1 + * + * CALLING ORDER OF YANG PARSE FILES + * ================================= + * yang_spec_parse_module + * | | + * v v v + * yang_spec_parse_file-> yang_parse_post->yang_parse_recurse->yang_parse_module + * \ / v + * yang_spec_load_dir ------------------------------------> yang_parse_filename + * v + * yang_parse_file + * v + * yang_parse_str + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* cligen */ +#include + +/* clicon */ +#include "clixon_log.h" +#include "clixon_err.h" +#include "clixon_string.h" +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_file.h" +#include "clixon_yang.h" +#include "clixon_yang_internal.h" +#include "clixon_hash.h" +#include "clixon_xml.h" +#include "clixon_plugin.h" +#include "clixon_data.h" +#include "clixon_options.h" +#include "clixon_yang_type.h" +#include "clixon_yang_parse.h" +#include "clixon_yang_cardinality.h" +#include "clixon_yang_module.h" +#include "clixon_yang_parse_lib.h" + +/* Size of json read buffer when reading from file*/ +#define BUFLEN 1024 + +/*! Resolve a grouping name from a point in the yang tree + * @param[in] ys Yang statement of "uses" statement doing the lookup + * @param[in] prefix Prefix of grouping to look for + * @param[in] name Name of grouping to look for + * @param[out] ygrouping0 A found grouping yang structure as result + * @retval 0 OK, ygrouping may be NULL + * @retval -1 Error, with clicon_err called + */ +static int +ys_grouping_resolve(yang_stmt *ys, + char *prefix, + char *name, + yang_stmt **ygrouping0) +{ + int retval = -1; + yang_stmt *ymodule; + yang_stmt *ygrouping = NULL; + yang_stmt *yn; + + /* find the grouping associated with argument and expand(?) */ + if (prefix){ /* Go to top and find import that matches */ + if ((ymodule = yang_find_module_by_prefix(ys, prefix)) != NULL) + ygrouping = yang_find(ymodule, Y_GROUPING, name); + } + else + while (1){ + /* Check upwards in hierarchy for matching groupings */ + if ((yn = yang_parent_get(ys)) == NULL || yang_keyword_get(yn) == Y_SPEC) + break; + /* Here find grouping */ + if ((ygrouping = yang_find(yn, Y_GROUPING, name)) != NULL) + break; + /* Proceed to next level */ + ys = (yang_stmt*)yn; + } + *ygrouping0 = ygrouping; + retval = 0; + // done: + return retval; +} + +/*! This is an augment node, augment the original datamodel. + * @param[in] ys The augment statement + * @param[in] yspec Yang specification + * @see RFC7950 Sec 7.17 + * The target node MUST be either a container, list, choice, case, input, + * output, or notification node. + * If the "augment" statement is on the top level the absolute form MUST be + * used. + * All data nodes defined in the "augment" statement are defined as XML + * elements in the XML namespace of the module where the "augment" is + * specified. + */ +static int +yang_augment_node(yang_stmt *ys, + yang_stmt *ysp) +{ + int retval = -1; + char *schema_nodeid; + yang_stmt *ytarget = NULL; + yang_stmt *yc; + int i; + yang_stmt *ymod; + + if ((ymod = ys_module(ys)) == NULL){ + clicon_err(OE_YANG, 0, "My yang module not found"); + goto done; + } + schema_nodeid = yang_argument_get(ys); + clicon_debug(2, "%s %s", __FUNCTION__, schema_nodeid); + /* Find the target */ + if (yang_abs_schema_nodeid(ysp, ys, schema_nodeid, -1, &ytarget) < 0) + goto done; + if (ytarget == NULL) + goto ok; + /* Extend ytarget with ys' children + * First enlarge ytarget vector + */ + for (i=0; iys_len; i++){ + if ((yc = ys_dup(ys->ys_stmt[i])) == NULL) + goto done; + yc->ys_mymodule = ymod; + if (yn_insert(ytarget, yc) < 0) + goto done; + } + ok: + retval = 0; + done: + return retval; +} + +/*! Find all top-level augments and change original datamodels. */ +static int +yang_augment_spec(yang_stmt *ysp, + int modnr) +{ + int retval = -1; + yang_stmt *ym; + yang_stmt *ys; + int i; + int j; + + i = modnr; + while (iys_len){ /* Loop through modules and sub-modules */ + ym = ysp->ys_stmt[i++]; + j = 0; + while (jys_len){ /* Top-level symbols in modules */ + ys = ym->ys_stmt[j++]; + switch (yang_keyword_get(ys)){ + case Y_AUGMENT: /* top-level */ + if (yang_augment_node(ys, ysp) < 0) + goto done; + break; + default: + break; + } + } + } + retval = 0; + done: + return retval; +} + +/*! Given a refine node, perform the refinement action on the target refine node + * The RFC is somewhat complicate in the rules for refine. + * Most nodes will be replaced, but some are added + * @param[in] yr Refine node + * @param[in] yt Refine target node (will be modified) + * @see RFC7950 Sec 7.13.2 + * There may be some missed cornercases + */ +static int +ys_do_refine(yang_stmt *yr, + yang_stmt *yt) +{ + int retval = -1; + yang_stmt *yrc; /* refine child */ + yang_stmt *yrc1; + yang_stmt *ytc; /* target child */ + enum rfc_6020 keyw; + int i; + + /* Loop through refine node children. First if remove do that first + * In some cases remove a set of nodes. + */ + yrc = NULL; + while ((yrc = yn_each(yr, yrc)) != NULL) { + keyw = yang_keyword_get(yrc); + switch (keyw){ + case Y_DEFAULT: /* remove old, add new */ + case Y_DESCRIPTION: + case Y_REFERENCE: + case Y_CONFIG: + case Y_MANDATORY: + case Y_PRESENCE: + case Y_MIN_ELEMENTS: + case Y_MAX_ELEMENTS: + case Y_EXTENSION: + /* Remove old matching, dont increment due to prune in loop */ + for (i=0; iys_len; ){ + ytc = yt->ys_stmt[i]; + if (keyw != yang_keyword_get(ytc)){ + i++; + continue; + } + ys_prune(yt, i); + ys_free(ytc); + } + /* fall through and add if not found */ + case Y_MUST: /* keep old, add new */ + case Y_IF_FEATURE: + break; + default: + break; + } + } + /* Second, add the node(s) */ + yrc = NULL; + while ((yrc = yn_each(yr, yrc)) != NULL) { + keyw = yang_keyword_get(yrc); + /* Make copy */ + if ((yrc1 = ys_dup(yrc)) == NULL) + goto done; + if (yn_insert(yt, yrc1) < 0) + goto done; + } + retval = 0; + done: + return retval; +} + +/*! Macro expansion of grouping/uses done in step 2 of yang parsing + * RFC7950: + * Identifiers appearing inside the grouping are resolved + * relative to the scope in which the grouping is defined, not where it is + * used. Prefix mappings, type names, grouping names, and extension usage are + * evaluated in the hierarchy where the "grouping" statement appears. + * The identifiers defined in the grouping are not bound to a namespace + * until the contents of the grouping are added to the schema tree via a + * "uses" statement that does not appear inside a "grouping" statement, + * at which point they are bound to the namespace of the current module. + */ +static int +yang_expand_grouping(yang_stmt *yn) +{ + int retval = -1; + yang_stmt *ys = NULL; + yang_stmt *ygrouping; /* grouping original */ + yang_stmt *ygrouping2; /* grouping copy */ + yang_stmt *yg; /* grouping child */ + yang_stmt *yr; /* refinement */ + int glen; + int i; + int j; + char *id = NULL; + char *prefix = NULL; + size_t size; + + /* Cannot use yang_apply here since child-list is modified (is destructive) */ + i = 0; + while (iys_len){ + ys = yn->ys_stmt[i]; + switch(yang_keyword_get(ys)){ + case Y_USES: + /* Split argument into prefix and name */ + if (nodeid_split(yang_argument_get(ys), &prefix, &id) < 0) + goto done; + if (ys_grouping_resolve(ys, prefix, id, &ygrouping) < 0) + goto done; + if (prefix){ + free(prefix); + prefix = NULL; + } + if (id){ + free(id); + id = NULL; + } + if (ygrouping == NULL){ + clicon_log(LOG_NOTICE, "%s: Yang error : grouping \"%s\" not found in module \"%s\"", + __FUNCTION__, yang_argument_get(ys), yang_argument_get(ys_module(ys))); + goto done; + break; + } + /* Check mark flag to see if this grouping (itself) has been expanded + If not, this needs to be done before we can insert it into + the 'uses' place */ + if (yang_flag_get(ygrouping, YANG_FLAG_MARK) == 0){ + if (yang_expand_grouping(ygrouping) < 0) + goto done; + yang_flag_set(ygrouping, YANG_FLAG_MARK); /* Mark as expanded */ + } + /* Make a copy of the grouping, then make refinements to this copy + */ + if ((ygrouping2 = ys_dup(ygrouping)) == NULL) + goto done; + /* Replace ys with ygrouping,... + * First enlarge parent vector + */ + glen = ygrouping2->ys_len; + /* + * yn is parent: the children of ygrouping replaces ys. + * Is there a case when glen == 0? YES AND THIS BREAKS + */ + if (glen != 1){ + size = (yn->ys_len - i - 1)*sizeof(struct yang_stmt *); + yn->ys_len += glen - 1; + if (glen && (yn->ys_stmt = realloc(yn->ys_stmt, (yn->ys_len)*sizeof(yang_stmt *))) == 0){ + clicon_err(OE_YANG, errno, "realloc"); + goto done; + } + /* Then move all existing elements up from i+1 (not uses-stmt) */ + if (size) + memmove(&yn->ys_stmt[i+glen], + &yn->ys_stmt[i+1], + size); + } + + /* Iterate through refinements and modify grouping copy + * See RFC 7950 7.13.2 yrt is the refine target node + */ + yr = NULL; + while ((yr = yn_each(ys, yr)) != NULL) { + yang_stmt *yrt; /* refine target node */ + if (yang_keyword_get(yr) != Y_REFINE) + continue; + /* Find a node */ + if (yang_desc_schema_nodeid(ygrouping2, + yang_argument_get(yr), + -1, + &yrt) < 0) + goto done; + /* Not found, try next */ + if (yrt == NULL) + continue; + /* Do the actual refinement */ + if (ys_do_refine(yr, yrt) < 0) + goto done; + /* RFC: The argument is a string that identifies a node in the + * grouping. I interpret that as only one node --> break */ + break; + } + /* Then copy and insert each child element */ + for (j=0; jys_stmt[j]; /* Child of refined copy */ + yn->ys_stmt[i+j] = yg; + yg->ys_parent = yn; + } + /* Remove 'uses' node */ + ys_free(ys); + /* Remove the grouping copy */ + ygrouping2->ys_len = 0; + ys_free(ygrouping2); + break; /* Note same child is re-iterated since it may be changed */ + default: + i++; + break; + } + } + /* Second pass since length may have changed */ + for (i=0; iys_len; i++){ + ys = yn->ys_stmt[i]; + if (yang_expand_grouping(ys) < 0) + goto done; + } + retval = 0; + done: + if (prefix) + free(prefix); + if (id) + free(id); + return retval; +} + +/*! Parse a string containing a YANG spec into a parse-tree + * + * Syntax parsing. A string is input and a syntax-tree is returned (or error). + * A variable record is also returned containing a list of (global) variable values. + * (cloned from cligen) + * @param[in] str String of yang statements + * @param[in] name Log string, typically filename + * @param[in] yspec Yang specification. + * @retval ymod Top-level yang (sub)module + * @retval NULL Error encountered + * See top of file for diagram of calling order + */ +static yang_stmt * +yang_parse_str(char *str, + const char *name, /* just for errs */ + yang_stmt *yspec) +{ + struct clicon_yang_yacc_arg yy = {0,}; + yang_stmt *ymod = NULL; + + if (yspec == NULL){ + clicon_err(OE_YANG, 0, "Yang parse need top level yang spec"); + goto done; + } + yy.yy_name = (char*)name; + yy.yy_linenum = 1; + yy.yy_parse_string = str; + yy.yy_stack = NULL; + yy.yy_module = NULL; /* this is the return value - the module/sub-module */ + if (ystack_push(&yy, yspec) == NULL) + goto done; + if (strlen(str)){ /* Not empty */ + if (yang_scan_init(&yy) < 0) + goto done; + if (yang_parse_init(&yy) < 0) + goto done; + if (clixon_yang_parseparse(&yy) != 0) { /* yacc returns 1 on error */ + clicon_log(LOG_NOTICE, "Yang error: %s on line %d", name, yy.yy_linenum); + if (clicon_errno == 0) + clicon_err(OE_YANG, 0, "yang parser error with no error code (should not happen)"); + yang_parse_exit(&yy); + yang_scan_exit(&yy); + goto done; + } + if (yang_parse_exit(&yy) < 0) + goto done; + if (yang_scan_exit(&yy) < 0) + goto done; + } + ymod = yy.yy_module; + done: + ystack_pop(&yy); + if (yy.yy_stack) + free (yy.yy_stack); + return ymod; /* top-level (sub)module */ +} + +/*! Parse yang spec from an open file descriptor + * @param[in] fd File descriptor containing the YANG file as ASCII characters + * @param[in] name For debug, eg filename + * @param[in] ysp Yang specification. Should have been created by caller using yspec_new + * @retval ymod Top-level yang (sub)module + * @retval NULL Error + * @note this function simply parse a yang spec, no dependencies or checks + */ +yang_stmt * +yang_parse_file(int fd, + const char *name, + yang_stmt *ysp) +{ + char *buf = NULL; + int i; + int c; + int len; + yang_stmt *ymod = NULL; + int ret; + + len = BUFLEN; /* any number is fine */ + if ((buf = malloc(len)) == NULL){ + perror("pt_file malloc"); + return NULL; + } + memset(buf, 0, len); + i = 0; /* position in buf */ + while (1){ /* read the whole file */ + if ((ret = read(fd, &c, 1)) < 0){ + clicon_err(OE_XML, errno, "read"); + break; + } + if (ret == 0) + break; /* eof */ + if (len==i){ + if ((buf = realloc(buf, 2*len)) == NULL){ + clicon_err(OE_XML, errno, "realloc"); + goto done; + } + memset(buf+len, 0, len); + len *= 2; + } + buf[i++] = (char)(c&0xff); + } /* read a line */ + if ((ymod = yang_parse_str(buf, name, ysp)) < 0) + goto done; + done: + if (buf) + free(buf); + return ymod; /* top-level (sub)module */ +} + +/*! Given a yang filename, extract the revision as an integer as YYYYMMDD + * @param[in] filename Filename on the form: name [+ @rev ] + .yang + * @param[out] basep "Base" filename, stripped: [+ @rev ] + .yang + * @param[out] revp Revision as YYYYMMDD (0 if not found) + */ +static int +filename2revision(const char *filename, + char **basep, + uint32_t *revp) +{ + int retval = -1; + char *base = NULL; + char *p; + + /* base = module name [+ @rev ] + .yang */ + if ((base = strdup(filename)) == NULL){ + clicon_err(OE_UNIX, errno, "strdup"); + goto done; + } + clicon_debug(1, "%s %s", __FUNCTION__, base); + if ((p = rindex(base, '.')) != NULL) /* strip postfix .yang */ + *p = '\0'; + if ((p = index(base, '@')) != NULL){ /* extract revision date */ + *p++ = '\0'; + if (ys_parse_date_arg(p, revp) < 0) + goto done; + } + if (basep){ + *basep = base; + base = NULL; + } + retval = 0; + done: + if (base) + free(base); + return retval; +} + +/*! No specific revision give. Match a yang file given module + * @param[in] h CLICON handle + * @param[in] module Name of main YANG module. + * @param[in] revision Revision or NULL + * @param[out] revactual Actual revision (if retval=1) + * @param[out] fbuf Buffer containing filename (if retval=1) + * @retval 1 Match found, Most recent entry returned in fbuf and revactual + * @retval 0 No matching entry found + * @retval -1 Error + * @note for bootstrapping, dir may have to be set. +*/ +static int +yang_parse_find_match(clicon_handle h, + const char *module, + const char *revision, + uint32_t *revactual, + cbuf *fbuf) +{ + int retval = -1; + struct dirent *dp = NULL; + int ndp; + cbuf *regex = NULL; + cxobj *x; + cxobj *xc; + char *dir; + + /* get clicon config file in xml form */ + if ((x = clicon_conf_xml(h)) == NULL) + goto ok; + if ((regex = cbuf_new()) == NULL){ + clicon_err(OE_YANG, errno, "cbuf_new"); + goto done; + } + /* RFC 6020: The name of the file SHOULD be of the form: + * module-or-submodule-name ['@' revision-date] ( '.yang' / '.yin' ) + * revision-date ::= 4DIGIT "-" 2DIGIT "-" 2DIGIT + */ + if (revision) + cprintf(regex, "^%s@%s(.yang)$", module, revision); + else + cprintf(regex, "^%s(@[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])?(.yang)$", + module); + xc = NULL; + while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) { + if (strcmp(xml_name(xc), "CLICON_YANG_DIR") != 0) + continue; + dir = xml_body(xc); + /* get all matching files in this directory */ + if ((ndp = clicon_file_dirent(dir, + &dp, + cbuf_get(regex), + S_IFREG)) < 0) + goto done; + /* Entries are sorted, last entry should be most recent date + */ + if (ndp != 0){ + cprintf(fbuf, "%s/%s", dir, dp[ndp-1].d_name); + retval = 1; + goto done; + } + } + ok: + retval = 0; + done: + if (regex) + cbuf_free(regex); + if (dp) + free(dp); + return retval; +} + +/*! Open a file, read into a string and invoke yang parsing + * + * Similar to clicon_yang_str(), just read a file first + * @param[in] filename Name of file + * @param[in] ysp Yang specification. Should have been created by caller using yspec_new + * @retval ymod Top-level yang (sub)module + * @retval NULL Error encountered + + * The database symbols are inserted in alphabetical order. + * See top of file for diagram of calling order + */ +yang_stmt * +yang_parse_filename(const char *filename, + yang_stmt *ysp) +{ + yang_stmt *ymod = NULL; + int fd = -1; + struct stat st; + + clicon_debug(1, "%s %s", __FUNCTION__, filename); + if (stat(filename, &st) < 0){ + clicon_err(OE_YANG, errno, "%s not found", filename); + goto done; + } + if ((fd = open(filename, O_RDONLY)) < 0){ + clicon_err(OE_YANG, errno, "open(%s)", filename); + goto done; + } + if ((ymod = yang_parse_file(fd, filename, ysp)) < 0) + goto done; + done: + if (fd != -1) + close(fd); + return ymod; /* top-level (sub)module */ +} + +/*! Given a (sub)module, parse all (sub)modules in turn recursively + * + * Find a yang module file, and then recursively parse all its imported modules. + * @param[in] h CLICON handle + * @param[in] module Module name + * @param[in] revision Revision (or NULL) + * @param[in] ysp Yang statement + * @retval 0 OK + * @retval -1 Error + * + * See top of file for diagram of calling order + */ +static yang_stmt * +yang_parse_module(clicon_handle h, + const char *module, + const char *revision, + yang_stmt *ysp) +{ + cbuf *fbuf = NULL; + char *filename; + int nr; + yang_stmt *ymod = NULL; + yang_stmt *yrev; /* yang revision */ + uint32_t revf = 0; /* revision in filename */ + uint32_t revm = 0; /* revision in parsed new module (should be same as revf) */ + + if ((fbuf = cbuf_new()) == NULL){ + clicon_err(OE_YANG, errno, "cbuf_new"); + goto done; + } + /* Match a yang file with or without revision in yang-dir list */ + if ((nr = yang_parse_find_match(h, module, revision, &revf, fbuf)) < 0) + goto done; + if (nr == 0){ + clicon_err(OE_YANG, errno, "No yang files found matching \"%s\" in the list of CLICON_YANG_DIRs", module); + goto done; + } + filename = cbuf_get(fbuf); + if ((ymod = yang_parse_filename(filename, ysp)) == NULL) + goto done; + if ((yrev = yang_find(ymod, Y_REVISION, NULL)) != NULL) + revm = cv_uint32_get(yang_cv_get(yrev)); + if (filename2revision(filename, NULL, &revf) < 0) + goto done; + /* Sanity check that file revision does not match internal rev stmt */ + if (revf && revm && revm != revf){ + clicon_err(OE_YANG, EINVAL, "Yang module file revision and in yang does not match: %s vs %u", filename, revm); + ymod = NULL; + goto done; + } + done: + if (fbuf) + cbuf_free(fbuf); + return ymod; /* top-level (sub)module */ +} + +/*! Given a (sub)module, parse all (sub)modules in turn recursively + * + * Find a yang module file, and then recursively parse all its imported modules. + * @param[in] h CLICON handle + * @param[in] ymod Yang module. + * @param[in] yspec Yang specification. + * @retval 0 OK + * @retval -1 Error + * + * See top of file for diagram of calling order + */ +static int +yang_parse_recurse(clicon_handle h, + yang_stmt *ymod, + yang_stmt *ysp) +{ + int retval = -1; + yang_stmt *yi = NULL; /* import */ + yang_stmt *yrev; + char *submodule; + char *subrevision; + yang_stmt *subymod; + enum rfc_6020 keyw; + + /* go through all import (modules) and include(submodules) of ysp */ + while ((yi = yn_each(ymod, yi)) != NULL){ + keyw = yang_keyword_get(yi); + if (keyw != Y_IMPORT && keyw != Y_INCLUDE) + continue; + /* common part */ + submodule = yang_argument_get(yi); + /* Is there a specific revision (or just latest)? */ + if ((yrev = yang_find(yi, Y_REVISION_DATE, NULL)) != NULL) + subrevision = yang_argument_get(yrev); + else + subrevision = NULL; + /* if already loaded, ignore, else parse the file */ + if (yang_find(ysp, + keyw==Y_IMPORT?Y_MODULE:Y_SUBMODULE, + submodule) == NULL){ + /* recursive call */ + if ((subymod = yang_parse_module(h, submodule, subrevision, ysp)) == NULL) + goto done; + /* Go through its sub-modules recursively */ + if (yang_parse_recurse(h, subymod, ysp) < 0){ + ymod = NULL; + goto done; + } + } + } + retval = 0; + done: + return retval; /* top-level (sub)module */ +} + +/*! + * @param[in] ys Yang statement + * @param[in] dummy Necessary for called in yang_apply + * @see yang_apply_fn + */ +static int +ys_schemanode_check(yang_stmt *ys, + void *dummy) +{ + int retval = -1; + yang_stmt *yspec; + yang_stmt *yres = NULL; + yang_stmt *yp; + char *arg; + enum rfc_6020 keyword; + char **vec = NULL; + char *v; + int nvec; + int i; + + yp = yang_parent_get(ys); + arg = yang_argument_get(ys); + keyword = yang_keyword_get(ys); + switch (yang_keyword_get(ys)){ + case Y_AUGMENT: + if (yang_keyword_get(yp) == Y_MODULE || /* Not top-level */ + yang_keyword_get(yp) == Y_SUBMODULE) + break; + /* fallthru */ + case Y_REFINE: + if (yang_desc_schema_nodeid(yp, arg, -1, &yres) < 0) + goto done; + if (yres == NULL){ + clicon_err(OE_YANG, 0, "schemanode sanity check of %s %s", + yang_key2str(keyword), arg); + goto done; + } + break; + case Y_UNIQUE:{ + /* Unique: Sec 7.8.3 It takes as an argument a string that contains a space- + separated list of schema node identifiers */ + if ((vec = clicon_strsep(arg, " \t\n", &nvec)) == NULL) + goto done; + for (i=0; iys_len; i++) + if (yang_parse_recurse(h, yspec->ys_stmt[i], yspec) < 0) + goto done; + + /* 2. Check cardinality maybe this should be done after grouping/augment */ + for (i=modnr; iys_len; i++) + if (yang_cardinality(h, yspec->ys_stmt[i], yang_argument_get(yspec->ys_stmt[i])) < 0) + goto done; + + /* 3: Check features: check if enabled and remove disabled features */ + for (i=modnr; iys_len; i++) /* XXX */ + if (yang_features(h, yspec->ys_stmt[i]) < 0) + goto done; + + /* 4: Go through parse tree and populate it with cv types */ + for (i=modnr; iys_len; i++) + if (yang_apply(yspec->ys_stmt[i], -1, ys_populate, (void*)h) < 0) + goto done; + + /* 5: Resolve all types: populate type caches. Requires eg length/range cvecs + * from ys_populate step. + * Must be done using static binding. + */ + for (i=modnr; iys_len; i++) + if (yang_apply(yspec->ys_stmt[i], Y_TYPE, ys_resolve_type, h) < 0) + goto done; + + /* Up to here resolving is made in the context they are defined, rather + * than the context they are used (except for submodules being merged w + * modules). Like static scoping. + * After this we expand all grouping/uses and unfold all macros into a + * single tree as they are used. + */ + + /* 6: Macro expansion of all grouping/uses pairs. Expansion needs marking */ + for (i=modnr; iys_len; i++){ + if (yang_expand_grouping(yspec->ys_stmt[i]) < 0) + goto done; + yang_apply(yspec->ys_stmt[i], -1, (yang_applyfn_t*)yang_flag_reset, (void*)YANG_FLAG_MARK); + } + + /* 7: Top-level augmentation of all modules. (Augment also in uses) */ + if (yang_augment_spec(yspec, modnr) < 0) + goto done; + + /* 4: Go through parse tree and do 2nd step populate (eg default) */ + for (i=modnr; iys_len; i++) + if (yang_apply(yspec->ys_stmt[i], -1, ys_populate2, (void*)h) < 0) + goto done; + + /* 8: sanity check of schemanode references, need more here */ + for (i=modnr; iys_len; i++) + if (yang_apply(yspec->ys_stmt[i], -1, ys_schemanode_check, NULL) < 0) + goto done; + retval = 0; + done: + return retval; +} + +/*! Parse yang specification and its dependencies recursively given module + * @param[in] h clicon handle + * @param[in] module Module name, or absolute filename (including dir) + * @param[in] revision Revision, or NULL + * @param[in] yspec Modules parse are added to this yangspec + * @retval 0 OK + * @retval -1 Error + * @see yang_spec_parse_file + */ +int +yang_spec_parse_module(clicon_handle h, + const char *module, + const char *revision, + yang_stmt *yspec) +{ + int retval = -1; + int modnr; /* Existing number of modules */ + char *base = NULL;; + + if (yspec == NULL){ + clicon_err(OE_YANG, EINVAL, "yang spec is NULL"); + goto done; + } + if (module == NULL){ + clicon_err(OE_YANG, EINVAL, "yang module not set"); + goto done; + } + /* Apply steps 2.. on new modules, ie ones after modnr. */ + modnr = yspec->ys_len; + /* Do not load module if it already exists */ + if (yang_find(yspec, Y_MODULE, module) != NULL) + goto ok; + if (yang_parse_module(h, module, revision, yspec) == NULL) + goto done; + if (yang_parse_post(h, yspec, modnr) < 0) + goto done; + ok: + retval = 0; + done: + if (base) + free(base); + return retval; +} + +/*! Parse yang specification and its dependencies recursively given filename + * @param[in] h clicon handle + * @param[in] filename Actual filename (including dir and revision) + * @param[in] yspec Modules parse are added to this yangspec + * @retval 0 OK + * @retval -1 Error + * @see yang_spec_parse_module for yang dir,module,revision instead of + * actual filename + * @see yang_spec_load_dir For loading all files in a directory + */ +int +yang_spec_parse_file(clicon_handle h, + char *filename, + yang_stmt *yspec) +{ + int retval = -1; + int modnr; /* Existing number of modules */ + char *base = NULL;; + + /* Apply steps 2.. on new modules, ie ones after modnr. */ + modnr = yspec->ys_len; + /* Find module, and do not load file if module already exists */ + if (basename(filename) == NULL){ + clicon_err(OE_YANG, errno, "No basename"); + goto done; + } + if ((base = strdup(basename(filename))) == NULL){ + clicon_err(OE_YANG, errno, "strdup"); + goto done; + } + if (index(base, '@') != NULL) + *index(base, '@') = '\0'; + if (yang_find(yspec, Y_MODULE, base) != NULL) + goto ok; + if (yang_parse_filename(filename, yspec) == NULL) + goto done; + if (yang_parse_post(h, yspec, modnr) < 0) + goto done; + ok: + retval = 0; + done: + if (base) + free(base); + return retval; +} + +/*! Load all yang modules in directory + * @param[in] h Clicon handle + * @param[in] dir Load all yang modules in this directory + * @param[in] yspec Modules parse are added to this yangspec + * @retval 0 OK + * @retval -1 Error + * @see yang_spec_parse_file + * Load all yang files in a directory as primary objects. + * Some details if several same yang module x exists: + * 1) If x is already loaded (eg via direct file loading) skip it + * 2) Prefer x.yang over x@rev.yang (no revision) + * 3) If only x@rev.yang's found, prefer newest (newest revision) + * There is also an extra failsafe which may not be necessary, which removes + * the oldest module if 1-3 for some reason fails. + */ +int +yang_spec_load_dir(clicon_handle h, + char *dir, + yang_stmt *yspec) +{ + int retval = -1; + int ndp; + struct dirent *dp = NULL; + int i; + int j; + char filename[MAXPATHLEN]; + char *base = NULL; /* filename without dir */ + int modnr; + yang_stmt *ym; /* yang module */ + yang_stmt *ym0; /* (existing) yang module */ + yang_stmt *yrev; /* yang revision */ + uint32_t revf = 0; /* revision in filename */ + uint32_t revm = 0; /* revision in parsed new module (should be same as revf) */ + uint32_t rev0; /* revision in existing module */ + char *oldbase = NULL; + int taken = 0; + + /* Get yang files names from yang module directory. Note that these + * are sorted alphatetically: + * a.yang, + * a@2000-01-01.yang, + * a@2111-11-11.yang + */ + if((ndp = clicon_file_dirent(dir, &dp, "(.yang)$", S_IFREG)) < 0) + goto done; + if (ndp == 0) + clicon_log(LOG_WARNING, "%s: No yang files found in %s", + __FUNCTION__, dir); + /* Apply post steps on new modules, ie ones after modnr. */ + modnr = yspec->ys_len; + /* Load all yang files in dir */ + for (i = 0; i < ndp; i++) { + /* base = module name [+ @rev ] + .yang */ + if (oldbase) + free(oldbase); + oldbase = base; + base = NULL; + revf = 0; + if (filename2revision(dp[i].d_name, &base, &revf) < 0) + goto done; + if (oldbase && strcmp(base, oldbase)) /* new yang file basename */ + taken = 0; + if (revf == 0) /* No revision: a.yang - take that */ + taken = 1; + else{ /* a@xxx.yang */ + if (taken) + continue; /* skip if already taken */ + /* Look forward: is there anyone else later? */ + if (i+1 ym0/rev0 */ + rev0 = 0; + if ((ym0 = yang_find(yspec, Y_MODULE, base)) != NULL || + (ym0 = yang_find(yspec, Y_SUBMODULE, base)) != NULL){ + yrev = yang_find(ym0, Y_REVISION, NULL); + rev0 = cv_uint32_get(yang_cv_get(yrev)); + continue; /* skip if already added by specific file or module */ + } + /* Create full filename */ + snprintf(filename, MAXPATHLEN-1, "%s/%s", dir, dp[i].d_name); + if ((ym = yang_parse_filename(filename, yspec)) == NULL) + goto done; + revm = 0; + if ((yrev = yang_find(ym, Y_REVISION, NULL)) != NULL) + revm = cv_uint32_get(yang_cv_get(yrev)); + /* Sanity check that file revision does not match internal rev stmt */ + if (revf && revm && revm != revf){ /* XXX */ + clicon_err(OE_YANG, EINVAL, "Yang module file revision and in yang does not match: %s(%u) vs %u", filename, revf, revm); + goto done; + } + /* If ym0 and ym exists, delete the yang with oldest revision + * This is a failsafe in case anything else fails + */ + if (revm && rev0){ + if (revm > rev0) /* Loaded module is older or eq -> remove ym */ + ym = ym0; + for (j=0; jys_len; j++) + if (yspec->ys_stmt[j] == ym) + break; + ys_prune(yspec, j); + ys_free(ym); + } + } + if (yang_parse_post(h, yspec, modnr) < 0) + goto done; + retval = 0; + done: + if (dp) + free(dp); + if (base) + free(base); + if (oldbase) + free(oldbase); + return retval; +} + +/*! parse yang date-arg string and return a uint32 useful for arithmetics + * @param[in] datearg yang revision string as "YYYY-MM-DD" + * @param[out] dateint Integer version as YYYYMMDD + * @retval 0 OK + * @retval -1 Error, eg str is not on the format "YYYY-MM-DD" + */ +int +ys_parse_date_arg(char *datearg, + uint32_t *dateint) +{ + int retval = -1; + int i; + uint32_t d = 0; + + if (strlen(datearg) != 10 || datearg[4] != '-' || datearg[7] != '-'){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); + goto done; + } + if ((i = cligen_tonum(4, datearg)) < 0){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); + goto done; + } + d = i*10000; /* year */ + if ((i = cligen_tonum(2, &datearg[5])) < 0){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); + goto done; + } + d += i*100; /* month */ + if ((i = cligen_tonum(2, &datearg[8])) < 0){ + clicon_err(OE_YANG, EINVAL, "Revision date %s, but expected: YYYY-MM-DD", datearg); + goto done; + } + d += i; /* day */ + *dateint = d; + retval = 0; + done: + return retval; +} + +/*! Parse argument as CV and save result in yang cv variable + * + * Note that some CV:s are parsed directly (eg fraction-digits) while others are parsed + * in third pass (ys_populate). The reason being that all information is not + * available in the first pass. Prefer to do stuff in ys_populate + */ +cg_var * +ys_parse(yang_stmt *ys, + enum cv_type cvtype) +{ + int cvret; + char *reason = NULL; + + assert(yang_cv_get(ys) == NULL); /* Cv:s are parsed in different places, difficult to separate */ + if ((ys->ys_cv = cv_new(cvtype)) == NULL){ + clicon_err(OE_YANG, errno, "cv_new"); + goto done; + } + if ((cvret = cv_parse1(yang_argument_get(ys), ys->ys_cv, &reason)) < 0){ /* error */ + clicon_err(OE_YANG, errno, "parsing cv"); + ys->ys_cv = NULL; + goto done; + } + if (cvret == 0){ /* parsing failed */ + clicon_err(OE_YANG, errno, "Parsing CV: %s", reason); + ys->ys_cv = NULL; + goto done; + } + /* cvret == 1 means parsing is OK */ + done: + if (reason) + free(reason); + return ys->ys_cv; +} + +/*! First round yang syntactic statement specific checks. No context checks. + * + * Specific syntax checks and variable creation for stand-alone yang statements. + * That is, siblings, etc, may not be there. Complete checks are made in + * ys_populate instead. + * @param[in] ys yang statement + * @param[in] extra Yang extra for cornercases (unknown/extension). Is consumed + * + * The cv:s created in parse-tree as follows: + * fraction-digits : Create cv as uint8, check limits [1:8] (must be made in 1st pass) + * revision: cv as uint32 date: Integer version as YYYYMMDD + * min-elements: cv as uint32 + * max-elements: cv as uint32, '0' means unbounded + * @see ys_populate + */ +int +ys_parse_sub(yang_stmt *ys, + char *extra) +{ + int retval = -1; + uint8_t fd; + uint32_t date = 0; + char *arg; + enum rfc_6020 keyword; + char *reason = NULL; + int ret; + uint32_t minmax; + + arg = yang_argument_get(ys); + keyword = yang_keyword_get(ys); + switch (keyword){ + case Y_FRACTION_DIGITS: + if (ys_parse(ys, CGV_UINT8) == NULL) + goto done; + fd = cv_uint8_get(ys->ys_cv); + if (fd < 1 || fd > 18){ + clicon_err(OE_YANG, errno, "%u: Out of range, should be [1:18]", fd); + goto done; + } + break; + case Y_REVISION: + case Y_REVISION_DATE: /* YYYY-MM-DD encoded as uint32 YYYYMMDD */ + if (ys_parse_date_arg(arg, &date) < 0) + goto done; + if ((ys->ys_cv = cv_new(CGV_UINT32)) == NULL){ + clicon_err(OE_YANG, errno, "cv_new"); + goto done; + } + cv_uint32_set(ys->ys_cv, date); + break; + case Y_STATUS: /* RFC7950 7.21.2: "current", "deprecated", or "obsolete". */ + if (strcmp(arg, "current") && + strcmp(arg, "deprecated") && + strcmp(arg, "obsolete")){ + clicon_err(OE_YANG, errno, "Invalid status: \"%s\", expected current, deprecated, or obsolete", arg); + goto done; + + } + break; + case Y_MAX_ELEMENTS: + case Y_MIN_ELEMENTS: + if ((ys->ys_cv = cv_new(CGV_UINT32)) == NULL){ + clicon_err(OE_YANG, errno, "cv_new"); + goto done; + } + if (keyword == Y_MAX_ELEMENTS && + strcmp(arg, "unbounded") == 0) + cv_uint32_set(ys->ys_cv, 0); /* 0 means unbounded for max */ + else{ + if ((ret = parse_uint32(arg, &minmax, &reason)) < 0){ + clicon_err(OE_YANG, errno, "parse_uint32"); + goto done; + } + if (ret == 0){ + clicon_err(OE_YANG, EINVAL, "element-min/max parse error: %s", reason); + if (reason) + free(reason); + goto done; + } + cv_uint32_set(ys->ys_cv, minmax); + } + break; + case Y_MODIFIER: + if (strcmp(yang_argument_get(ys), "invert-match")){ + clicon_err(OE_YANG, EINVAL, "modifier %s, expected invert-match", yang_argument_get(ys)); + goto done; + } + break; + case Y_UNKNOWN:{ /* save (optional) argument in ys_cv */ + if (extra == NULL) + break; + if ((ys->ys_cv = cv_new(CGV_STRING)) == NULL){ + clicon_err(OE_YANG, errno, "cv_new"); + goto done; + } + if ((ret = cv_parse1(extra, ys->ys_cv, &reason)) < 0){ /* error */ + clicon_err(OE_YANG, errno, "parsing cv"); + goto done; + } + if (ret == 0){ /* parsing failed */ + clicon_err(OE_YANG, errno, "Parsing CV: %s", reason); + goto done; + } + break; + } + default: + break; + } + retval = 0; + done: + if (extra) + free(extra); + return retval; +} diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index 9016cdc3..b77c4352 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -91,13 +91,12 @@ #include "clixon_hash.h" #include "clixon_handle.h" #include "clixon_regex.h" -#include "clixon_yang.h" #include "clixon_hash.h" +#include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_plugin.h" #include "clixon_options.h" -#include "clixon_yang.h" -#include "clixon_yang_internal.h" /* internal */ +#include "clixon_yang_module.h" #include "clixon_yang_type.h" /* @@ -166,176 +165,6 @@ yang_builtin(char *type) return 0; } -/*! Set type cache for yang type - * @param[in] rxmode Kludge to know which regexp engine is used - * @see yang_type_cache_regexp_set where cache is extended w compiled regexps - */ -static int -yang_type_cache_set(yang_type_cache **ycache0, - yang_stmt *resolved, - int options, - cvec *cvv, - cvec *patterns, - uint8_t fraction) -{ - int retval = -1; - yang_type_cache *ycache = *ycache0; - - assert (ycache == NULL); - if ((ycache = (yang_type_cache *)malloc(sizeof(*ycache))) == NULL){ - clicon_err(OE_UNIX, errno, "malloc"); - goto done; - } - memset(ycache, 0, sizeof(*ycache)); - *ycache0 = ycache; - ycache->yc_resolved = resolved; - ycache->yc_options = options; - if (cvv){ - if ((ycache->yc_cvv = cvec_dup(cvv)) == NULL){ - clicon_err(OE_UNIX, errno, "cvec_dup"); - goto done; - } - } - if (patterns && (ycache->yc_patterns = cvec_dup(patterns)) == NULL){ - clicon_err(OE_UNIX, errno, "cvec_dup"); - goto done; - } - ycache->yc_fraction = fraction; - retval = 0; - done: - return retval; -} - -/*! Extend yang type cache with compiled regexps - * Compiled Regexps are computed in validate code - after initial cache set - * @param[in] regexps - */ -static int -yang_type_cache_regexp_set(yang_stmt *ytype, - int rxmode, - cvec *regexps) -{ - int retval = -1; - yang_type_cache *ycache; - - assert(regexps); - assert(yang_keyword_get(ytype) == Y_TYPE); - assert((ycache = ytype->ys_typecache) != NULL); - assert(ycache->yc_regexps == NULL); - ycache->yc_rxmode = rxmode; - if ((ycache->yc_regexps = cvec_dup(regexps)) == NULL){ - clicon_err(OE_UNIX, errno, "cvec_dup"); - goto done; - } - retval = 0; - done: - return retval; -} - -/*! Get individual fields (direct/destructively) from yang type cache. - * @param[out] patterns Initialized cvec of regexp patterns strings - */ -static int -yang_type_cache_get(yang_type_cache *ycache, - yang_stmt **resolved, - int *options, - cvec **cvv, - cvec *patterns, - int *rxmode, - cvec *regexps, - uint8_t *fraction) -{ - int retval = -1; - cg_var *cv = NULL; - - if (resolved) - *resolved = ycache->yc_resolved; - if (options) - *options = ycache->yc_options; - if (cvv) - *cvv = ycache->yc_cvv; - if (patterns){ - cv = NULL; - while ((cv = cvec_each(ycache->yc_patterns, cv)) != NULL) - cvec_append_var(patterns, cv); - } - if (regexps){ - cv = NULL; - while ((cv = cvec_each(ycache->yc_regexps, cv)) != NULL) - cvec_append_var(regexps, cv); - } - if (rxmode) - *rxmode = ycache->yc_rxmode; - if (fraction) - *fraction = ycache->yc_fraction; - retval = 0; - // done: - return retval; -} - -int -yang_type_cache_cp(yang_type_cache **ycnew, - yang_type_cache *ycold) -{ - int retval = -1; - int options; - cvec *cvv; - cvec *patterns = NULL; - uint8_t fraction; - yang_stmt *resolved; - - if ((patterns = cvec_new(0)) == NULL){ - clicon_err(OE_UNIX, errno, "cvec_new"); - goto done; - } - /* Note, regexps are not copied since they are voids, if they were, they - * could not be freed in a simple way since copies are made at augment/group - */ - yang_type_cache_get(ycold, &resolved, &options, &cvv, patterns, NULL, NULL, &fraction); - if (yang_type_cache_set(ycnew, resolved, options, cvv, patterns, fraction) < 0) - goto done; - retval = 0; - done: - if (patterns) - cvec_free(patterns); - return retval; -} - -int -yang_type_cache_free(yang_type_cache *ycache) -{ - cg_var *cv; - void *p; - - if (ycache->yc_cvv) - cvec_free(ycache->yc_cvv); - if (ycache->yc_patterns) - cvec_free(ycache->yc_patterns); - if (ycache->yc_regexps){ - cv = NULL; - while ((cv = cvec_each(ycache->yc_regexps, cv)) != NULL){ - /* need to store mode since clicon_handle is not available */ - switch (ycache->yc_rxmode){ - case REGEXP_POSIX: - cligen_regex_posix_free(cv_void_get(cv)); - break; - case REGEXP_LIBXML2: - cligen_regex_libxml2_free(cv_void_get(cv)); - break; - default: - break; - } - if ((p = cv_void_get(cv)) != NULL){ - free(p); - cv_void_set(cv, NULL); - } - } - cvec_free(ycache->yc_regexps); - } - free(ycache); - return 0; -} - /*! Compile yang patterns in string form to regex compiled void* form * and re-store into "patterns" cvec. * This is done here instead of deep in resolve code (resolve_restrictions) @@ -415,7 +244,7 @@ ys_resolve_type(yang_stmt *ys, /* Recursively resolve ys -> resolve with restrictions(options, etc) * Note that the resolved type could be ys itself. */ - if (yang_type_resolve(ys->ys_parent, ys->ys_parent, + if (yang_type_resolve(yang_parent_get(ys), yang_parent_get(ys), ys, &resolved, &options, &cvv, patterns, NULL, &fraction) < 0) goto done; @@ -423,8 +252,7 @@ ys_resolve_type(yang_stmt *ys, /* Cache the type resolving locally. Only place where this is done. * Why not do it in yang_type_resolve? (compile regexps needs clicon_handle) */ - if (yang_type_cache_set(&ys->ys_typecache, - resolved, options, cvv, + if (yang_type_cache_set(ys, resolved, options, cvv, patterns, fraction) < 0) goto done; retval = 0; @@ -512,13 +340,15 @@ clicon_type2cv(char *origtype, enum cv_type *cvtype) { int retval = -1; + yang_stmt *ym; *cvtype = CGV_ERR; + ym = ys_module(ys); if (restype != NULL){ yang2cv_type(restype, cvtype); if (*cvtype == CGV_ERR){ clicon_err(OE_YANG, 0, "%s: \"%s\" type not translated", - ys_module(ys)->ys_argument, restype); + yang_argument_get(ym), restype); goto done; } } @@ -530,7 +360,7 @@ clicon_type2cv(char *origtype, yang2cv_type(origtype, cvtype); if (*cvtype == CGV_ERR){ clicon_err(OE_YANG, 0, "%s:\"%s\": type not resolved", - ys_module(ys)->ys_argument, origtype); + yang_argument_get(ym), origtype); goto done; } } @@ -832,9 +662,9 @@ cv_validate1(clicon_handle h, yi = NULL; if (str != NULL) while ((yi = yn_each(yrestype, yi)) != NULL){ - if (yi->ys_keyword != Y_ENUM) + if (yang_keyword_get(yi) != Y_ENUM) continue; - if (strcmp(yi->ys_argument, str) == 0){ + if (strcmp(yang_argument_get(yi), str) == 0){ found++; break; } @@ -860,9 +690,9 @@ cv_validate1(clicon_handle h, found = 0; yi = NULL; while ((yi = yn_each(yrestype, yi)) != NULL){ - if (yi->ys_keyword != Y_BIT) + if (yang_keyword_get(yi) != Y_BIT) continue; - if (strcmp(yi->ys_argument, v) == 0){ + if (strcmp(yang_argument_get(yi), v) == 0){ found++; break; } @@ -944,7 +774,7 @@ ys_cv_validate_union_one(clicon_handle h, if (yang_type_resolve(ys, ys, yt, &yrt, &options, &cvv, patterns, regexps, &fraction) < 0) goto done; - restype = yrt?yrt->ys_argument:NULL; + restype = yrt?yang_argument_get(yrt):NULL; if (restype && strcmp(restype, "union") == 0){ /* recursive union */ if ((retval = ys_cv_validate_union(h, ys, reason, yrt, type, val)) < 0) goto done; @@ -957,6 +787,10 @@ ys_cv_validate_union_one(clicon_handle h, clicon_err(OE_UNIX, errno, "cv_new"); goto done; } + if (val == NULL){ /* Fail validation on NULL */ + retval = 0; + goto done; + } if ((retval = cv_parse1(val, cvt, reason)) < 0){ clicon_err(OE_UNIX, errno, "cv_parse"); goto done; @@ -988,7 +822,7 @@ ys_cv_validate_union_one(clicon_handle h, return retval; } -/*! +/*! Validate union * @param[out] reason If given, and return value is 0, contains malloced string * @retval -1 Error (fatal), with errno set to indicate error * @retval 0 Validation not OK, malloced reason is returned. Free reason with free() @@ -1007,7 +841,7 @@ ys_cv_validate_union(clicon_handle h, char *reason1 = NULL; /* saved reason */ while ((yt = yn_each(yrestype, yt)) != NULL){ - if (yt->ys_keyword != Y_TYPE) + if (yang_keyword_get(yt) != Y_TYPE) continue; if ((retval = ys_cv_validate_union_one(h, ys, reason, yt, type, val)) < 0) goto done; @@ -1069,11 +903,11 @@ ys_cv_validate(clicon_handle h, if (reason) *reason=NULL; - if (ys->ys_keyword != Y_LEAF && ys->ys_keyword != Y_LEAF_LIST){ + if (yang_keyword_get(ys) != Y_LEAF && yang_keyword_get(ys) != Y_LEAF_LIST){ retval = 1; goto done; } - ycv = ys->ys_cv; + ycv = yang_cv_get(ys); if ((regexps = cvec_new(0)) == NULL){ clicon_err(OE_UNIX, errno, "cvec_new"); goto done; @@ -1087,7 +921,7 @@ ys_cv_validate(clicon_handle h, patterns, regexps, &fraction) < 0) goto done; - restype = yrestype?yrestype->ys_argument:NULL; + restype = yrestype?yang_argument_get(yrestype):NULL; if (clicon_type2cv(origtype, restype, ys, &cvtype) < 0) goto done; @@ -1104,7 +938,11 @@ ys_cv_validate(clicon_handle h, /* Note restype can be NULL here for example with unresolved hardcoded uuid */ if (restype && strcmp(restype, "union") == 0){ assert(cvtype == CGV_REST); - val = cv_string_get(cv); + /* Instead of NULL, give an empty string to validate, this is to avoid cv_parse + * errors and may actually be the wrong thing to do. + */ + if ((val = cv_string_get(cv)) == NULL) + val = ""; if ((retval2 = ys_cv_validate_union(h, ys, reason, yrestype, origtype, val)) < 0) goto done; retval = retval2; /* invalid (0) with latest reason or valid 1 */ @@ -1146,8 +984,8 @@ ys_cv_validate(clicon_handle h, static inline int ys_typedef(yang_stmt *ys) { - return ys->ys_keyword == Y_MODULE || ys->ys_keyword == Y_SUBMODULE || - ys->ys_keyword == Y_CONTAINER || ys->ys_keyword == Y_LIST; + return yang_keyword_get(ys) == Y_MODULE || yang_keyword_get(ys) == Y_SUBMODULE || + yang_keyword_get(ys) == Y_CONTAINER || yang_keyword_get(ys) == Y_LIST; } /* find next ys up which can contain a typedef */ @@ -1157,9 +995,9 @@ ys_typedef_up(yang_stmt *ys) yang_stmt *yn; while (ys != NULL && !ys_typedef(ys)){ - yn = ys->ys_parent; + yn = yang_parent_get(ys); /* Some extra stuff to ensure ys is a stmt */ - if (yn && yn->ys_keyword == Y_SPEC) + if (yn && yang_keyword_get(yn) == Y_SPEC) yn = NULL; ys = (yang_stmt*)yn; } @@ -1222,8 +1060,8 @@ yang_find_identity(yang_stmt *ys, if ((yid = yang_find(ys, Y_IDENTITY, id)) != NULL) break; /* Did not find a matching typedef there, proceed to next level */ - yn = ys->ys_parent; - if (yn && yn->ys_keyword == Y_SPEC) + yn = yang_parent_get(ys); + if (yn && yang_keyword_get(yn) == Y_SPEC) yn = NULL; ys = (yang_stmt*)yn; } @@ -1263,12 +1101,12 @@ yang_type_resolve_restrictions(yang_stmt *ytype, if (options && cvv && (ys = yang_find(ytype, Y_RANGE, NULL)) != NULL){ - *cvv = ys->ys_cvec; + *cvv = yang_cvec_get(ys); *options |= YANG_OPTIONS_RANGE; } if (options && cvv && (ys = yang_find(ytype, Y_LENGTH, NULL)) != NULL){ - *cvv = ys->ys_cvec; + *cvv = yang_cvec_get(ys); *options |= YANG_OPTIONS_LENGTH; } /* Find all patterns */ @@ -1281,7 +1119,7 @@ yang_type_resolve_restrictions(yang_stmt *ytype, clicon_err(OE_UNIX, errno, "cvec_add"); goto done; } - pattern = ys->ys_argument; /* clear text pattern */ + pattern = yang_argument_get(ys); /* clear text pattern */ /* Check 1.1 invert pattern */ if (yang_find(ys, Y_MODIFIER, "invert-match") != NULL) cv_flag_set(cv, V_INVERT); @@ -1290,7 +1128,7 @@ yang_type_resolve_restrictions(yang_stmt *ytype, } if (options && fraction && (ys = yang_find(ytype, Y_FRACTION_DIGITS, NULL)) != NULL){ - *fraction = cv_uint8_get(ys->ys_cv); + *fraction = cv_uint8_get(yang_cv_get(ys)); *options |= YANG_OPTIONS_FRACTION_DIGITS; } retval = 0; @@ -1339,6 +1177,7 @@ yang_type_resolve(yang_stmt *yorig, int retval = -1; yang_stmt *yn; yang_stmt *yrmod; /* module where resolved type is looked for */ + int ret; if (options) *options = 0x0; @@ -1347,13 +1186,20 @@ yang_type_resolve(yang_stmt *yorig, if (nodeid_split(yang_argument_get(ytype), &prefix, &type) < 0) goto done; /* Cache does not work for eg string length 32? */ +#if 1 + if ((ret = yang_type_cache_get(ytype, yrestype, + options, cvv, patterns, NULL, regexps, fraction)) < 0) + goto done; + if (ret == 1) + goto ok; +#else if (ytype->ys_typecache != NULL){ - if (yang_type_cache_get(ytype->ys_typecache, yrestype, + if (yang_type_cache_get(ytype, yrestype, options, cvv, patterns, NULL, regexps, fraction) < 0) goto done; goto ok; } - +#endif /* Check if type is basic type. If so, return that */ if ((prefix == NULL && yang_builtin(type))){ *yrestype = ytype; @@ -1366,7 +1212,7 @@ yang_type_resolve(yang_stmt *yorig, if (prefix){ /* Go to top and find import that matches */ if ((yrmod = yang_find_module_by_prefix(ytype, prefix)) == NULL){ clicon_err(OE_DB, 0, "Type not resolved: \"%s:%s\" in module %s", - prefix, type, ys_module(yorig)->ys_argument); + prefix, type, yang_argument_get(ys_module(yorig))); goto done; } if ((rytypedef = yang_find(yrmod, Y_TYPEDEF, type)) == NULL) @@ -1384,8 +1230,8 @@ yang_type_resolve(yang_stmt *yorig, if ((rytypedef = yang_find(ys, Y_TYPEDEF, type)) != NULL) break; /* Did not find a matching typedef there, proceed to next level */ - yn = ys->ys_parent; - if (yn && (yn->ys_keyword == Y_SPEC)) + yn = yang_parent_get(ys); + if (yn && (yang_keyword_get(yn) == Y_SPEC)) yn = NULL; ys = (yang_stmt*)yn; } @@ -1488,8 +1334,6 @@ yang_type_get(yang_stmt *ys, if (yang_type_resolve(ys, ys, ytype, yrestype, options, cvv, patterns, regexps, fraction) < 0) goto done; - clicon_debug(3, "%s: %s %s->%s", __FUNCTION__, ys->ys_argument, type, - *yrestype?(*yrestype)->ys_argument:"null"); retval = 0; done: if (type) @@ -1512,7 +1356,7 @@ yang_type2cv(yang_stmt *ys) if (yang_type_get(ys, &origtype, &yrestype, NULL, NULL, NULL, NULL, NULL) < 0) goto done; - restype = yrestype?yrestype->ys_argument:NULL; + restype = yrestype?yang_argument_get(yrestype):NULL; if (clicon_type2cv(origtype, restype, ys, &cvtype) < 0) /* This handles non-resolved also */ goto done; done: diff --git a/test/Makefile.in b/test/Makefile.in index aba38b55..5ad45aac 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -1,7 +1,7 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2009-2020 Olof Hagsand # # This file is part of CLIXON # @@ -33,12 +33,6 @@ VPATH = @srcdir@ srcdir = @srcdir@ top_srcdir = @top_srcdir@ -CC = @CC@ -CFLAGS = @CFLAGS@ -LDFLAGS = @LDFLAGS@ -LIBS = @LIBS@ - -SHELL = /bin/sh .PHONY: all clean distclean depend install uninstall diff --git a/test/README.md b/test/README.md index fd34175f..0830fd74 100644 --- a/test/README.md +++ b/test/README.md @@ -17,6 +17,8 @@ In the CI process, the system is built and configured and then the [clixon test container](../docker/system) is built and the tests in this directory is executed. +There are also [manual cicd scripts here](cicd/README.md) + ## Getting started You need to build and install the clixon utility programs before running the tests as some of the tests rely on them: diff --git a/test/cicd/Makefile.in b/test/cicd/Makefile.in new file mode 100644 index 00000000..52bf296e --- /dev/null +++ b/test/cicd/Makefile.in @@ -0,0 +1,74 @@ +# +# ***** BEGIN LICENSE BLOCK ***** +# +# Copyright (C) 2009-2020 Olof Hagsand +1# +# This file is part of CLIXON +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Alternatively, the contents of this file may be used under the terms of +# the GNU General Public License Version 3 or later (the "GPL"), +# in which case the provisions of the GPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of the GPL, and not to allow others to +# use your version of this file under the terms of Apache License version 2, +# indicate your decision by deleting the provisions above and replace them with +# the notice and other provisions required by the GPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the Apache License version 2 or the GPL. +# +# ***** END LICENSE BLOCK ***** +# + +VPATH = @srcdir@ +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +SHELL = /bin/sh + +.PHONY: all clean distclean depend install uninstall + +HOSTS = vandal.hagsand.com # i86_32 ubuntu +HOSTS += clixon.dogwood.com # FreeBSD x86_64 +#HOSTS += nuc1.hagsand.com # x86_64 ubuntu +#HOSTS += pi2.hagsand.com # arm Raspian + +SCRIPTS = cligen-mk.sh +SCRIPTS += clixon-mk.sh +SCRIPTS += clixon-config.sh + +.PHONY: all clean distclean depend install uninstall $(HOSTS) + +all: $(HOSTS) + +$(HOSTS): + for s in $(SCRIPTS); do \ + (scp $$s $@:/tmp/ ; ssh $@ chmod 750 /tmp/$$s || exit 1) \ + done; + ./cicd.sh $@ 2>&1 | tee $@.log + +clean: + rm -f *.log + +distclean: clean + rm -f Makefile *~ .depend + +depend: + +install-include: + +install: + +uninstall: + diff --git a/test/cicd/README.md b/test/cicd/README.md new file mode 100644 index 00000000..3c54ae07 --- /dev/null +++ b/test/cicd/README.md @@ -0,0 +1,14 @@ +CICD scripts +============ +Manual scripts for running committed code on a set of hosts. + +The script then uses a Makefile and logs in to each host, pulls from +git, configure, makes and runs through the tests. Make is used to get +concurrency - non-trivial with bash, eg with `make -j 10` + +Note there are other cicd scripts than this, such as the the "travis" scrips. + +The Makefile contains a configurable HOSTS variable, please edit. + +Logs appear in : .log. + diff --git a/test/cicd/cicd.sh b/test/cicd/cicd.sh new file mode 100755 index 00000000..20789633 --- /dev/null +++ b/test/cicd/cicd.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# CI/CD script complementing trevor github +# Login in to a number of hosts and fo the following: +# 0. Create and transfer sub-scripts used in main script: cligen-mk.sh clixon-mk.sh clixon-config.sh +# 1. pull latest version +# 2. Run configure +# 3. Compile and install (assume mk.sh) +# 4. Run tests +# Assume: +# - subscripts SCRIPTS exists locally where this script is executed +# - A test/site.sh file is handmade on each host +# - some commands are passwordless using +# sudo visudo -f /etc/sudoers.d/clixonci +# ALL = (root)NOPASSWD : ALL +# ALL = (www-data)NOPASSWD : ALL +# ALL = (clicon)NOPASSWD : /usr/local/sbin/clixon_backend +# Experiment in identifying all commands: /usr/bin/make,/usr/local/sbin/clixon_backend,/usr/bin/pkill,/usr/local/bin/clixon_util_socket,/usr/bin/tee,/bin/rm,/usr/bin/touch,/bin/chmod +# +# Typical run: ./cicd.sh 2>&1 | tee cilog + +set -eux # x + +if [ $# -ne 1 ]; then + echo "usage: $0 " + exit -1 +fi + +h=$1 + +ssh -t $h "test -d src/cligen || (cd src;git clone https://github.com/olofhagsand/cligen.git)" +ssh -t $h "(cd src/cligen;git pull)" +ssh -t $h "(cd src/cligen;./configure)" +ssh -t $h "(cd src/cligen; /tmp/cligen-mk.sh)" +ssh -t $h "test -d src/clixon || (cd src;git clone https://github.com/clicon/clixon.git)" +ssh -t $h "(cd src/clixon;git pull)" +ssh -t $h "(cd src/clixon; /tmp/clixon-config.sh)" +ssh -t $h "(cd src/clixon; /tmp/clixon-mk.sh)" +ssh -t $h sudo ldconfig +ssh -t $h "(cd src/clixon/test; ./sum.sh)" diff --git a/test/cicd/cligen-mk.sh b/test/cicd/cligen-mk.sh new file mode 100644 index 00000000..58cdc79c --- /dev/null +++ b/test/cicd/cligen-mk.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# A top-level maker for cligen +set -eux +if [ $(uname) = "FreeBSD" ]; then + MAKE=$(which gmake) +else + MAKE=$(which make) +fi +$MAKE clean +$MAKE -j10 +sudo $MAKE install diff --git a/test/cicd/clixon-config.sh b/test/cicd/clixon-config.sh new file mode 100644 index 00000000..8bb0f046 --- /dev/null +++ b/test/cicd/clixon-config.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# A top-level configurer for clixon +set -eux +if [ $(uname) = "FreeBSD" ]; then + ./configure --with-cligen=/usr/local --with-wwwuser=www --enable-optyangs +else + ./configure --enable-optyangs +fi diff --git a/test/cicd/clixon-mk.sh b/test/cicd/clixon-mk.sh new file mode 100644 index 00000000..20cbe7d2 --- /dev/null +++ b/test/cicd/clixon-mk.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# A top-level maker for clixon +set -eux +if [ $(uname) = "FreeBSD" ]; then + MAKE=$(which gmake) +else + MAKE=$(which make) +fi +$MAKE clean +$MAKE -j10 +sudo $MAKE install +sudo $MAKE install-include +(cd example; $MAKE) +(cd util; $MAKE) +(cd example; sudo $MAKE install) +(cd util; sudo $MAKE install) diff --git a/test/lib.sh b/test/lib.sh index 93673f55..4beaac79 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -339,7 +339,7 @@ expecteq(){ # Evaluate and return # like expecteq but partial match is OK -# Example: expecteq $(fn arg) 0 "my return" +# Example: expectpart $(fn arg) 0 "my return" # - evaluated expression # - expected command return value (0 if OK) # - expected stdout outcome* @@ -364,15 +364,15 @@ expectpart(){ let i=0; for exp in "$@"; do if [ $i -gt 1 ]; then -# echo "exp:$exp" - match=`echo $ret | grep --null -o "$exp"` # XXX -EZo: -E cant handle {} - if [ -z "$match" ]; then - err "$exp" "$ret" - fi - fi - let i++; - done - +# echo "echo \"$ret\" | grep --null -o \"$exp"\" + match=$(echo "$ret" | grep --null -o "$exp") # XXX -EZo: -E cant handle {} + r=$? + if [ $r != 0 ]; then + err "$exp" "$ret" + fi + fi + let i++; + done # if [[ "$ret" != "$expect" ]]; then # err "$expect" "$ret" # fi diff --git a/test/mem.sh b/test/mem.sh index 672fc0d6..a5b33714 100755 --- a/test/mem.sh +++ b/test/mem.sh @@ -98,9 +98,9 @@ done # Then actual run testnr=0 -for cmd in $cmds; do +for cmd1 in $cmds; do if [ $testnr != 0 ]; then echo; fi - println "Mem test $cmd begin" - memonce $cmd - println "Mem test $cmd done" + println "Mem test $cmd1 begin" + memonce $cmd1 + println "Mem test $cmd1 done" done diff --git a/test/test_api.sh b/test/test_api.sh new file mode 100755 index 00000000..fd1feb6f --- /dev/null +++ b/test/test_api.sh @@ -0,0 +1,265 @@ +#!/bin/bash +# Advanced API XML test. Compile a backend plugin and start the backend, and then send an RPC to +# trigger that plugin +# The plugin looks in an XML tree using three different methods: +# 1. xml_each and xml_find +# 2. xpath_first +# 3. binary_search + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +# Which format to use as datastore format internally +: ${format:=xml} + +APPNAME=example + +cfg=$dir/conf_yang.xml +fyang=$dir/example-api.yang +cfile=$dir/example-api.c +pdir=$dir/plugin +sofile=$pdir/example-api.so + +if [ ! -d $pdir ]; then + mkdir $pdir +fi + +cat < $cfg + + /tmp/conf_yang.xml + /usr/local/share/clixon + $IETFRFC + $fyang + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + false + $pdir + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + $format + +EOF + +cat < $fyang +module example-api{ + yang-version 1.1; + namespace "urn:example:api"; + prefix ex; + container c { + leaf-list y0 { + ordered-by user; + type string; + } + leaf-list y1 { + ordered-by system; + type string; + } + list y2 { + ordered-by user; + key "k"; + leaf k { + type int32; + } + leaf val { + type string; + } + } + list y3 { + ordered-by system; + key "k"; + leaf k { + type int32; + } + leaf val { + type string; + } + } + } + rpc trigger { + description "trigger an action in the backend"; + } +} +EOF + +cat< $cfile +#include +#include +#include +#include +#include +#include +#include +#include + +/* clicon */ +#include + +/* Clicon library functions. */ +#include + +/* These include signatures for plugin and transaction callbacks. */ +#include + +static int +trigger_rpc(clicon_handle h, /* Clicon handle */ + cxobj *xe, /* Request: */ + cbuf *cbret, /* Reply eg ... */ + void *arg, /* client_entry */ + void *regarg) /* Argument given at register */ +{ + int retval = -1; + cxobj *xret = NULL; + cxobj *xc = NULL; + cxobj *x = NULL; + char *k; + char *val; + cvec *cvk = NULL; + cg_var *cv; + cxobj **xvec = NULL; + size_t xlen = 0; + + if (xmldb_get(h, "running", NULL, "/c", &xret) < 0) + goto done; + clicon_debug(1, "%s xret:%s", __FUNCTION__, xml_name(xret)); + xc = xpath_first(xret, NULL, "/c"); + clicon_debug(1, "%s xc:%s", __FUNCTION__, xml_name(xc)); + + /* Method 1 loop */ + x = NULL; + val = NULL; + while ((x = xml_child_each(xc, x, -1)) != NULL) { + if (strcmp(xml_name(x), "y3") != 0) + continue; + if ((k = xml_find_body(x, "k")) != NULL && + strcmp(k, "5") == 0){ + val = xml_find_body(x, "val"); + break; + } + } + clicon_debug(1, "%s Method 1: val:%s", __FUNCTION__, val?val:"null"); + + /* Method 2 xpath */ + val = NULL; + if ((x = xpath_first(xc, NULL, "y3[k=5]")) != NULL) + val = xml_find_body(x, "val"); + clicon_debug(1, "%s Method 2: val:%s", __FUNCTION__, val?val:"null"); + + /* Method 3 binsearch */ + val = NULL; + /* Add key/value vector */ + if ((cvk = cvec_new(0)) == NULL){ + clicon_err(OE_YANG, errno, "cvec_new"); + goto done; + } + if ((cv = cvec_add(cvk, CGV_STRING)) == NULL) + goto done; + cv_name_set(cv, "k"); + cv_string_set(cv, "5"); + /* Use form 2c use spec of xc + name */ + if (clixon_xml_find_index(xc, NULL, NULL, "y3", cvk, &xvec, &xlen) < 0) + goto done; + if (xlen) + val = xml_find_body(xvec[0], "val"); + else + val = NULL; + clicon_debug(1, "%s Method 3: val:%s", __FUNCTION__, val?val:"null"); + + cprintf(cbret, ""); + retval = 0; + done: + clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); + if (cvk) + cvec_free(cvk); + if (xret) + xml_free(xret); + if (xvec) + free(xvec); + return retval; +} + +clixon_plugin_api *clixon_plugin_init(clicon_handle h); + +static clixon_plugin_api api = { + "order", /* name */ /*--- Common fields. ---*/ + clixon_plugin_init, /* init */ +}; + +/*! Backend plugin initialization + * @param[in] h Clixon handle + * @retval NULL Error with clicon_err set + * @retval api Pointer to API struct + */ +clixon_plugin_api * +clixon_plugin_init(clicon_handle h) +{ + clicon_debug(1, "%s test-order", __FUNCTION__); + + /* From example.yang (clicon) */ + if (rpc_callback_register(h, trigger_rpc, + NULL, + "urn:example:api", + "trigger"/* Xml tag when callback is made */ + ) < 0) + return NULL; + return &api; +} + +EOF + +new "compile $cfile" +gcc -g -Wall -rdynamic -fPIC -shared $cfile -o $sofile + +new "test params: -s running -f $cfg" + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend" + start_backend -s running -f $cfg + +fi + +new "kill old restconf daemon" +sudo pkill -u www-data -f "/www-data/clixon_restconf" + +new "start restconf daemon" +start_restconf -f $cfg + +new "waiting" +wait_backend +wait_restconf + +XML='235zorro7' + +# Add a set of entries using restconf +new "PUT a set of entries" +expectpart "$(curl -si -X PUT -H 'Content-Type: application/yang-data+xml' http://localhost/restconf/data/example-api:c -d "$XML")" 0 "HTTP/1.1 201 Created" + +new "Check entries" +expectpart "$(curl -si -X GET http://localhost/restconf/data/example-api:c -H 'Accept: application/yang-data+xml')" 0 'HTTP/1.1 200 OK' "$XML" + +new "Send a trigger" +expectpart "$(curl -si -X POST http://localhost/restconf/operations/example-api:trigger -H 'Accept: application/yang-data+json')" 0 'HTTP/1.1 204 No Content' + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=`pgrep -u root -f clixon_backend` +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +# unset conditional parameters +unset format + +rm -rf $dir diff --git a/test/test_api_path.sh b/test/test_api_path.sh new file mode 100755 index 00000000..9a59402b --- /dev/null +++ b/test/test_api_path.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# API-PATH tests +# Most tests are: generate lists, then access +# Tests include single and double indexes. +# string and int indexes +# Lists and leaf-lists +# Augmented yang where two lists are inside each other (depth) +# Multiple matches + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +: ${clixon_util_path:=clixon_util_path -a -D $DBG} + +# Number of list/leaf-list entries +: ${nr:=100} + +# XML file (alt provide it in stdin after xpath) +for (( i=1; i<7; i++ )); do + eval xml$i=$dir/xml$i.xml +done +ydir=$dir/yang + +if [ ! -d $ydir ]; then + mkdir $ydir +fi + +# XPATH binary search in ordered-by system lists +cat < $ydir/moda.yang +module moda{ + namespace "urn:example:a"; + prefix a; + container x1{ + description "list with single string key"; + list y{ + ordered-by system; + key k1; + leaf k1{ + type string; + } + leaf z{ + type string; + } + } + } + container x2{ + description "list with single int key"; + list y{ + ordered-by system; + key k1; + leaf k1{ + type uint32; + } + leaf z{ + type string; + } + } + } + container x3{ + description "list with double string keys"; + list y{ + ordered-by system; + key "k1 k2"; + leaf k1{ + type string; + } + leaf k2{ + type string; + } + leaf z{ + type string; + } + } + } + container x4{ + description "leaf-list with int key"; + leaf-list y{ + type string; + } + } + list x5{ + ordered-by system; + description "Direct under root"; + key "k1"; + leaf k1{ + type string; + } + leaf z{ + type string; + } + } + augment "/b:x6/b:yy" { + list y{ + ordered-by system; + key "k1 k2"; + leaf k1{ + type string; + } + leaf k2{ + type string; + } + leaf-list z{ + type string; + } + } + } +} +EOF + +# This is for augment usecase +cat < $ydir/modb.yang +module modb{ + namespace "urn:example:b"; + prefix b; + container x6{ + description "deep tree and augment"; + list yy{ + ordered-by system; + key "kk1 kk2"; + leaf kk1{ + type string; + } + leaf kk2{ + type string; + } + leaf-list zz{ + type string; + } + } + } +} +EOF + +rnd=$(( ( RANDOM % $nr ) )) + +# Single string key +new "generate list with $nr single string key to $xml1" +echo -n '' > $xml1 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ifoo$i" >> $xml1 +done +echo -n '' >> $xml1 + +new "api-path single string key k1=a$rnd" +echo "$clixon_util_path -f $xml1 -y $ydir -p /moda:x1/y=a$rnd" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:x1/y=a$rnd)" 0 "^0: a$rndfoo$rnd$" + +new "api-path single string key /x1" +echo "$clixon_util_path -f $xml1 -y $ydir -p /moda:x1" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:x1)" 0 "0: a0foo0a1foo1" # Assume at least two elements + +new "api-path single string key omit key" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:x1/y)" 0 '^0: a0foo0 +1: a0foo0' + +new "api-path single string wrong module, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /modxxx:x1/y=a$rnd 2> /dev/null)" 255 '^$' + +new "api-path single string no module, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /x1/y=a$rnd 2> /dev/null)" 255 '^$' + +new "api-path single string wrong top-symbol, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:xxx/y=a$rnd 2> /dev/null)" 255 '^$' + +new "api-path single string wrong list-symbol, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:x1/xxx=a$rnd 2> /dev/null)" 255 '^$' + +new "api-path single string two keys, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:x1/y=a$rnd,a$rnd 2> /dev/null)" 255 '^$' + +new "api-path single string sub-element, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /moda:x1/y=a$rnd/xxx 2> /dev/null)" 255 '^$' + +# Single int key +new "generate list with $nr single int key to $xml2" +echo -n '' > $xml2 +for (( i=0; i<$nr; i++ )); do + echo -n "$ifoo$i" >> $xml2 +done +echo -n '' >> $xml2 + +new "api-path single int key k1=$rnd" +echo "$clixon_util_path -f $xml2 -y $ydir -p /moda:x2/y=$rnd" +expectpart "$($clixon_util_path -f $xml2 -y $ydir -p /moda:x2/y=$rnd)" 0 "^0: $rndfoo$rnd$" + +# Double string key +new "generate list with $nr double string keys to $xml3 (two k2 entries per k1 key)" +echo -n '' > $xml3 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ia$ifoo$i" >> $xml3 + echo -n "a$ib$ifoob$i" >> $xml3 +done +# Add two rules with empty k2 string +echo -n "a0foo0" >> $xml3 +echo -n "a1foo1" >> $xml3 +echo -n '' >> $xml3 + +new "api-path double string key k1=a$rnd,b$rnd" +echo "$clixon_util_path -f $xml3 -y $ydir -p /moda:x3/y=a$rnd,b$rnd" +expectpart "$($clixon_util_path -f $xml3 -y $ydir -p /moda:x3/y=a$rnd,b$rnd)" 0 "0: a$rndb$rndfoob$rnd" + +new "api-path double string key k1=a$rnd, - empty k2 string" +echo "$clixon_util_path -f $xml3 -y $ydir -p /moda:x3/y=a1," +expectpart "$($clixon_util_path -f $xml3 -y $ydir -p /moda:x3/y=a1,)" 0 "0: a1foo1" + +new "api-path double string key k1=a$rnd, - no k2 string - three matches" +echo "$clixon_util_path -f $xml3 -y $ydir -p /moda:x3/y=a1" +expecteq "$($clixon_util_path -f $xml3 -y $ydir -p /moda:x3/y=a1)" 0 "0: a1foo1 +1: a1a1foo1 +2: a1b1foob1" + +# Leaf-list +new "generate leaf-list int keys to $xml4" +echo -n '' > $xml4 +for (( i=0; i<$nr; i++ )); do + echo -n "a$i" >> $xml4 +done +echo -n '' >> $xml4 + +new "api-path leaf-list k1=a$rnd" +echo "$clixon_util_path -f $xml4 -y $ydir -p /moda:x4/y=a$rnd" +expectpart "$($clixon_util_path -f $xml4 -y $ydir -p /moda:x4/y=a$rnd)" 0 "^0: a$rnd$" + +# Single string key direct under root +new "generate list with $nr single string key to $xml5" +echo -n '' > $xml5 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ifoo$i" >> $xml5 +done + +new "api-path direct under root single string key k1=a$rnd" +echo "$clixon_util_path -f $xml5 -y $ydir -p /moda:x5=a$rnd" +expectpart "$($clixon_util_path -f $xml5 -y $ydir -p /moda:x5=a$rnd)" 0 "^0: a$rndfoo$rnd$" + +# Depth and augment +# Deep augmented xml path +new "generate deep list with augment" +echo -n '' > $xml6 +for (( i=0; i<$nr; i++ )); do + echo -n "b$ib$ifoo$i" >> $xml6 + for (( j=0; j<3; j++ )); do + echo -n "a$ja$jfoo$j" >> $xml6 + done + echo -n "" >> $xml6 +done +echo -n '' >> $xml6 + +new "api-path double string key k1=b$rnd,b$rnd in modb" +echo "$clixon_util_path -f $xml6 -y $ydir -p /modb:x6/yy=b$rnd,b$rnd" +expectpart "$($clixon_util_path -f $xml6 -y $ydir -p /modb:x6/yy=b$rnd,b$rnd)" 0 "0: b$rndb$rndfoo$rnda0a0foo0a1a1foo1a2a2foo2" + +new "api-path double string key k1=a$rnd,b$rnd in modb + augmented in moda" +echo "$clixon_util_path -f $xml6 -y $ydir -p /modb:x6/yy=b$rnd,b$rnd/moda:y=a1,a1" +expectpart "$($clixon_util_path -f $xml6 -y $ydir -p /modb:x6/yy=b$rnd,b$rnd/moda:y=a1,a1/z=foo1)" 0 "0: foo1" + +# unset conditional parameters +unset nr +unset clixon_util_path + +rm -rf $dir diff --git a/test/test_augment.sh b/test/test_augment.sh index ccb7af20..c3c9ec3d 100755 --- a/test/test_augment.sh +++ b/test/test_augment.sh @@ -36,7 +36,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile false - 1 /usr/local/var/$APPNAME true diff --git a/test/test_choice.sh b/test/test_choice.sh index 129908db..17419bb6 100755 --- a/test/test_choice.sh +++ b/test/test_choice.sh @@ -27,7 +27,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock false /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_cli.sh b/test/test_cli.sh index 52a24828..23a438a4 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -73,7 +73,7 @@ new "cli configure using encoded chars name <&" expectfn "$clixon_cli -1 -f $cfg set interfaces interface fddi&< type ianaift:ethernetCsmacd" 0 "" new "cli failed validate" -expectfn "$clixon_cli -1 -f $cfg -l o validate" 255 "Validate failed. Edit and try again or discard changes: application missing-element Mandatory variable type" +expectfn "$clixon_cli -1 -f $cfg -l o validate" 255 "Netconf error: application missing-element Mandatory variable type. Validate failed. Edit and try again or discard changes" new "cli configure ip addr" expectfn "$clixon_cli -1 -f $cfg set interfaces interface eth/0/0 ipv4 address 1.2.3.4 prefix-length 24" 0 "^$" diff --git a/test/test_cli_history.sh b/test/test_cli_history.sh index 24ab0160..2be7e4c3 100755 --- a/test/test_cli_history.sh +++ b/test/test_cli_history.sh @@ -27,7 +27,6 @@ cat < $cfg 10 /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF @@ -125,5 +124,5 @@ fi # kill backend stop_backend -f $cfg - +unset nr rm -rf $dir diff --git a/test/test_copy_config.sh b/test/test_copy_config.sh index 60bf7eb2..d0c5b638 100755 --- a/test/test_copy_config.sh +++ b/test/test_copy_config.sh @@ -46,7 +46,6 @@ cat < $cfg $APPNAME $dir/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 $dir EOF diff --git a/test/test_datastore.sh b/test/test_datastore.sh index 02c636f5..e0fc758e 100755 --- a/test/test_datastore.sh +++ b/test/test_datastore.sh @@ -161,7 +161,8 @@ diff $mydir/kalle_db $mydir/candidate_db new "datastore lock" expectfn "$clixon_util_datastore $conf lock 756" 0 "" -#leaf-list +# unset conditional parameters +unset clixon_util_datastore rm -rf $mydir diff --git a/test/test_feature.sh b/test/test_feature.sh index eda6b56e..60440377 100755 --- a/test/test_feature.sh +++ b/test/test_feature.sh @@ -39,7 +39,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME true diff --git a/test/test_identity.sh b/test/test_identity.sh index ce272ef6..18c8a965 100755 --- a/test/test_identity.sh +++ b/test/test_identity.sh @@ -250,15 +250,15 @@ expectfn "$clixon_cli -1 -f $cfg -l o validate" 0 "^$" new "CLI set wrong acl-type" expectfn "$clixon_cli -1 -f $cfg -l o set acls acl x type undefined" 0 "^$" -new "cli validate" -expectfn "$clixon_cli -1 -f $cfg -l o validate" 255 "Identityref validation failed" +new "cli validate acl-type" +expectfn "$clixon_cli -1 -f $cfg -l o validate" 255 "Netconf error: application operation-failed Identityref validation failed, undefined not derived from acl-base . Validate failed. Edit and try again or discard changes" # test empty identityref list new "cli set empty" expectfn "$clixon_cli -1 -f $cfg -l o set e undefined" 0 "^$" -new "cli validate" -expectfn "$clixon_cli -1 -f $cfg -l o validate" 255 "Identityref validation failed" +new "cli validate empty" +expectfn "$clixon_cli -1 -f $cfg -l o validate" 255 "Netconf error: application operation-failed Identityref validation failed, undefined not derived from acl-base . Validate failed. Edit and try again or discard changes" new "netconf discard-changes" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" diff --git a/test/test_insert.sh b/test/test_insert.sh index 0e6f7ba8..80180389 100755 --- a/test/test_insert.sh +++ b/test/test_insert.sh @@ -211,3 +211,6 @@ new "adv list add leaf-list" testrun "$x0" "32" rm -rf $dir + +# unset conditional parameters +unset clixon_util_insert diff --git a/test/test_install.sh b/test/test_install.sh index 78b85ecb..7bc4612e 100755 --- a/test/test_install.sh +++ b/test/test_install.sh @@ -63,3 +63,6 @@ if [ -n "$l" ]; then fi rm -rf $dir + +# unset conditional parameters +unset make diff --git a/test/test_instance_id.sh b/test/test_instance_id.sh new file mode 100755 index 00000000..d905562a --- /dev/null +++ b/test/test_instance_id.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +# INSTANCE-ID tests + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +: ${clixon_util_path:=clixon_util_path -D $DBG} + +# Number of list/leaf-list entries +: ${nr:=100} + +# XML file (alt provide it in stdin after xpath) +for (( i=1; i<9; i++ )); do + eval xml$i=$dir/xml$i.xml +done +ydir=$dir/yang + +if [ ! -d $ydir ]; then + mkdir $ydir +fi + +# XPATH binary search in ordered-by system lists +cat < $ydir/moda.yang +module moda{ + namespace "urn:example:a"; + prefix a; + container x1{ + description "list with single string key"; + list y{ + ordered-by system; + key k1; + leaf k1{ + type string; + } + leaf z{ + type string; + } + } + } + container x2{ + description "list with single int key"; + list y{ + ordered-by system; + key k1; + leaf k1{ + type uint32; + } + leaf z{ + type string; + } + } + } + container x3{ + description "list with double string keys"; + list y{ + ordered-by system; + key "k1 k2"; + leaf k1{ + type string; + } + leaf k2{ + type string; + } + leaf z{ + type string; + } + } + } + container x4{ + description "leaf-list with int key"; + leaf-list y{ + type string; + } + } + list x5{ + ordered-by system; + description "Direct under root"; + key "k1"; + leaf k1{ + type string; + } + leaf z{ + type string; + } + } + + augment "/b:x6/b:yy" { + list y{ + ordered-by system; + key "k1 k2"; + leaf k1{ + type string; + } + leaf k2{ + type string; + } + leaf-list z{ + type string; + } + } + } + container x7{ + description "Single list, ordered by user"; + list y{ + ordered-by user; + key k1; + leaf k1{ + type string; + } + leaf z{ + type string; + } + } + } + container x8{ + description "Single list state data"; + config false; + list y{ + key k1; + leaf k1{ + type string; + } + leaf z{ + type string; + } + } + } +} +EOF + +# This is for augment usecase +cat < $ydir/modb.yang +module modb{ + namespace "urn:example:b"; + prefix b; + container x6{ + description "deep tree and augment"; + list yy{ + ordered-by system; + key "kk1 kk2"; + leaf kk1{ + type string; + } + leaf kk2{ + type string; + } + leaf-list zz{ + type string; + } + } + } +} +EOF + +rnd=$(( ( RANDOM % $nr ) )) + +# Single string key +new "generate list with $nr single string key to $xml1" +echo -n '' > $xml1 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ifoo$i" >> $xml1 +done +echo -n '' >> $xml1 + +new "instance-id single string key k1=a$rnd" +echo "$clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:k1=\"a$rnd\"]" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:k1=\"a$rnd\"])" 0 "^0: a$rndfoo$rnd$" + +new "instance-id single string key /x1" +echo "$clixon_util_path -f $xml1 -y $ydir -p /a:x1" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1)" 0 "0: a0foo0a1foo1" # Assume at least two elements + +new "instance-id position specific position 5" +echo "$clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[5]" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[5])" 0 "0: a13foo13" # sort alphanumerivc wrong 1,10,2 + +new "instance-id single string key omit key" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y)" 0 '^0: a0foo0 +1: a0foo0' + +# Fails and error handling +new "instance-id single string search non-index" +echo "$clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:z=\"foo$rnd\"]" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:z=\"foo$rnd\"] )" 0 "a$rndfoo$rnd$" + +new "instance-id single string search non-index (two variables, index first)" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:k1=\"a$rnd\"][a:z=\"foo$rnd\"] )" 0 "a$rndfoo$rnd$" + +new "instance-id single string search non-index (two variables, index last)" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:z=\"foo$rnd\"][a:k1=\"a$rnd\"] )" 0 "a$rndfoo$rnd$" + +new "instance-id single string wrong module, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /xxx:x1/a:y[a:k1=\"a$rnd\"] 2> /dev/null)" 255 '^$' + +new "instance-id single string no module, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /x1/a:y[a:k1=\"a$rnd\"] 2> /dev/null)" 255 '^$' + +new "instance-id single string no sub-prefixes, notfound" +echo "$clixon_util_path -f $xml1 -y $ydir -p /a:x1/y[k1=\"a$rnd\"]" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/y[k1=\"a$rnd\"] 2> /dev/null)" 255 '^$' + +new "instance-id single string two keys, notfound" +expectpart "$($clixon_util_path -f $xml1 -y $ydir -p /a:x1/a:y[a:k1=a$rnd][a:k2=a$rnd] 2> /dev/null)" 255 '^$' + +# Single int key +new "generate list with $nr single int key to $xml2" +echo -n '' > $xml2 +for (( i=0; i<$nr; i++ )); do + echo -n "$ifoo$i" >> $xml2 +done +echo -n '' >> $xml2 + +new "instance-id single int key k1=$rnd" +echo "$clixon_util_path -f $xml2 -y $ydir -p /a:x2/a:y[a:k1=\"$rnd\"]" +expectpart "$($clixon_util_path -f $xml2 -y $ydir -p /a:x2/a:y[a:k1=\"$rnd\"])" 0 "^0: $rndfoo$rnd$" + +# Double string key +new "generate list with $nr double string keys to $xml3 (two k2 entries per k1 key)" +echo -n '' > $xml3 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ia$ifoo$i" >> $xml3 + echo -n "a$ib$ifoob$i" >> $xml3 +done +# Add two rules with empty k2 string +echo -n "a0foo0" >> $xml3 +echo -n "a1foo1" >> $xml3 +echo -n '' >> $xml3 + +new "instance-id double string key k1=a$rnd k2=b$rnd" +echo "$clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[k1=\"a$rnd\"][k2=\"b$rnd\"]" +expectpart "$($clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[k1=\"a$rnd\"][k2=\"b$rnd\"])" 0 "0: a$rndb$rndfoob$rnd" + +new "instance-id double string key k1=a$rnd, - empty k2 string" +echo "$clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[k1=\"a1\"][k2=\"\"]" +expectpart "$($clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[k1=\"a1\"][k2=\"\"])" 0 "0: a1foo1" + +new "instance-id double string key k1=a$rnd, - no k2 string - three matches" +echo "$clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[k1=\"a1\"]" +expecteq "$($clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[k1=\"a1\"])" 0 "0: a1foo1 +1: a1a1foo1 +2: a1b1foob1" + +new "instance-id double string specific position 5" +echo "$clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[5]" +expectpart "$($clixon_util_path -f $xml3 -y $ydir -p /a:x3/a:y[5])" 0 "0: a1b1foob1" # sort alphanumerivc wrong 1,10,2 + +# Leaf-list +new "generate leaf-list int keys to $xml4" +echo -n '' > $xml4 +for (( i=0; i<$nr; i++ )); do + echo -n "a$i" >> $xml4 +done +echo -n '' >> $xml4 + +new "instance-id leaf-list k1=a$rnd" +echo "$clixon_util_path -f $xml4 -y $ydir -p /a:x4/a:y[.=\"a$rnd\"]" +expectpart "$($clixon_util_path -f $xml4 -y $ydir -p /a:x4/a:y[.=\"a$rnd\"])" 0 "^0: a$rnd$" + +# Single string key direct under root +new "generate list with $nr single string key to $xml5" +echo -n '' > $xml5 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ifoo$i" >> $xml5 +done + +new "instance-id direct under root single string key k1=a$rnd" +echo "" +expectpart "$($clixon_util_path -f $xml5 -y $ydir -p /a:x5[k1=\"a$rnd\"])" 0 "^0: a$rndfoo$rnd$" + +# Depth and augment +# Deep augmented xml path +new "generate deep list with augment" +echo -n '' > $xml6 +for (( i=0; i<$nr; i++ )); do + echo -n "b$ib$ifoo$i" >> $xml6 + for (( j=0; j<3; j++ )); do + echo -n "a$ja$jfoo$j" >> $xml6 + done + echo -n "" >> $xml6 +done +echo -n '' >> $xml6 + +new "instance-id double string key b$rnd,b$rnd in mod b" +echo "$clixon_util_path -f $xml6 -y $ydir -p /b:x6/b:yy[kk1=\"b$rnd\"][kk2=\"b$rnd\"]" +expectpart "$($clixon_util_path -f $xml6 -y $ydir -p /b:x6/b:yy[kk1=\"b$rnd\"][kk2=\"b$rnd\"])" 0 "0: b$rndb$rndfoo$rnda0a0foo0a1a1foo1a2a2foo2" + +new "instance-id double string key a$rnd,b$rnd in modb + augmented in moda" +expectpart "$($clixon_util_path -f $xml6 -y $ydir -p /b:x6/b:yy[kk1=\"b$rnd\"][kk2=\"b$rnd\"]/a:y[k1=\"a1\"][k2=\"a1\"]/a:z[.=\"foo1\"])" 0 "0: foo1" + +# Single list ordered by user +new "generate list with $nr single string key to $xml7" +echo -n '' > $xml7 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ifoo$i" >> $xml7 +done +echo -n '' >> $xml7 + +new "instance-id single string key k1=a$rnd ordered by user" +echo "$clixon_util_path -f $xml7 -y $ydir -p /a:x7/a:y[a:k1=\"a$rnd\"]" +expectpart "$($clixon_util_path -f $xml7 -y $ydir -p /a:x7/a:y[a:k1=\"a$rnd\"])" 0 "^0: a$rndfoo$rnd$" + +# Single list state data (non-config) +new "generate list with $nr single string key to $xml8" +echo -n '' > $xml8 +for (( i=0; i<$nr; i++ )); do + echo -n "a$ifoo$i" >> $xml8 +done +echo -n '' >> $xml8 + +new "instance-id single string key k1=a$rnd ordered by user" +expectpart "$($clixon_util_path -f $xml8 -y $ydir -p /a:x8/a:y[a:k1=\"a$rnd\"])" 0 "^0: a$rndfoo$rnd$" + +rm -rf $dir + +unset nr +unset clixon_util_path # for other script reusing it + + diff --git a/test/test_json.sh b/test/test_json.sh index 7779ebc5..da2eafe1 100755 --- a/test/test_json.sh +++ b/test/test_json.sh @@ -129,3 +129,7 @@ expecteofx "$clixon_util_json -j -y $fyang" 0 "$JSON" "$JSON" fi rm -rf $dir + +# unset conditional parameters +unset clixon_util_json +unset clixon_util_xml diff --git a/test/test_leafref.sh b/test/test_leafref.sh index fabad06b..a76e53dc 100755 --- a/test/test_leafref.sh +++ b/test/test_leafref.sh @@ -21,7 +21,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF @@ -141,7 +140,7 @@ new "leafref add non-existing ref" expecteof "$clixon_netconf -qf $cfg" 0 'eth3
10.0.4.6
]]>]]>' '^]]>]]>$' new "leafref validate" -expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^applicationbad-elementeth3errorLeafref validation failed: No such leaf]]>]]>$' +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^applicationbad-elementeth3errorLeafref validation failed: No leaf eth3 matching path /if:interfaces/if:interface/if:name]]>]]>$' #new "leafref wrong ref" #expecteof "$clixon_netconf -qf $cfg" 0 'eth3
10.0.4.6
]]>]]>' '^]]>]]>$' @@ -171,7 +170,7 @@ new "leafref delete leaf" expecteof "$clixon_netconf -qf $cfg" 0 'eth0]]>]]>' '^' new "leafref validate (should fail)" -expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^applicationbad-elementeth0errorLeafref validation failed: No such leaf]]>]]>$' +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^applicationbad-elementeth0errorLeafref validation failed: No leaf eth0 matching path /if:interfaces/if:interface/if:name]]>]]>$' new "leafref discard-changes" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" diff --git a/test/test_leafref_augment.sh b/test/test_leafref_augment.sh new file mode 100755 index 00000000..ffdca878 --- /dev/null +++ b/test/test_leafref_augment.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash +# Yang leafref + augment + grouping taking from a more complex netgate errorcase +# A main yang spec: leafref +# and a secondary yang spec: augment +# module leafref has a primary construct (sender) and a leafref typedef +# module augment has an augment and a grouping from where it uses the leafref typedef +# Which means that you should first have xml such as: +# +# x +# +# and you can then track it via for example (extra levels for debugging): +# +# y +# # original +# # augment +# # grouping +# +# x <---- +# +# +# +# +# +# +# There is also test for using prefixes or not, as well as swithcing prefix between the main module and +# it import statement. + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +cfg=$dir/conf_yang.xml +fyang1=$dir/leafref.yang +fyang2=$dir/augment.yang + +cat < $cfg + + $cfg + $dir + /usr/local/share/clixon + $fyang2 + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/$APPNAME/$APPNAME.pidfile + /usr/local/var/$APPNAME + +EOF + +# NOTE prefix "example" used in module different from "ex" used in mport of that module +cat < $fyang1 +module leafref{ + yang-version 1.1; + namespace "urn:example:example"; + prefix example; + typedef sender-ref { + description "For testing leafref across augment and grouping"; + type leafref { + path "/ex:sender/ex:name"; + } + } + typedef sender-ref-local { + description "For testing leafref local"; + type leafref { + path "/example:sender/example:name"; + } + } + list sender{ + key name; + leaf name{ + type string; + } + container stub{ + description "Here is where augmentation is done"; + } + leaf ref{ + description "top-level ref (wrong prefix)"; + type sender-ref; + } + leaf ref-local{ + description "top-level ref (right prefix)"; + type sender-ref-local; + } + + } +} +EOF + +cat < $fyang2 +module augment{ + yang-version 1.1; + namespace "urn:example:augment"; + prefix aug; + import leafref { + description "Note different from canonical (leafref module own prefix is 'example'"; + prefix "ex"; + } + grouping attributes { + container track{ + description "replicates original structure but only references original"; + list sender{ + description "reference using path in typedef"; + key name; + leaf name{ + type ex:sender-ref; + } + } + list senderdata{ + description "reference using path inline in data (not typedef)"; + key name; + leaf name{ + type leafref { + path "/ex:sender/ex:name"; + } + } + } + } + } + augment "/ex:sender/ex:stub" { + description "Main leafref/sender stub."; + container extra{ + presence "ensuring it is there"; + uses attributes; + } + } +} +EOF + + +new "test params: -f $cfg" + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg + + new "waiting" + wait_backend +fi + +# Test top-level, default prefix, wring leafref prefix and typedef path +XML=$(cat < + x + + + y + x + +EOF +) + +new "leafref augment+leafref config top-level" +expecteof "$clixon_netconf -qf $cfg" 0 "$XML]]>]]>" '^]]>]]>$' + +new "leafref augment+leafref validate top-level wrong prefix" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^applicationbad-elementxerrorLeafref validation failed: No leaf x matching path /ex:sender/ex:name]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# Test top-level, default prefix, correct leafref and typedef path +XML=$(cat < + x + + + y + x + +EOF +) + +# Use augment + explicit prefixes, correct leafref and typedef path +XML=$(cat < + x + + + y + + + + + x + + + + + +EOF +) + +new "leafref augment+leafref config" +expecteof "$clixon_netconf -qf $cfg" 0 "$XML]]>]]>" '^]]>]]>$' + +new "leafref augment+leafref validate" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# Use augment, default prefixes, wrong leafref and typedef path +XML=$(cat < + x + + + y + + + + + xxx + + + + + +EOF +) + +new "leafref augment+leafref config wrong ref" +expecteof "$clixon_netconf -qf $cfg" 0 "$XML]]>]]>" '^]]>]]>$' + +new "leafref augment+leafref validate wrong ref" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^applicationbad-elementxxxerrorLeafref validation failed: No leaf xxx matching path /ex:sender/ex:name]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# Use augment, default prefixes, correct leafref and in-data path +XML=$(cat < + x + + + y + + + + + x + + + + + +EOF +) + +new "leafref augment+leafref config in-data" +expecteof "$clixon_netconf -qf $cfg" 0 "$XML]]>]]>" '^]]>]]>$' + +new "leafref augment+leafref validate in-data" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + + + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=$(pgrep -u root -f clixon_backend) +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +rm -rf $dir diff --git a/test/test_leafref_state.sh b/test/test_leafref_state.sh new file mode 100755 index 00000000..5f6acb8a --- /dev/null +++ b/test/test_leafref_state.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Yang leafref + state tests +# The difficulty here is a "leafref" in state data that references config-data. +# Problem being that config-data from running needs to be mergedwith state data and filtered/cropped +# correctly +# +# The YANG has two parts, one config part (sender-config) and one state part (sender-state) +# The leafref in the sender-state part references a leaf in the sender-config part +# Netconf tests are made to get state, state+config, using content attribute config/nonconfig/all +# with different paths. +# Using the -sS state capability of the main example, that is why CLICON_BACKEND_DIR is +# /usr/local/lib/$APPNAME/backend so that the main backend plugins is included. + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +cfg=$dir/conf_yang.xml +fstate=$dir/state.xml +fyang=$dir/leafref.yang + +cat < $cfg + + $cfg + $dir + /usr/local/share/clixon + $fyang + /usr/local/lib/$APPNAME/backend + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + false + /usr/local/var/$APPNAME/$APPNAME.pidfile + /usr/local/var/$APPNAME + +EOF + +# NOTE prefix "example" used in module different from "ex" used in mport of that module +cat < $fyang +module leafref{ + yang-version 1.1; + namespace "urn:example:example"; + prefix ex; + list sender-config{ + description "Main config of senders"; + key name; + leaf name{ + type string; + } + } + list sender-state{ + description "State referencing configured senders"; + config false; + key ref; + leaf ref{ + type leafref { + path "/ex:sender-config/ex:name"; + } + } + } +} +EOF + +cat < $fstate + + x + +EOF + +new "test params: -f $cfg -- -sS $fstate" + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg -- -sS $fstate" + start_backend -s init -f $cfg -- -sS $fstate + new "waiting" + wait_backend +fi + +# Test top-level, default prefix, wring leafref prefix and typedef path +XML=$(cat < + x + +EOF +) + +new "leafref config sender x" +expecteof "$clixon_netconf -qf $cfg" 0 "$XML]]>]]>" '^]]>]]>$' + +new "netconf commit" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# Get path=/, state vs config +new "netconf get / config+state" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^xx]]>]]>$' + +new "netconf get / state-only" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +new "netconf get / config-only" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +# Get path=/sender-state, state vs config +new "netconf get /sender-state config+state" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +new "netconf get /sender-state state-only" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +new "netconf get /sender-state config-only" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^]]>]]>$' + +# Get path=/sender-config, state vs config +new "netconf get /sender-config config+state" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +new "netconf get /sender-config state-only" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^]]>]]>$' + +new "netconf get /sender-config config-only" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^x]]>]]>$' + +# delete x, add y +XML=$(cat < + x + + + y + +EOF +) +# Negative tests, start with remove x and and add y instead +new "leafref config delete sender x add y" +expecteof "$clixon_netconf -qf $cfg" 0 "$XML]]>]]>" '^]]>]]>$' + +new "netconf commit" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# Leafref wrong +new "netconf get / config+state should fail" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^applicationoperation-failedxerrorLeafref validation failed: No leaf x matching path /ex:sender-config/ex:name Internal error, state callback returned invalid XML]]>]]>$' + +new "netconf get / state-only should fail" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^applicationoperation-failedxerrorLeafref validation failed: No leaf x matching path /ex:sender-config/ex:name Internal error, state callback returned invalid XML]]>]]>$' + +new "netconf get / config-only ok" +expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' '^y]]>]]>$' + + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=$(pgrep -u root -f clixon_backend) +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + +rm -rf $dir diff --git a/test/test_minmax.sh b/test/test_minmax.sh index 6599ed10..eee5da2c 100755 --- a/test/test_minmax.sh +++ b/test/test_minmax.sh @@ -20,7 +20,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_nacm.sh b/test/test_nacm.sh index 2da70b4b..605f9300 100755 --- a/test/test_nacm.sh +++ b/test/test_nacm.sh @@ -28,7 +28,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/lib/$APPNAME/backend /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME false internal diff --git a/test/test_nacm_credentials.sh b/test/test_nacm_credentials.sh index f67c2bfa..f50443f5 100755 --- a/test/test_nacm_credentials.sh +++ b/test/test_nacm_credentials.sh @@ -227,3 +227,6 @@ new "Credentials: mode=except, fam=UNIX user=admin sudo" testrun except $USER IPv4 127.0.0.1 "$ERROR" "" rm -rf $dir + +# unset conditional parameters +unset clixon_util_socket diff --git a/test/test_nacm_default.sh b/test/test_nacm_default.sh index 84ef39b2..eab10de3 100755 --- a/test/test_nacm_default.sh +++ b/test/test_nacm_default.sh @@ -27,7 +27,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/lib/$APPNAME/backend /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 $dir /usr/local/lib/xmldb/text.so false @@ -197,3 +196,6 @@ new "nacm enabled, exec default permit, write permit (expect fail)" testrun true deny permit permit 2 0 2 rm -rf $dir + +# unset conditional parameters +unset format diff --git a/test/test_nacm_ext.sh b/test/test_nacm_ext.sh index da066b30..759d6092 100755 --- a/test/test_nacm_ext.sh +++ b/test/test_nacm_ext.sh @@ -31,7 +31,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock false /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME false external diff --git a/test/test_nacm_module_read.sh b/test/test_nacm_module_read.sh index 4d2a3550..55ea2630 100755 --- a/test/test_nacm_module_read.sh +++ b/test/test_nacm_module_read.sh @@ -42,7 +42,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/lib/$APPNAME/backend /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME false internal diff --git a/test/test_nacm_protocol.sh b/test/test_nacm_protocol.sh index 5f8d64d8..edc36884 100755 --- a/test/test_nacm_protocol.sh +++ b/test/test_nacm_protocol.sh @@ -48,7 +48,6 @@ cat < $cfg /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/lib/$APPNAME/backend /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME false internal diff --git a/test/test_netconf.sh b/test/test_netconf.sh index c451f061..180a2b4d 100755 --- a/test/test_netconf.sh +++ b/test/test_netconf.sh @@ -28,7 +28,6 @@ cat < $cfg $APPNAME $dir/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_openconfig.sh b/test/test_openconfig.sh index 1f1158ae..f91dcd12 100755 --- a/test/test_openconfig.sh +++ b/test/test_openconfig.sh @@ -70,7 +70,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME true diff --git a/test/test_order.sh b/test/test_order.sh index 39f7c3ac..fbf0aa6d 100755 --- a/test/test_order.sh +++ b/test/test_order.sh @@ -403,3 +403,6 @@ fi stop_backend -f $cfg rm -rf $dir + +# unset conditional parameters +unset format diff --git a/test/test_pattern.sh b/test/test_pattern.sh index 003ffa5d..bbcf0115 100755 --- a/test/test_pattern.sh +++ b/test/test_pattern.sh @@ -37,7 +37,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME true @@ -753,3 +752,6 @@ if [ $BE -ne 0 ]; then fi rm -rf $dir + +# unset conditional parameters +unset regex diff --git a/test/test_perf.sh b/test/test_perf.sh index 063e5d02..c9408ff7 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -60,8 +60,6 @@ cat < $cfg example /usr/local/lib/example/cli /usr/local/lib/example/clispec - 1 - VARS 0 ietf-netconf:startup @@ -246,5 +244,9 @@ fi # kill backend stop_backend -f $cfg - rm -rf $dir + +# unset conditional parameters +unset format +unset perfnr +unset perfreg diff --git a/test/test_perf_startup.sh b/test/test_perf_startup.sh index 19351488..3632abf8 100755 --- a/test/test_perf_startup.sh +++ b/test/test_perf_startup.sh @@ -48,8 +48,6 @@ cat < $cfg example /usr/local/lib/example/cli /usr/local/lib/example/clispec - 1 - VARS 0 ietf-netconf:startup @@ -119,3 +117,6 @@ for mode in startup running; do done rm -rf $dir + +# unset conditional parameters +unset perfnr diff --git a/test/test_perf_state.sh b/test/test_perf_state.sh index 05935413..31e1f4de 100755 --- a/test/test_perf_state.sh +++ b/test/test_perf_state.sh @@ -38,8 +38,6 @@ cat < $cfg example /usr/local/lib/example/cli /usr/local/lib/example/clispec - 1 - VARS 0 ietf-netconf:startup @@ -154,5 +152,11 @@ fi # kill backend stop_backend -f $cfg - rm -rf $dir + +# unset conditional parameters +unset format +unset perfnr +unset perfreg + + diff --git a/test/test_perf_xml.sh b/test/test_perf_xml.sh index 17d7d6ce..216af1ef 100755 --- a/test/test_perf_xml.sh +++ b/test/test_perf_xml.sh @@ -26,3 +26,7 @@ expecteof_file "time $clixon_util_xml" 0 "$fxml" rm -rf $dir +# unset conditional parameters +unset clixon_util_xml +unset perfnr + diff --git a/test/test_restconf.sh b/test/test_restconf.sh index 10275f56..4f776bca 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -27,7 +27,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME true diff --git a/test/test_rpc.sh b/test/test_rpc.sh index 492d2a7a..ee4100f7 100755 --- a/test/test_rpc.sh +++ b/test/test_rpc.sh @@ -27,7 +27,6 @@ cat < $cfg false /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_sock.sh b/test/test_sock.sh index ca702790..dd6d8941 100755 --- a/test/test_sock.sh +++ b/test/test_sock.sh @@ -91,3 +91,6 @@ testrun IPv4 127.0.0.1 #testrun IPv6 ::1 7878 rm -rf $dir + +# unset conditional parameters +unset clixon_util_socket diff --git a/test/test_startup.sh b/test/test_startup.sh index 876e6ace..f785d8a7 100755 --- a/test/test_startup.sh +++ b/test/test_startup.sh @@ -189,3 +189,6 @@ if [ $valgrindtest -ne 2 ]; then fi rm -rf $dir + +# unset conditional parameters +unset format diff --git a/test/test_stream.sh b/test/test_stream.sh index 280f84fa..6b60c552 100755 --- a/test/test_stream.sh +++ b/test/test_stream.sh @@ -302,3 +302,6 @@ fi stop_backend -f $cfg rm -rf $dir + +# unset conditional parameters +unset clixon_util_stream diff --git a/test/test_submodule.sh b/test/test_submodule.sh index 823de222..45c549a6 100755 --- a/test/test_submodule.sh +++ b/test/test_submodule.sh @@ -41,7 +41,6 @@ cat < $cfg example_backend.so$ /usr/local/lib/$APPNAME/restconf false - 1 /usr/local/var/$APPNAME false diff --git a/test/test_transaction.sh b/test/test_transaction.sh index 4d7e7723..5d67e00f 100755 --- a/test/test_transaction.sh +++ b/test/test_transaction.sh @@ -76,7 +76,6 @@ cat < $cfg $APPNAME $dir/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME $format @@ -319,3 +318,6 @@ fi stop_backend -f $cfg rm -rf $dir + +# unset conditional parameters +unset format diff --git a/test/test_type.sh b/test/test_type.sh index 2ef9a177..0e30b050 100755 --- a/test/test_type.sh +++ b/test/test_type.sh @@ -226,7 +226,6 @@ testrun(){ $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME $dbcache $format diff --git a/test/test_type_range.sh b/test/test_type_range.sh index 6fed607d..c6dc66bc 100755 --- a/test/test_type_range.sh +++ b/test/test_type_range.sh @@ -180,13 +180,40 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME $format EOF -# Type range tests. +# Test invalid numbers or out of range errors for each type. Check error message on validation. +# These are the built-in ranges, typed ranges are checked in testrange +# Parameters: 1: type (eg uint8) +# 2: value to test +# 3: min value (for errmsg) +# 4: max value (for errmsg) +# 5: post (eg .000 - special for decimal64, others should have "") (NYI) +testbuiltin(){ + t=$1 + val=$2 + rmin=$3 + rmax=$4 + post=$5 + + errmsg="Number $val$post out of range: $rmin$post - $rmax$post" + + new "Netconf set invalid $t leaf" + echo "$clixon_netconf -qf $cfg" + echo "$val]]>]]>" + expecteof "$clixon_netconf -qf $cfg" 0 "$val]]>]]>" "^]]>]]>$" + + new "netconf validate invalid range" + expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^applicationbad-elementr$terror$errmsg]]>]]>$" + + new "discard" + expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" +} + +# Type explicit typed range tests. # Parameters: 1: type (eg uint8) # 2: val OK # 3: eval Invalid value @@ -205,7 +232,7 @@ testrange(){ fi new "generated cli set $t leaf invalid" - expectpart "$(clixon_cli -1f $cfg -l o set l$t $eval)" 255 "$errmsg" + expectpart "$($clixon_cli -1f $cfg -l o set l$t $eval)" 255 "$errmsg" new "generated cli set $t leaf OK" expectfn "$clixon_cli -1f $cfg -l o set l$t $val" 0 '^$' @@ -243,28 +270,6 @@ testrange(){ expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" } -# Type unlimited value range test. Only test invalid number out of range of type -# Parameters: 1: type (eg uint8) -# 2: val -# 3: post (eg .000 - special for decimal64, others should have "") -testunlimit(){ - t=$1 - val=$2 - rmin=$3 - rmax=$4 - post=$5 - - errmsg="Number $val$post out of range: $rmin$post - $rmax$post" - - new "Netconf set invalid $t leaf" - expecteof "$clixon_netconf -qf $cfg" 0 "$val]]>]]>" "^]]>]]>$" - - new "netconf validate invalid range" - expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^applicationbad-elementr$terror$errmsg]]>]]>$" - - new "discard" - expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" -} if [ $BE -ne 0 ]; then new "kill old backend" @@ -282,15 +287,28 @@ fi new "test params: -f $cfg" # Test all int types -testunlimit int8 300 -128 127 "" -testunlimit int16 73000 -32768 32767 "" -testunlimit int32 4900000000 -2147483648 2147483647 "" -testunlimit int64 49739274983274983274983274 -9223372036854775808 9223372036854775807 "" -testunlimit uint8 300 0 255 "" -testunlimit uint16 73000 0 65535 "" -testunlimit uint32 4900000000 0 4294967295 "" -testunlimit uint64 49739274983274983274983274 0 18446744073709551615 "" -#testunlimit decimal64 49739274983274983274983274 -9223372036854775808 9223372036854775807 ".000" +testbuiltin int8 300 -128 127 "" +testbuiltin int8 -300 -128 127 "" +testbuiltin int16 73000 -32768 32767 "" +testbuiltin int16 -73000 -32768 32767 "" +testbuiltin int32 4900000000 -2147483648 2147483647 "" +testbuiltin int32 -4900000000 -2147483648 2147483647 "" +testbuiltin int64 49739274983274983274983274 -9223372036854775808 9223372036854775807 "" +testbuiltin int64 -49739274983274983274983274 -9223372036854775808 9223372036854775807 "" + +testbuiltin uint8 300 0 255 "" +testbuiltin uint8 -300 0 255 "" +testbuiltin uint8 -1 0 255 "" +testbuiltin uint16 73000 0 65535 "" +testbuiltin uint16 -73000 0 65535 "" +testbuiltin uint16 -1 0 65535 "" +testbuiltin uint32 4900000000 0 4294967295 "" +testbuiltin uint32 -4900000000 0 4294967295 "" +testbuiltin uint32 -1 0 4294967295 "" +testbuiltin uint64 49739274983274983274983274 0 18446744073709551615 "" +testbuiltin uint64 -49739274983274983274983274 0 18446744073709551615 "" +testbuiltin uint64 -1 0 18446744073709551615 "" +#testbuiltin decimal64 49739274983274983274983274 -9223372036854775808 9223372036854775807 ".000" # Test all int types for t in int8 int16 int32 int64 uint8 uint16 uint32 uint64; do @@ -315,3 +333,6 @@ if [ $BE -ne 0 ]; then fi rm -rf $dir + +# unset conditional parameters +unset format diff --git a/test/test_union.sh b/test/test_union.sh index 0e1c6cce..ce0c754e 100755 --- a/test/test_union.sh +++ b/test/test_union.sh @@ -24,7 +24,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_unique.sh b/test/test_unique.sh index 6b68bd0c..5b983e0b 100755 --- a/test/test_unique.sh +++ b/test/test_unique.sh @@ -25,7 +25,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_when_must.sh b/test/test_when_must.sh index fecd986b..14d1483a 100755 --- a/test/test_when_must.sh +++ b/test/test_when_must.sh @@ -21,7 +21,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_with_default.sh b/test/test_with_default.sh index eaea16b7..c6ba8c33 100755 --- a/test/test_with_default.sh +++ b/test/test_with_default.sh @@ -34,7 +34,6 @@ cat < $cfg $APPNAME $dir/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME EOF diff --git a/test/test_xml.sh b/test/test_xml.sh index a25184d4..f800a4ed 100755 --- a/test/test_xml.sh +++ b/test/test_xml.sh @@ -180,3 +180,5 @@ expecteof "$clixon_util_xml -o" 0 "$XML" "$XML" rm -rf $dir +# unset conditional parameters +unset clixon_util_xml diff --git a/test/test_xpath.sh b/test/test_xpath.sh index 14466b09..f2bbbcfe 100755 --- a/test/test_xpath.sh +++ b/test/test_xpath.sh @@ -10,11 +10,6 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi xml=$dir/xml.xml xml2=$dir/xml2.xml xml3=$dir/xml3.xml -ydir=$dir/yang - -if [ ! -d $ydir ]; then - mkdir $ydir -fi cat < $xml @@ -81,7 +76,6 @@ cat < $xml3 EOF new "xpath /" -echo "$clixon_util_xpath -f $xml -p /" expecteof "$clixon_util_xpath -f $xml -p /" 0 "" "^nodeset:0:429922$" new "xpath /aaa" @@ -209,3 +203,6 @@ new "xpath derived-from-or-self" expecteof "$clixon_util_xpath -f $xml3 -p 'derived-from-or-self(../../change-operation,modify)'" 0 "" "derived-from-or-self" rm -rf $dir + +# unset conditional parameters +unset clixon_util_xpath diff --git a/test/test_xpath_canonical.sh b/test/test_xpath_canonical.sh index a83feffc..7f538a95 100755 --- a/test/test_xpath_canonical.sh +++ b/test/test_xpath_canonical.sh @@ -63,3 +63,6 @@ new "xpath canonical form (wrong namespace should fail)" expectpart "$($clixon_util_xpath -c -y $ydir -p /i:x/j:y -n i:urn:example:c -n j:urn:example:b)" 255 rm -rf $dir + +# unset conditional parameters +unset clixon_util_xpath diff --git a/test/test_yang_extension.sh b/test/test_yang_extension.sh index cb359851..dc47d434 100755 --- a/test/test_yang_extension.sh +++ b/test/test_yang_extension.sh @@ -37,7 +37,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME true diff --git a/test/test_yang_load.sh b/test/test_yang_load.sh index 7951a145..17e25ae6 100755 --- a/test/test_yang_load.sh +++ b/test/test_yang_load.sh @@ -70,7 +70,6 @@ cat < $cfg /usr/local/lib/$APPNAME/clispec /usr/local/lib/$APPNAME/cli $APPNAME - 1 EOF @@ -124,7 +123,6 @@ cat < $cfg /usr/local/lib/$APPNAME/clispec /usr/local/lib/$APPNAME/cli $APPNAME - 1 EOF diff --git a/test/test_yang_models.sh b/test/test_yang_models.sh index 0e3fec47..2c1535a5 100755 --- a/test/test_yang_models.sh +++ b/test/test_yang_models.sh @@ -45,8 +45,6 @@ cat < $cfg $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 - 1 /usr/local/var/$APPNAME true diff --git a/test/test_yang_namespace.sh b/test/test_yang_namespace.sh index 6c700a74..f56d0b8e 100755 --- a/test/test_yang_namespace.sh +++ b/test/test_yang_namespace.sh @@ -25,7 +25,6 @@ cat < $cfg false /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile - 1 /usr/local/var/$APPNAME true diff --git a/util/Makefile.in b/util/Makefile.in index eb06802e..bc05aaa8 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -1,7 +1,8 @@ # # ***** BEGIN LICENSE BLOCK ***** # -# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren +# Copyright (C) 2017-2020 Olof Hagsand # # This file is part of CLIXON # @@ -72,6 +73,7 @@ APPSRC = clixon_util_xml.c APPSRC += clixon_util_json.c APPSRC += clixon_util_yang.c APPSRC += clixon_util_xpath.c +APPSRC += clixon_util_path.c APPSRC += clixon_util_datastore.c APPSRC += clixon_util_insert.c APPSRC += clixon_util_regexp.c @@ -106,6 +108,9 @@ clixon_util_yang: clixon_util_yang.c $(LIBDEPS) clixon_util_xpath: clixon_util_xpath.c $(LIBDEPS) $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ +clixon_util_path: clixon_util_path.c $(LIBDEPS) + $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ + clixon_util_datastore: clixon_util_datastore.c $(LIBDEPS) $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ diff --git a/util/clixon_util_path.c b/util/clixon_util_path.c new file mode 100644 index 00000000..ae79483a --- /dev/null +++ b/util/clixon_util_path.c @@ -0,0 +1,269 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2020 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * "api-path" is "URI-encoded path expression" definition in RFC8040 3.5.3 + * "Instance-identifier" is a subset of XML Xpaths and defined in Yang, used in NACM for example. + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* cligen */ +#include + +/* clixon */ +#include "clixon/clixon.h" + +/* Command line options to be passed to getopt(3) */ +#define UTIL_PATH_OPTS "hD:f:ap:y:Y:" + +static int +usage(char *argv0) +{ + fprintf(stderr, "usage:%s [options]\n" + "where options are\n" + "\t-h \t\tHelp\n" + "\t-D \tDebug\n" + "\t-f \tXML file\n" + "\t-a \t\tUse API-PATH (default INSTANCE-ID)\n" + "\t-p \tPATH string\n" + "\t-y \tYang filename or dir (load all files)\n" + "\t-Y \tYang dirs (can be several)\n" + "and the following extra rules:\n" + "\tif -f is not given, XML input is expected on stdin\n" + "\tif -p is not given, is expected as the first line on stdin\n" + "This means that with no arguments, and XML is expected on stdin.\n", + argv0 + ); + exit(0); +} + +int +main(int argc, + char **argv) +{ + int retval = -1; + char *argv0 = argv[0]; + int i; + cxobj *x = NULL; + cxobj **xvec = NULL; + size_t xlen; + int c; + int len; + char *buf = NULL; + int ret; + int fd = 0; /* unless overriden by argv[1] */ + char *yang_file_dir = NULL; + yang_stmt *yspec = NULL; + char *path = NULL; + char *filename; + cbuf *cb = NULL; + int api_path_p = 0; /* api-path or instance-id */ + clicon_handle h; + struct stat st; + + clicon_log_init("api-path", LOG_DEBUG, CLICON_LOG_STDERR); + if ((h = clicon_handle_init()) == NULL) + goto done; + optind = 1; + opterr = 0; + while ((c = getopt(argc, argv, UTIL_PATH_OPTS)) != -1) + switch (c) { + case 'h': + usage(argv0); + break; + case 'D': + if (sscanf(optarg, "%d", &debug) != 1) + usage(argv0); + break; + case 'f': /* XML file */ + filename = optarg; + if ((fd = open(filename, O_RDONLY)) < 0){ + clicon_err(OE_UNIX, errno, "open(%s)", argv[1]); + goto done; + } + break; + case 'a': /* API-PATH instead of INSTANCE-ID */ + api_path_p++; + break; + case 'p': /* API-PATH string */ + path = optarg; + break; + case 'y': + yang_file_dir = optarg; + break; + case 'Y': + if (clicon_option_add(h, "CLICON_YANG_DIR", optarg) < 0) + goto done; + break; + default: + usage(argv[0]); + break; + } + /* Parse yang */ + if (yang_file_dir){ + if ((yspec = yspec_new()) == NULL) + goto done; + if (stat(yang_file_dir, &st) < 0){ + clicon_err(OE_YANG, errno, "%s not found", yang_file_dir); + goto done; + } + if (S_ISDIR(st.st_mode)){ + if (yang_spec_load_dir(h, yang_file_dir, yspec) < 0) + goto done; + } + else{ + if (yang_spec_parse_file(h, yang_file_dir, yspec) < 0) + goto done; + } + } + + if (path==NULL){ + /* First read api-path from file */ + len = 1024; /* any number is fine */ + if ((buf = malloc(len)) == NULL){ + perror("pt_file malloc"); + return -1; + } + memset(buf, 0, len); + i = 0; + while (1){ + if ((ret = read(0, &c, 1)) < 0){ + perror("read"); + goto done; + } + if (ret == 0) + break; + if (c == '\n') + break; + if (len==i){ + if ((buf = realloc(buf, 2*len)) == NULL){ + fprintf(stderr, "%s: realloc: %s\n", __FUNCTION__, strerror(errno)); + return -1; + } + memset(buf+len, 0, len); + len *= 2; + } + buf[i++] = (char)(c&0xff); + } + path = buf; + } + + /* + * If fd=0, then continue reading from stdin (after CR) + * If fd>0, reading from file opened as argv[1] + */ + if (xml_parse_file(fd, "", NULL, &x) < 0){ + fprintf(stderr, "Error: parsing: %s\n", clicon_err_reason); + return -1; + } + + /* Validate XML as well */ + if (yang_file_dir){ + cxobj *x1; + cxobj *xerr = NULL; /* malloced must be freed */ + + x1 = NULL; + while ((x1 = xml_child_each(x, x1, CX_ELMNT)) != NULL) { + /* Populate */ + if (xml_apply0(x1, CX_ELMNT, xml_spec_populate, yspec) < 0) + goto done; + /* Sort */ + if (xml_apply0(x1, CX_ELMNT, xml_sort, h) < 0) + goto done; + /* Add default values */ + if (xml_apply(x1, CX_ELMNT, xml_default, h) < 0) + goto done; + if (xml_apply0(x1, -1, xml_sort_verify, h) < 0) + clicon_log(LOG_NOTICE, "%s: sort verify failed", __FUNCTION__); + if ((ret = xml_yang_validate_all_top(h, x1, &xerr)) < 0) + goto done; + if (ret > 0 && (ret = xml_yang_validate_add(h, x1, &xerr)) < 0) + goto done; + if (ret == 0){ + if ((cb = cbuf_new()) ==NULL){ + clicon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + if (netconf_err2cb(xerr, cb) < 0) + goto done; + fprintf(stderr, "xml validation error: %s\n", cbuf_get(cb)); + goto done; + } + } + } + if (api_path_p){ + if ((ret = clixon_xml_find_api_path(x, yspec, &xvec, &xlen, "%s", path)) < 0) + goto done; + } + else{ + if ((ret = clixon_xml_find_instance_id(x, yspec, &xvec, &xlen, "%s", path)) < 0) + goto done; + } + if (ret == 0){ + fprintf(stderr, "Fail\n"); + goto done; + } + /* Print results */ + for (i=0; i 0) + close(fd); + return retval; +} diff --git a/util/clixon_util_xpath.c b/util/clixon_util_xpath.c index 7e9e30b1..daa6246e 100644 --- a/util/clixon_util_xpath.c +++ b/util/clixon_util_xpath.c @@ -2,7 +2,7 @@ * ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON. @@ -58,7 +58,7 @@ See https://www.w3.org/TR/xpath/ #include "clixon/clixon.h" /* Command line options to be passed to getopt(3) */ -#define XPATH_OPTS "hD:f:p:i:n:cy:Y:x" +#define XPATH_OPTS "hD:f:p:i:n:cy:Y:" static int usage(char *argv0) @@ -74,11 +74,10 @@ usage(char *argv0) "\t-c \t\tMap xpath to canonical form\n" "\t-y \tYang filename or dir (load all files)\n" "\t-Y \tYang dirs (can be several)\n" - "\t-x \t\tXPath optimize\n" "and the following extra rules:\n" "\tif -f is not given, XML input is expected on stdin\n" "\tif -p is not given, is expected as the first line on stdin\n" - "This means that with no arguments, and XML is expected on stadin.\n", + "This means that with no arguments, and XML is expected on stdin.\n", argv0 ); exit(0); @@ -197,10 +196,6 @@ main(int argc, if (clicon_option_add(h, "CLICON_YANG_DIR", optarg) < 0) goto done; break; - case 'x': /* xpath optimize. Only if XPATH_LIST_OPTIMIZE is set */ - - xpath_list_optimize_set(1); - break; default: usage(argv[0]); break; diff --git a/yang/clixon/clixon-config@2019-09-11.yang b/yang/clixon/clixon-config@2019-09-11.yang index 6cfd354f..745defb2 100644 --- a/yang/clixon/clixon-config@2019-09-11.yang +++ b/yang/clixon/clixon-config@2019-09-11.yang @@ -12,7 +12,7 @@ module clixon-config { description "Clixon configuration file ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2020 Olof Hagsand This file is part of CLIXON @@ -369,7 +369,7 @@ module clixon-config { loaded Yang modules. This CLI tree can be accessed in CLI spec files using the tree reference syntax (eg @datamodel). See also CLICON_CLI_MODEL_TREENAME. - (Consider boolean)"; + (consider boolean)"; } leaf CLICON_CLI_MODEL_TREENAME { type string; @@ -383,7 +383,7 @@ module clixon-config { type int32; default 1; description "Generate code for CLI completion of existing db symbols. - Consider boolean type"; + (consider boolean)"; } leaf CLICON_CLI_GENMODEL_TYPE { type cli_genmodel_type; @@ -395,7 +395,8 @@ module clixon-config { default 1; description "Dont include keys in cvec in cli vars callbacks, - ie a & k in 'a k ' ignored"; + ie a & k in 'a k ' ignored + (consider boolean)"; } leaf CLICON_CLI_LINESCROLLING { type int32; @@ -425,7 +426,8 @@ module clixon-config { description "Set to 1 to enable CLIgen UTF-8 experimental mode. Note that this feature is EXPERIMENTAL and may not properly handle - scrolling, control characters, etc"; + scrolling, control characters, etc + (consider boolean)"; } leaf CLICON_CLI_HIST_FILE { type string; @@ -505,7 +507,8 @@ module clixon-config { default 0; description "Set if all configuration changes are committed automatically - on every edit change. Explicit commit commands unnecessary"; + on every edit change. Explicit commit commands unnecessary + (consider boolean)"; } leaf CLICON_XMLDB_DIR { type string;