From a804e05375f01591a51bbf3a807d97075be73aa9 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Thu, 23 May 2019 22:48:33 +0200 Subject: [PATCH] * Added clicon_handle as parameter to all validate functions * Added libxml2 XSD regexp mode as alternative to posix translation * Added `CLICON_YANG_REGEXP` option with possible values libxml2 and posix --- CHANGELOG.md | 6 +- apps/backend/backend_client.c | 4 +- apps/backend/backend_commit.c | 15 +- apps/backend/backend_main.c | 6 + apps/cli/cli_generate.c | 18 +- apps/cli/cli_main.c | 10 + apps/netconf/netconf_main.c | 2 +- apps/netconf/netconf_rpc.c | 8 +- apps/restconf/restconf_methods.c | 4 +- configure | 22 +- configure.ac | 10 +- include/clixon_config.h.in | 3 + lib/clixon/clixon.h.in | 1 + lib/clixon/clixon_options.h | 3 + lib/clixon/clixon_regex.h | 46 ++++ lib/clixon/clixon_string.h | 1 - lib/clixon/clixon_xml_map.h | 10 +- lib/clixon/clixon_yang_type.h | 2 +- lib/src/Makefile.in | 2 +- lib/src/clixon_options.c | 2 +- lib/src/clixon_regex.c | 272 ++++++++++++++++++++++ lib/src/clixon_string.c | 118 ---------- lib/src/clixon_xml_changelog.c | 4 +- lib/src/clixon_xml_map.c | 31 +-- lib/src/clixon_yang_type.c | 148 +++--------- test/test_pattern.sh | 8 +- yang/clixon/clixon-config@2019-03-05.yang | 34 ++- 27 files changed, 501 insertions(+), 289 deletions(-) create mode 100644 lib/clixon/clixon_regex.h create mode 100644 lib/src/clixon_regex.c diff --git a/CHANGELOG.md b/CHANGELOG.md index 8177e840..26eec9d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,8 @@ ### API changes on existing features (you may need to change your code) +* Added clicon_handle as parameter to all validate functions + * Just add `clixon_handle h` to all calls. * Clixon transaction mechanism has changed which may affect your backend plugin callbacks: * Validate-only transactions are terminated by an `end` or `abort` callback. Now all started transactions are terminated either by an `end` or `abort` without exceptions * Validate-only transactions used to be terminated by `complete` @@ -140,11 +142,13 @@ ### Minor changes * Regexp improvements + * Added libxml2 XSD regexp mode as alternative to posix translation * Better compliance with XSD regexps (when transforming to Posix regexps) * Added `\p{L}` and `\p{N}` * Added escaping of `$` + * Added `CLICON_YANG_REGEXP`option with possible values libxml2 and posix * Added clixon_util_regexp utility function - * Added regexp [test/test_pattern.sh] + * Added regexp test [test/test_pattern.sh] * Yang state get improvements * Integrated state and config into same tree on retrieval, not separate trees * Added cli functions `cli_show_config_state()` and `cli_show_auto_state()` for showing combined config and state info. diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index c278bfd2..723f4b0e 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -470,7 +470,7 @@ from_client_edit_config(clicon_handle h, goto ok; } /* xmldb_put (difflist handling) requires list keys */ - if ((ret = xml_yang_validate_list_key_only(xc, cbret)) < 0) + if ((ret = xml_yang_validate_list_key_only(h, xc, cbret)) < 0) goto done; if (ret == 0) goto ok; @@ -1147,7 +1147,7 @@ from_client_msg(clicon_handle h, * maybe not necessary since it should be */ if (xml_spec_populate_rpc(h, x, yspec) < 0) goto done; - if ((ret = xml_yang_validate_rpc(x, cbret)) < 0) + if ((ret = xml_yang_validate_rpc(h, x, cbret)) < 0) goto done; if (ret == 0) goto reply; diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 60bbd688..eb31cc6b 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -87,7 +87,8 @@ * @retval 1 Validation OK */ static int -generic_validate(yang_stmt *yspec, +generic_validate(clicon_handle h, + yang_stmt *yspec, transaction_data_t *td, cbuf *cbret) { @@ -99,7 +100,7 @@ generic_validate(yang_stmt *yspec, int ret; /* All entries */ - if ((ret = xml_yang_validate_all_top(td->td_target, cbret)) < 0) + if ((ret = xml_yang_validate_all_top(h, td->td_target, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -108,7 +109,7 @@ generic_validate(yang_stmt *yspec, x1 = td->td_scvec[i]; /* source changed */ x2 = td->td_tcvec[i]; /* target changed */ /* Should this be recursive? */ - if ((ret = xml_yang_validate_add(x2, cbret)) < 0) + if ((ret = xml_yang_validate_add(h, x2, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -126,7 +127,7 @@ generic_validate(yang_stmt *yspec, /* added entries */ for (i=0; itd_alen; i++){ x2 = td->td_avec[i]; - if ((ret = xml_yang_validate_add(x2, cbret)) < 0) + if ((ret = xml_yang_validate_add(h, x2, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -224,7 +225,7 @@ startup_common(clicon_handle h, /* 5. Make generic validation on all new or changed data. Note this is only call that uses 3-values */ clicon_debug(1, "Validating startup %s", db); - if ((ret = generic_validate(yspec, td, cbret)) < 0) + if ((ret = generic_validate(h, yspec, td, cbret)) < 0) goto done; if (ret == 0) goto fail; /* STARTUP_INVALID */ @@ -408,7 +409,7 @@ from_validate_common(clicon_handle h, * But xml_diff requires some basic validation, at least check that yang-specs * have been assigned */ - if ((ret = xml_yang_validate_all_top(td->td_target, cbret)) < 0) + if ((ret = xml_yang_validate_all_top(h, td->td_target, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -461,7 +462,7 @@ from_validate_common(clicon_handle h, /* 5. Make generic validation on all new or changed data. Note this is only call that uses 3-values */ - if ((ret = generic_validate(yspec, td, cbret)) < 0) + if ((ret = generic_validate(h, yspec, td, cbret)) < 0) goto done; if (ret == 0) goto fail; diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index 9e663cba..b8110db1 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -497,6 +497,12 @@ main(int argc, if (help) usage(h, argv[0]); +#ifndef HAVE_LIBXML2 + if (strcmp(clicon_yang_regexp(h), "libxml2")==0){ + clicon_err(OE_FATAL, 0, "CLICON_YANG_REGEXP set to libxml2, but HAVE_LIBXM2 not set (Either change CLICON_YANG_REGEXP to posix, or install libxml2?))"); + goto done; + } +#endif /* Check pid-file, if zap kil the old daemon, else return here */ if ((pidfile = clicon_backend_pidfile(h)) == NULL){ clicon_err(OE_FATAL, 0, "pidfile not set"); diff --git a/apps/cli/cli_generate.c b/apps/cli/cli_generate.c index 9a23741f..914f785f 100644 --- a/apps/cli/cli_generate.c +++ b/apps/cli/cli_generate.c @@ -359,12 +359,18 @@ yang2cli_var_sub(clicon_handle h, goto done; } if (options & YANG_OPTIONS_PATTERN){ - char *posix = NULL; - if (regexp_xsd2posix(pattern, &posix) < 0) - goto done; - cprintf(cb, " regexp:\"%s\"", posix); - if (posix) - free(posix); + char *mode; + mode = clicon_yang_regexp(h); + if (strcmp(mode, "posix") == 0){ + char *posix = NULL; + if (regexp_xsd2posix(pattern, &posix) < 0) + goto done; + cprintf(cb, " regexp:\"%s\"", posix); + if (posix) + free(posix); + } + else + cprintf(cb, " regexp:\"%s\"", pattern); } cprintf(cb, ">"); if (helptext) diff --git a/apps/cli/cli_main.c b/apps/cli/cli_main.c index cc0ab648..de7c7a17 100644 --- a/apps/cli/cli_main.c +++ b/apps/cli/cli_main.c @@ -446,6 +446,16 @@ main(int argc, char **argv) if (help) usage(h, argv[0]); + if (strcmp(clicon_yang_regexp(h), "libxml2")==0){ +#ifdef HAVE_LIBXML2 + /* Enable XSD libxml2 */ + cligen_regex_set(cli_cligen(h), 1); +#else + clicon_err(OE_FATAL, 0, "CLICON_YANG_REGEXP set to libxml2, but HAVE_LIBXM2 not set (Either change CLICON_YANG_REGEXP to posix, or install libxml2?))"); + goto done; +#endif + } + /* Setup signal handlers */ cli_signal_init(h); diff --git a/apps/netconf/netconf_main.c b/apps/netconf/netconf_main.c index 68d82705..f1432ec0 100644 --- a/apps/netconf/netconf_main.c +++ b/apps/netconf/netconf_main.c @@ -125,7 +125,7 @@ netconf_input_packet(clicon_handle h, isrpc++; if (xml_spec_populate_rpc(h, xrpc, yspec) < 0) goto done; - if ((ret = xml_yang_validate_rpc(xrpc, cbret)) < 0) + if ((ret = xml_yang_validate_rpc(h, xrpc, cbret)) < 0) goto done; if (ret == 0){ netconf_output_encap(1, cbret, "rpc-error"); diff --git a/apps/netconf/netconf_rpc.c b/apps/netconf/netconf_rpc.c index 5974a4c3..3204de9a 100644 --- a/apps/netconf/netconf_rpc.c +++ b/apps/netconf/netconf_rpc.c @@ -587,13 +587,13 @@ netconf_application_rpc(clicon_handle h, xml_spec_set(xn, yinput); /* needed for xml_spec_populate */ if (xml_apply(xn, CX_ELMNT, xml_spec_populate, yspec) < 0) goto done; - if ((ret = xml_yang_validate_all_top(xn, cbret)) < 0) + if ((ret = xml_yang_validate_all_top(h, xn, cbret)) < 0) goto done; if (ret == 0){ netconf_output_encap(1, cbret, "rpc-error"); goto ok; } - if ((ret = xml_yang_validate_add(xn, cbret)) < 0) + if ((ret = xml_yang_validate_add(h, xn, cbret)) < 0) goto done; if (ret == 0){ netconf_output_encap(1, cbret, "rpc-error"); @@ -622,13 +622,13 @@ netconf_application_rpc(clicon_handle h, if (xml_apply(xoutput, CX_ELMNT, xml_spec_populate, yspec) < 0) goto done; - if ((ret = xml_yang_validate_all_top(xoutput, cbret)) < 0) + if ((ret = xml_yang_validate_all_top(h, xoutput, cbret)) < 0) goto done; if (ret == 0){ clicon_log(LOG_WARNING, "Errors in output netconf %s", cbuf_get(cbret)); goto ok; } - if ((ret = xml_yang_validate_add(xoutput, cbret)) < 0) + if ((ret = xml_yang_validate_add(h, xoutput, cbret)) < 0) goto done; if (ret == 0){ clicon_log(LOG_WARNING, "Errors in output netconf %s", cbuf_get(cbret)); diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index d6b5bfc4..bd4b0482 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -1513,7 +1513,7 @@ api_operations_post_output(clicon_handle h, if ((ret = xml_yang_validate_all(xoutput, cbret)) < 0) goto done; if (ret == 1 && - (ret = xml_yang_validate_add(xoutput, cbret)) < 0) + (ret = xml_yang_validate_add(h, xoutput, cbret)) < 0) goto done; if (ret == 0){ /* validation failed */ if (xml_parse_string(cbuf_get(cbret), yspec, &xerr) < 0) @@ -1749,7 +1749,7 @@ api_operations_post(clicon_handle h, /* 6. Validate incoming RPC and fill in defaults */ if (xml_spec_populate_rpc(h, xtop, yspec) < 0) /* */ goto done; - if ((ret = xml_yang_validate_rpc(xtop, cbret)) < 0) + if ((ret = xml_yang_validate_rpc(h, xtop, cbret)) < 0) goto done; if (ret == 0){ if (xml_parse_string(cbuf_get(cbret), NULL, &xret) < 0) diff --git a/configure b/configure index ba501256..bd23e4cb 100755 --- a/configure +++ b/configure @@ -2162,7 +2162,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu -# Default CFLAGS unless set by environment +# Default CFLAGS and INSTALLFLAGS unless set by environment : ${CFLAGS="-O2 -Wall"} : ${INSTALLFLAGS="-s"} @@ -4417,8 +4417,7 @@ _ACEOF fi -# This is for Libxml2 code which we can use for XSD regexp -# AC_CHECK_HEADERS([libxml2/libxml/xmlexports.h]) +# This is for libxml2 code which we can use for XSD regexp { $as_echo "$as_me:${as_lineno-$LINENO}: checking for xmlRegexpCompile in -lxml2" >&5 $as_echo_n "checking for xmlRegexpCompile in -lxml2... " >&6; } if ${ac_cv_lib_xml2_xmlRegexpCompile+:} false; then : @@ -4465,6 +4464,23 @@ _ACEOF fi +# Libxml2 may hide its include files under /usr/include/libxml2/libxml +# You may need to prepend CFLAGS="-I/usr/include/libxml2 to configure +for ac_header in libxml/xmlregexp.h +do : + ac_fn_c_check_header_compile "$LINENO" "libxml/xmlregexp.h" "ac_cv_header_libxml_xmlregexp_h" "#include \"libxml/xmlversion.h\" +" +if test "x$ac_cv_header_libxml_xmlregexp_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBXML_XMLREGEXP_H 1 +_ACEOF + +fi + +done + + +# for ac_func in inet_aton sigaction sigvec strlcpy strsep strndup alphasort versionsort do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` diff --git a/configure.ac b/configure.ac index b070e3fa..bd811155 100644 --- a/configure.ac +++ b/configure.ac @@ -38,7 +38,7 @@ AC_INIT(lib/clixon/clixon.h.in) -# Default CFLAGS unless set by environment +# Default CFLAGS and INSTALLFLAGS unless set by environment : ${CFLAGS="-O2 -Wall"} : ${INSTALLFLAGS="-s"} @@ -220,10 +220,14 @@ AC_CHECK_LIB(socket, socket) AC_CHECK_LIB(nsl, xdr_char) AC_CHECK_LIB(dl, dlopen) -# This is for Libxml2 code which we can use for XSD regexp -# AC_CHECK_HEADERS([libxml2/libxml/xmlexports.h]) +# This is for libxml2 code which we can use for XSD regexp AC_CHECK_LIB(xml2, xmlRegexpCompile) +# Libxml2 may hide its include files under /usr/include/libxml2/libxml +# You may need to prepend CFLAGS="-I/usr/include/libxml2 to configure +AC_CHECK_HEADERS([libxml/xmlregexp.h], [], [], [#include "libxml/xmlversion.h"]) + +# AC_CHECK_FUNCS(inet_aton sigaction sigvec strlcpy strsep strndup alphasort versionsort) # CLIXON_DATADIR is where clixon installs the "system" yang files in yang/Makfile diff --git a/include/clixon_config.h.in b/include/clixon_config.h.in index 7bd27d76..66ef2062 100644 --- a/include/clixon_config.h.in +++ b/include/clixon_config.h.in @@ -60,6 +60,9 @@ /* Define to 1 if you have the `xml2' library (-lxml2). */ #undef HAVE_LIBXML2 +/* Define to 1 if you have the header file. */ +#undef HAVE_LIBXML_XMLREGEXP_H + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_IF_VLAN_H diff --git a/lib/clixon/clixon.h.in b/lib/clixon/clixon.h.in index ce307152..4751b1ad 100644 --- a/lib/clixon/clixon.h.in +++ b/lib/clixon/clixon.h.in @@ -84,6 +84,7 @@ #include #include #include +#include #include #include #include diff --git a/lib/clixon/clixon_options.h b/lib/clixon/clixon_options.h index 5f183d14..8e4d5922 100644 --- a/lib/clixon/clixon_options.h +++ b/lib/clixon/clixon_options.h @@ -129,6 +129,9 @@ static inline char *clicon_yang_module_main(clicon_handle h){ static inline char *clicon_yang_module_revision(clicon_handle h){ return clicon_option_str(h, "CLICON_YANG_MODULE_REVISION"); } +static inline char *clicon_yang_regexp(clicon_handle h){ + return clicon_option_str(h, "CLICON_YANG_REGEXP"); +} static inline char *clicon_backend_dir(clicon_handle h){ return clicon_option_str(h, "CLICON_BACKEND_DIR"); } diff --git a/lib/clixon/clixon_regex.h b/lib/clixon/clixon_regex.h new file mode 100644 index 00000000..ccd1d555 --- /dev/null +++ b/lib/clixon/clixon_regex.h @@ -0,0 +1,46 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + */ + +#ifndef _CLIXON_REGEX_H_ +#define _CLIXON_REGEX_H_ + +/* + * Prototypes + */ +int regexp_xsd2posix(char *xsd, char **posix); +int regex_compile(clicon_handle h, char *regexp, void **recomp); +int regex_exec(clicon_handle h, void *recomp, char *string); + +#endif /* _CLIXON_REGEX_H_ */ diff --git a/lib/clixon/clixon_string.h b/lib/clixon/clixon_string.h index eb106c10..90717b74 100644 --- a/lib/clixon/clixon_string.h +++ b/lib/clixon/clixon_string.h @@ -89,7 +89,6 @@ int clicon_str2int(const map_str2int *mstab, char *str); int clicon_str2int_search(const map_str2int *mstab, char *str, int upper); int nodeid_split(char *nodeid, char **prefix, char **id); char *clixon_trim(char *str); -int regexp_xsd2posix(char *xsd, char **posix); #ifndef HAVE_STRNDUP char *clicon_strndup (const char *, size_t); #endif /* ! HAVE_STRNDUP */ diff --git a/lib/clixon/clixon_xml_map.h b/lib/clixon/clixon_xml_map.h index 792099f7..a15e68b8 100644 --- a/lib/clixon/clixon_xml_map.h +++ b/lib/clixon/clixon_xml_map.h @@ -48,11 +48,11 @@ int xml2txt(FILE *f, cxobj *x, int level); int xml2cli(FILE *f, cxobj *x, char *prepend, enum genmodel_type gt); int xml_yang_root(cxobj *x, cxobj **xr); int xmlns_assign(cxobj *x); -int xml_yang_validate_rpc(cxobj *xrpc, cbuf *cbret); -int xml_yang_validate_list_key_only(cxobj *xt, cbuf *cbret); -int xml_yang_validate_add(cxobj *xt, cbuf *cbret); -int xml_yang_validate_all(cxobj *xt, cbuf *cbret); -int xml_yang_validate_all_top(cxobj *xt, cbuf *cbret); +int xml_yang_validate_rpc(clicon_handle h, cxobj *xrpc, cbuf *cbret); +int xml_yang_validate_list_key_only(clicon_handle h, cxobj *xt, cbuf *cbret); +int xml_yang_validate_add(clicon_handle h, cxobj *xt, cbuf *cbret); +int xml_yang_validate_all(clicon_handle h, cxobj *xt, cbuf *cbret); +int xml_yang_validate_all_top(clicon_handle h, cxobj *xt, cbuf *cbret); int xml2cvec(cxobj *xt, yang_stmt *ys, cvec **cvv0); int cvec2xml_1(cvec *cvv, char *toptag, cxobj *xp, cxobj **xt0); diff --git a/lib/clixon/clixon_yang_type.h b/lib/clixon/clixon_yang_type.h index 86597f1b..34c3b663 100644 --- a/lib/clixon/clixon_yang_type.h +++ b/lib/clixon/clixon_yang_type.h @@ -67,7 +67,7 @@ int ys_resolve_type(yang_stmt *ys, void *arg); int yang2cv_type(char *ytype, enum cv_type *cv_type); char *cv2yang_type(enum cv_type cv_type); yang_stmt *yang_find_identity(yang_stmt *ys, char *identity); -int ys_cv_validate(cg_var *cv, yang_stmt *ys, char **reason); +int ys_cv_validate(clicon_handle h, cg_var *cv, yang_stmt *ys, char **reason); int clicon_type2cv(char *type, char *rtype, yang_stmt *ys, enum cv_type *cvtype); int yang_type_get(yang_stmt *ys, char **otype, yang_stmt **restype, int *options, cvec **cvv, char **pattern, diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index 23723854..bba70a2f 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -67,7 +67,7 @@ CPPFLAGS = @CPPFLAGS@ INCLUDES = -I. @INCLUDES@ -I$(top_srcdir)/lib/clixon -I$(top_srcdir)/include -I$(top_srcdir) SRC = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \ - clixon_string.c clixon_handle.c \ + clixon_string.c clixon_regex.c clixon_handle.c \ clixon_xml.c clixon_xml_sort.c clixon_xml_map.c clixon_file.c \ clixon_json.c clixon_yang.c clixon_yang_type.c clixon_yang_module.c \ clixon_yang_cardinality.c clixon_xml_changelog.c \ diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index b42373b0..57b1b254 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -198,7 +198,7 @@ parse_configfile(clicon_handle h, clicon_err(OE_XML, errno, "cbuf_new"); goto done; } - if ((ret = xml_yang_validate_add(xc, cbret)) < 0) + if ((ret = xml_yang_validate_add(h, xc, cbret)) < 0) goto done; if (ret == 0){ clicon_err(OE_CFG, 0, "Config file validation: %s", cbuf_get(cbret)); diff --git a/lib/src/clixon_regex.c b/lib/src/clixon_regex.c new file mode 100644 index 00000000..602489ef --- /dev/null +++ b/lib/src/clixon_regex.c @@ -0,0 +1,272 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + * + * Clixon regular expression code for Yang type patterns following XML Schema + * regex. + * Two modes: libxml2 and posix-translation + * @see http://www.w3.org/TR/2004/REC-xmlschema-2-20041028 + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBXML_XMLREGEXP_H +#include +#endif + +#include + +/* clicon */ +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_err.h" +#include "clixon_log.h" +#include "clixon_yang.h" +#include "clixon_options.h" +#include "clixon_regex.h" + +/*-------------------------- POSIX translation -------------------------*/ + +/*! Transform from XSD regex to posix ERE + * The usecase is that Yang (RFC7950) supports XSD regular expressions but + * CLIgen supports POSIX ERE + * POSIX ERE regexps according to man regex(3). + * @param[in] xsd Input regex string according XSD + * @param[out] posix Output (malloced) string according to POSIX ERE + * @see https://www.w3.org/TR/2004/REC-xmlschema-2-20041028/#regexs + * @see https://www.regular-expressions.info/posixbrackets.html#class translation + * @see https://www.regular-expressions.info/xml.html + * Translation is not complete but covers some character sequences: + * \d decimal digit + * \w all characters except the set of "punctuation", "separator" and + * "other" characters: #x0000-#x10FFFF]-[\p{P}\p{Z}\p{C}] + * \i letters + underscore and colon + * \c XML Namechar, see: https://www.w3.org/TR/2008/REC-xml-20081126/#NT-NameChar + * + * Not implemented: + * \p{X} category escape. the ones identified in openconfig and yang-models are: + * \p{L} Letters [ultmo]? + * \p{M} Marks [nce]? + * \p{N} Numbers [dlo]? + * \p{P} Punctuation [cdseifo]? + * \p{Z} Separators [slp]? + * \p{S} Symbols [mcko]? + * \p{O} Other [cfon]? + */ +int +regexp_xsd2posix(char *xsd, + char **posix) +{ + int retval = -1; + cbuf *cb = NULL; + char x; + int i; + int j; /* lookahead */ + int esc; + int minus = 0; + + if ((cb = cbuf_new()) == NULL){ + clicon_err(OE_UNIX, errno, "cbuf_new"); + goto done; + } + esc=0; + for (i=0; i #include #include -#include #include #include @@ -694,120 +690,6 @@ clixon_trim(char *str) return s; } -/*! Transform from XSD regex to posix ERE - * The usecase is that Yang (RFC7950) supports XSD regular expressions but - * CLIgen supports POSIX ERE - * POSIX ERE regexps according to man regex(3). - * @param[in] xsd Input regex string according XSD - * @param[out] posix Output (malloced) string according to POSIX ERE - * @see https://www.w3.org/TR/2004/REC-xmlschema-2-20041028/#regexs - * @see https://www.regular-expressions.info/posixbrackets.html#class translation - * @see https://www.regular-expressions.info/xml.html - * Translation is not complete but covers some character sequences: - * \d decimal digit - * \w all characters except the set of "punctuation", "separator" and - * "other" characters: #x0000-#x10FFFF]-[\p{P}\p{Z}\p{C}] - * \i letters + underscore and colon - * \c XML Namechar, see: https://www.w3.org/TR/2008/REC-xml-20081126/#NT-NameChar - * - * Not implemented: - * \p{X} category escape. the ones identified in openconfig and yang-models are: - * \p{L} Letters [ultmo]? - * \p{N} Numbers [dlo]? - */ -int -regexp_xsd2posix(char *xsd, - char **posix) -{ - int retval = -1; - cbuf *cb = NULL; - char x; - int i; - int j; /* lookahead */ - int esc; - int minus = 0; - - if ((cb = cbuf_new()) == NULL){ - clicon_err(OE_UNIX, errno, "cbuf_new"); - goto done; - } - esc=0; - for (i=0; iys_argument, reason) < 0) goto done; goto fail; @@ -1096,7 +1098,7 @@ xml_yang_validate_add(cxobj *xt, } x = NULL; while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) { - if ((ret = xml_yang_validate_add(x, cbret)) < 0) + if ((ret = xml_yang_validate_add(h, x, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -1116,7 +1118,8 @@ xml_yang_validate_add(cxobj *xt, /*! Some checks done only at edit_config, eg keys in lists */ int -xml_yang_validate_list_key_only(cxobj *xt, +xml_yang_validate_list_key_only(clicon_handle h, + cxobj *xt, cbuf *cbret) { int retval = -1; @@ -1134,7 +1137,7 @@ xml_yang_validate_list_key_only(cxobj *xt, } x = NULL; while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) { - if ((ret = xml_yang_validate_list_key_only(x, cbret)) < 0) + if ((ret = xml_yang_validate_list_key_only(h, x, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -1168,7 +1171,8 @@ xml_yang_validate_list_key_only(cxobj *xt, * @note Should need a variant accepting cxobj **xret */ int -xml_yang_validate_all(cxobj *xt, +xml_yang_validate_all(clicon_handle h, + cxobj *xt, cbuf *cbret) { int retval = -1; @@ -1247,7 +1251,7 @@ xml_yang_validate_all(cxobj *xt, } x = NULL; while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) { - if ((ret = xml_yang_validate_all(x, cbret)) < 0) + if ((ret = xml_yang_validate_all(h, x, cbret)) < 0) goto done; if (ret == 0) goto fail; @@ -1275,7 +1279,8 @@ xml_yang_validate_all(cxobj *xt, * @retval -1 Error */ int -xml_yang_validate_all_top(cxobj *xt, +xml_yang_validate_all_top(clicon_handle h, + cxobj *xt, cbuf *cbret) { int ret; @@ -1283,7 +1288,7 @@ xml_yang_validate_all_top(cxobj *xt, x = NULL; while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) { - if ((ret = xml_yang_validate_all(x, cbret)) < 1) + if ((ret = xml_yang_validate_all(h, x, cbret)) < 1) return ret; } if ((ret = check_list_unique_minmax(xt, cbret)) < 1) diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index c6efd79c..2d48a0bf 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -62,6 +62,7 @@ #include "clixon_queue.h" #include "clixon_hash.h" #include "clixon_handle.h" +#include "clixon_regex.h" #include "clixon_yang.h" #include "clixon_hash.h" #include "clixon_xml.h" @@ -128,74 +129,6 @@ static const map_str2int ytmap2[] = { {NULL, -1} }; -/*! Regular expression compiling - * @retval -1 Error - * @retval 0 regex problem (no match?) - * @retval 1 OK Match - * @see match_regexp the CLIgen original composite function - */ -static int -regex_compile(char *pattern0, - regex_t *re) -{ - int retval = -1; - char pattern[1024]; - // char errbuf[1024]; - int len0; - int status; - - len0 = strlen(pattern0); - if (len0 > sizeof(pattern)-5){ - clicon_err(OE_XML, EINVAL, "pattern too long"); - goto done; - } - strncpy(pattern, "^(", 2); - strncpy(pattern+2, pattern0, sizeof(pattern)-2); - strncat(pattern, ")$", sizeof(pattern)-len0-1); - if ((status = regcomp(re, pattern, REG_NOSUB|REG_EXTENDED)) != 0) { -#if 0 /* ignore error msg for now */ - regerror(status, re, errbuf, sizeof(errbuf)); -#endif - goto fail; - } - retval = 1; - done: - return retval; - fail: - retval = 0; - goto done; -} - -/*! Regular expression execution - * @retval -1 Error - * @retval 0 regex problem (no match?) - * @retval 1 OK Match - * @see match_regexp the CLIgen original composite function - */ -static int -regex_exec(regex_t *re, - char *string) -{ - int retval = -1; - int status; - // char errbuf[1024]; - - status = regexec(re, string, (size_t) 0, NULL, 0); - if (status != 0) { -#if 0 /* ignore error msg for now */ - regerror(status, re, errbuf, sizeof(errbuf)); -#endif - goto fail; - } - retval = 1; - done: - return retval; - fail: - retval = 0; - goto done; -} - - /* return 1 if built-in, 0 if not */ static int yang_builtin(char *type) @@ -447,7 +380,8 @@ clicon_type2cv(char *origtype, * @retval 1 Validation OK */ static int -cv_validate1(cg_var *cv, +cv_validate1(clicon_handle h, + cg_var *cv, enum cv_type cvtype, int options, cvec *cvv, @@ -459,7 +393,7 @@ cv_validate1(cg_var *cv, int retval = 1; /* OK */ cg_var *cv1; cg_var *cv2; - int retval2; + int ret; yang_stmt *yi = NULL; char *str = NULL; int found; @@ -620,51 +554,30 @@ cv_validate1(cg_var *cv, } } } - if ((options & YANG_OPTIONS_PATTERN) != 0){ - char *posix = NULL; - regex_t *re = NULL; - - if ((re = yang_regex_cache_get(yrestype)) == NULL){ - /* Transform to posix regex */ - if (regexp_xsd2posix(pattern, &posix) < 0) - goto done; - /* Create regex cache */ - if ((re = malloc(sizeof(*re))) == NULL){ - clicon_err(OE_UNIX, errno, "malloc"); - goto done; + if ((options & YANG_OPTIONS_PATTERN) != 0) { + void *re = NULL; + if ((re = yang_regex_cache_get(yrestype)) == NULL){ + if ((ret = regex_compile(h, pattern, &re)) < 0) + goto done; + if (ret == 0){ + if (reason) + *reason = cligen_reason("regexp compile fail: \"%s\"", + pattern); + goto fail; + break; + } + yang_regex_cache_set(yrestype, re); } - memset(re, 0, sizeof(*re)); - /* Compute regex pattern for use in patterns */ - if ((retval2 = regex_compile(posix, re)) < 0) + if ((ret = regex_exec(h, re, str?str:"")) < 0) goto done; - if (retval2 == 0){ - if (reason) - *reason = cligen_reason("regexp match fail: \"%s\" does not match %s", - str, pattern); - goto fail; - break; - } - yang_regex_cache_set(yrestype, re); - if (posix) - free(posix); - } - if ((retval2 = regex_exec(re, str?str:"")) < 0) - goto done; - if (retval2 == 0){ + if (ret == 0){ if (reason) *reason = cligen_reason("regexp match fail: \"%s\" does not match %s", str, pattern); goto fail; break; } - if (retval2 == 0){ - if (reason) - *reason = cligen_reason("regexp match fail: \"%s\" does not match %s", - str, pattern); - goto fail; - break; } - } break; case CGV_VOID: break; /* empty type OK */ @@ -688,8 +601,8 @@ cv_validate1(cg_var *cv, } /* Forward */ -static int ys_cv_validate_union(yang_stmt *ys, char **reason, yang_stmt *yrestype, - char *type, char *val); +static int ys_cv_validate_union(clicon_handle h,yang_stmt *ys, char **reason, + yang_stmt *yrestype, char *type, char *val); /*! * @retval -1 Error (fatal), with errno set to indicate error @@ -697,7 +610,8 @@ static int ys_cv_validate_union(yang_stmt *ys, char **reason, yang_stmt *yrestyp * @retval 1 Validation OK */ static int -ys_cv_validate_union_one(yang_stmt *ys, +ys_cv_validate_union_one(clicon_handle h, + yang_stmt *ys, char **reason, yang_stmt *yt, char *type, /* orig type */ @@ -718,7 +632,7 @@ ys_cv_validate_union_one(yang_stmt *ys, goto done; restype = yrt?yrt->ys_argument:NULL; if (restype && strcmp(restype, "union") == 0){ /* recursive union */ - if ((retval = ys_cv_validate_union(ys, reason, yrt, type, val)) < 0) + if ((retval = ys_cv_validate_union(h, ys, reason, yrt, type, val)) < 0) goto done; } else { @@ -735,7 +649,7 @@ ys_cv_validate_union_one(yang_stmt *ys, } if (retval == 0) goto done; - if ((retval = cv_validate1(cvt, cvtype, options, cvv, + if ((retval = cv_validate1(h, cvt, cvtype, options, cvv, pattern, yrt, restype, reason)) < 0) goto done; } @@ -751,7 +665,8 @@ ys_cv_validate_union_one(yang_stmt *ys, * @retval 1 Validation OK */ static int -ys_cv_validate_union(yang_stmt *ys, +ys_cv_validate_union(clicon_handle h, + yang_stmt *ys, char **reason, yang_stmt *yrestype, char *type, /* orig type */ @@ -764,7 +679,7 @@ ys_cv_validate_union(yang_stmt *ys, while ((yt = yn_each(yrestype, yt)) != NULL){ if (yt->ys_keyword != Y_TYPE) continue; - if ((retval = ys_cv_validate_union_one(ys, reason, yt, type, val)) < 0) + if ((retval = ys_cv_validate_union_one(h, ys, reason, yt, type, val)) < 0) goto done; /* If validation failed, save reason, reset error and continue, * save latest reason if noithing validates. @@ -800,7 +715,8 @@ ys_cv_validate_union(yang_stmt *ys, * See also cv_validate - the code is similar. */ int -ys_cv_validate(cg_var *cv, +ys_cv_validate(clicon_handle h, + cg_var *cv, yang_stmt *ys, char **reason) { @@ -846,12 +762,12 @@ ys_cv_validate(cg_var *cv, if (restype && strcmp(restype, "union") == 0){ assert(cvtype == CGV_REST); val = cv_string_get(cv); - if ((retval2 = ys_cv_validate_union(ys, reason, yrestype, type, val)) < 0) + if ((retval2 = ys_cv_validate_union(h, ys, reason, yrestype, type, val)) < 0) goto done; retval = retval2; /* invalid (0) with latest reason or valid 1 */ } else - if ((retval = cv_validate1(cv, cvtype, options, cvv, pattern, + if ((retval = cv_validate1(h, cv, cvtype, options, cvv, pattern, yrestype, restype, reason)) < 0) goto done; done: diff --git a/test/test_pattern.sh b/test/test_pattern.sh index 7b4a3c01..513edcfb 100755 --- a/test/test_pattern.sh +++ b/test/test_pattern.sh @@ -22,12 +22,16 @@ APPNAME=example cfg=$dir/pattern.xml fyang=$dir/pattern.yang +# Regexp mode: posix or libxml2 +: ${regex:=posix} + cat < $cfg $cfg /usr/local/share/clixon $dir $fyang + $regex /usr/local/lib/$APPNAME/clispec /usr/local/lib/$APPNAME/cli $APPNAME @@ -457,7 +461,9 @@ new "Test for pattern leaf $pnr ipv6-prefix" testrun "$pnr" 1 '::9a95::A54:63:e001:6E1:15/17' testrun "$pnr" 1 ':::7:fc:c::eDe:/3' testrun "$pnr" 1 '7dE::D1e:8:8eBC::/98' -testrun "$pnr" 1 ':29:F36:6:46.53.251.2/100' # This does not work w libxml2 +if [ $regex != libxml2 ]; then + testrun "$pnr" 1 ':29:F36:6:46.53.251.2/100' # This does not work w libxml2 +fi testrun "$pnr" 1 '::CE2e:A:AB:234.220.225.250/1' let pnr=15 diff --git a/yang/clixon/clixon-config@2019-03-05.yang b/yang/clixon/clixon-config@2019-03-05.yang index 22c962c1..1949a813 100644 --- a/yang/clixon/clixon-config@2019-03-05.yang +++ b/yang/clixon/clixon-config@2019-03-05.yang @@ -113,7 +113,6 @@ module clixon-config { } } } - typedef cli_genmodel_type{ description "How to generate CLI from YANG model, @@ -148,6 +147,30 @@ module clixon-config { } } } + typedef regexp_mode{ + description + "The regular expression engine Clixon uses in its validation of + Yang patterns, and in the CLI. + Yang RFC 7950 stipulates XSD XML Schema regexps + according to W3 CXML Schema Part 2: Datatypes Second Edition, + see http://www.w3.org/TR/2004/REC-xmlschema-2-20041028#regexs"; + type enumeration{ + enum posix { + description + "Translate XSD XML Schema regexp:s to Posix regexp. This is + not a complete translation, but can be considered good-enough + for Yang use-cases as defined by openconfig and yang-models + for example."; + } + enum libxml2 { + description + "Use libxml2 XSD XML Schema regexp engine. This is a complete + XSD regexp engine.. + Requires libxml2 to be available at configure time + (HAVE_LIBXML2 should be set)"; + } + } + } container clixon-config { leaf-list CLICON_FEATURE { description @@ -203,6 +226,15 @@ module clixon-config { [@]. Used together with CLICON_YANG_MODULE_MAIN"; } + leaf CLICON_YANG_REGEXP { + type regexp_mode; + default posix; + description + "The regular expression engine Clixon uses in its validation of + Yang patterns, and in the CLI. + There is a 'good-enough' posix translation mode and a complete + libxml2 mode"; + } leaf CLICON_BACKEND_DIR { type string; description