From bc54f2d04c7f81fcc562b3a59ef46622ab72af95 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 20 May 2019 16:03:29 +0200 Subject: [PATCH] * Regexp improvements * Added check for libxml in configure'; * Added clixon_util_regexp utility function * Yang state get improvements * Integrated state and config into same tree on retrieval, not separate trees * Added cli functions `cli_show_config_state()` and `cli_show_auto_state()` for showing combined config and state info. * Added integrated state in the main example: `interface/oper-state`. * Added performance tests for getting state, see [test/test_perf_state.sh]. --- CHANGELOG.md | 8 ++ apps/backend/backend_client.c | 5 +- apps/backend/backend_plugin.c | 2 + apps/cli/cli_show.c | 138 ++++++++++++++++++--- apps/cli/clixon_cli_api.h | 5 +- configure | 48 ++++++++ configure.ac | 4 + example/main/example_backend.c | 35 +++++- example/main/example_cli.cli | 7 +- include/clixon_config.h.in | 3 + lib/src/clixon_yang.c | 7 +- lib/src/clixon_yang_module.c | 26 +++- lib/src/clixon_yang_parse.h | 3 +- test/long.sh | 21 ++-- test/test_datastore.sh | 1 - test/test_feature.sh | 32 ++--- test/test_nacm_ext.sh | 3 +- test/test_nacm_module_write.sh | 1 + test/test_netconf.sh | 4 +- test/test_perf.sh | 181 +++++++++++++--------------- test/test_perf_startup.sh | 121 +++++++++++++++++++ test/test_perf_state.sh | 157 ++++++++++++++++++++++++ test/test_restconf.sh | 28 ++--- util/Makefile.in | 14 ++- util/clixon_util_insert.c | 12 +- util/clixon_util_regexp.c | 214 +++++++++++++++++++++++++++++++++ 26 files changed, 895 insertions(+), 185 deletions(-) create mode 100755 test/test_perf_startup.sh create mode 100755 test/test_perf_state.sh create mode 100644 util/clixon_util_regexp.c diff --git a/CHANGELOG.md b/CHANGELOG.md index f1af4285..f2f8ae12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,14 @@ ### Minor changes +* Regexp improvements + * Added check for libxml in configure'; + * Added clixon_util_regexp utility function +* Yang state get improvements + * Integrated state and config into same tree on retrieval, not separate trees + * Added cli functions `cli_show_config_state()` and `cli_show_auto_state()` for showing combined config and state info. + * Added integrated state in the main example: `interface/oper-state`. + * Added performance tests for getting state, see [test/test_perf_state.sh]. * Improved submodule implementation (as part of [Yang submodule import prefix restrictions #60](https://github.com/clicon/clixon/issues/60)). * Submodules share same namespace as modules, which means that functions looking for symbols under a module were extended to also look in that module's included submodules, also recursively (submodules can include submodules in Yang 1.0). * Submodules are no longer merged with modules in the code. This is necessary to have separate local import prefixes, for example. diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 9a059027..c278bfd2 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -260,7 +260,10 @@ client_statedata(clicon_handle h, goto done; if (ret == 0) goto fail; - /* Code complex to filter out anything that is outside of xpath */ + /* Code complex to filter out anything that is outside of xpath + * Actually this is a safety catch, should realy be done in plugins + * and modules_state functions. + */ if (xpath_vec(*xret, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) goto done; /* If vectors are specified then mark the nodes found and diff --git a/apps/backend/backend_plugin.c b/apps/backend/backend_plugin.c index 4a64b6f9..754d4caf 100644 --- a/apps/backend/backend_plugin.c +++ b/apps/backend/backend_plugin.c @@ -138,6 +138,8 @@ clixon_plugin_statedata(clicon_handle h, goto done; if (fn(h, xpath, x) < 0) goto fail; /* Dont quit here on user callbacks */ + if (xml_apply(x, CX_ELMNT, xml_spec_populate, yspec) < 0) + goto done; if ((ret = netconf_trymerge(x, yspec, xret)) < 0) goto done; if (ret == 0) diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index 00ac5d3e..d2471a95 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -381,24 +381,27 @@ show_yang(clicon_handle h, return 0; } -/*! Generic show configuration CLIGEN callback - * Utility function used by cligen spec file +/*! Show configuration and state internal function + * * @param[in] h CLICON handle + * @param[in] state If set, show both config and state, otherwise only config * @param[in] cvv Vector of variables from CLIgen command-line * @param[in] argv String vector: [] * Format of argv: - * "running"|"candidate"|"startup" + * "running"|"candidate"|"startup" # note only running state=1 * "text"|"xml"|"json"|"cli"|"netconf" (see format_enum) * xpath expression, that may contain one %, eg "/sender[name="%s"]" * optional name of variable in cvv. If set, xpath must have a '%s' * @code * show config id , cli_show_config("running","xml","iface[name="%s"]","n"); * @endcode + * @note if state parameter is set, then db must be running */ -int -cli_show_config(clicon_handle h, - cvec *cvv, - cvec *argv) +static int +cli_show_config1(clicon_handle h, + int state, + cvec *cvv, + cvec *argv) { int retval = -1; char *db; @@ -462,9 +465,19 @@ cli_show_config(clicon_handle h, } else cprintf(cbxpath, "%s", xpath); - /* Get configuration from database */ - if (clicon_rpc_get_config(h, db, cbuf_get(cbxpath), &xt) < 0) - goto done; + + if (state == 0){ /* Get configuration-only from database */ + if (clicon_rpc_get_config(h, db, cbuf_get(cbxpath), &xt) < 0) + goto done; + } + else { /* Get configuration and state from database */ + if (strcmp(db, "running") != 0){ + clicon_err(OE_FATAL, 0, "Show state only for running database, not %s", db); + goto done; + } + if (clicon_rpc_get(h, cbuf_get(cbxpath), &xt) < 0) + goto done; + } if ((xerr = xpath_first(xt, "/rpc-error")) != NULL){ clicon_rpc_generate_error("Get configuration", xerr); goto done; @@ -518,6 +531,52 @@ done: return retval; } +/*! Show configuration and state CLIGEN callback function + * + * @param[in] h CLICON handle + * @param[in] cvv Vector of variables from CLIgen command-line + * @param[in] argv String vector: [] + * Format of argv: + * "running"|"candidate"|"startup" + * "text"|"xml"|"json"|"cli"|"netconf" (see format_enum) + * xpath expression, that may contain one %, eg "/sender[name="%s"]" + * optional name of variable in cvv. If set, xpath must have a '%s' + * @code + * show config id , cli_show_config("running","xml","iface[name="%s"]","n"); + * @endcode + * @see cli_show_config_state For config and state data (not only config) + */ +int +cli_show_config(clicon_handle h, + cvec *cvv, + cvec *argv) +{ + return cli_show_config1(h, 0, cvv, argv); +} + +/*! Show configuration and state CLIgen callback function + * + * @param[in] h CLICON handle + * @param[in] cvv Vector of variables from CLIgen command-line + * @param[in] argv String vector: [] + * Format of argv: + * "running" + * "text"|"xml"|"json"|"cli"|"netconf" (see format_enum) + * xpath expression, that may contain one %, eg "/sender[name="%s"]" + * optional name of variable in cvv. If set, xpath must have a '%s' + * @code + * show config id , cli_show_config_state("running","xml","iface[name="%s"]","n"); + * @endcode + * @see cli_show_config For config-only, no state + */ +int +cli_show_config_state(clicon_handle h, + cvec *cvv, + cvec *argv) +{ + return cli_show_config1(h, 1, cvv, argv); +} + /*! Show configuration as text given an xpath * Utility function used by cligen spec file * @param[in] h CLICON handle @@ -583,15 +642,21 @@ int cli_show_version(clicon_handle h, } /*! Generic show configuration CLIGEN callback using generated CLI syntax + * @param[in] h CLICON handle + * @param[in] state If set, show both config and state, otherwise only config + * @param[in] cvv Vector of variables from CLIgen command-line + * @param[in] argv String vector: [] * Format of argv: * Generated API PATH * "running"|"candidate"|"startup" * "text"|"xml"|"json"|"cli"|"netconf" (see format_enum) + * @note if state parameter is set, then db must be running */ -int -cli_show_auto(clicon_handle h, - cvec *cvv, - cvec *argv) +static int +cli_show_auto1(clicon_handle h, + int state, + cvec *cvv, + cvec *argv) { int retval = 1; yang_stmt *yspec; @@ -635,9 +700,19 @@ cli_show_auto(clicon_handle h, if (xpath[strlen(xpath)-1] == '/') xpath[strlen(xpath)-1] = '\0'; - /* Get configuration from database */ - if (clicon_rpc_get_config(h, db, xpath, &xt) < 0) - goto done; + if (state == 0){ /* Get configuration-only from database */ + if (clicon_rpc_get_config(h, db, xpath, &xt) < 0) + goto done; + } + else{ /* Get configuration and state from database */ + if (strcmp(db, "running") != 0){ + clicon_err(OE_FATAL, 0, "Show state only for running database, not %s", db); + goto done; + } + if (clicon_rpc_get(h, xpath, &xt) < 0) + goto done; + } + if ((xerr = xpath_first(xt, "/rpc-error")) != NULL){ clicon_rpc_generate_error("Get configuration", xerr); goto done; @@ -674,4 +749,33 @@ cli_show_auto(clicon_handle h, return retval; } +/*! Generic show configuration CLIgen callback using generated CLI syntax + * Format of argv: + * Generated API PATH + * "running"|"candidate"|"startup" + * "text"|"xml"|"json"|"cli"|"netconf" (see format_enum) + * @see cli_show_auto_state For config and state + */ +int +cli_show_auto(clicon_handle h, + cvec *cvv, + cvec *argv) +{ + return cli_show_auto1(h, 0, cvv, argv); +} + +/*! Generic show config and state CLIgen callback using generated CLI syntax + * Format of argv: + * Generated API PATH + * "running" + * "text"|"xml"|"json"|"cli"|"netconf" (see format_enum) + * @see cli_show_auto For config only + */ +int +cli_show_auto_state(clicon_handle h, + cvec *cvv, + cvec *argv) +{ + return cli_show_auto1(h, 1, cvv, argv); +} diff --git a/apps/cli/clixon_cli_api.h b/apps/cli/clixon_cli_api.h index 5ed8f782..832f04c5 100644 --- a/apps/cli/clixon_cli_api.h +++ b/apps/cli/clixon_cli_api.h @@ -138,9 +138,12 @@ int show_yang(clicon_handle h, cvec *vars, cvec *argv); int show_conf_xpath(clicon_handle h, cvec *cvv, cvec *argv); - int cli_show_config(clicon_handle h, cvec *cvv, cvec *argv); +int cli_show_config_state(clicon_handle h, cvec *cvv, cvec *argv); + int cli_show_auto(clicon_handle h, cvec *cvv, cvec *argv); +int cli_show_state_auto(clicon_handle h, cvec *cvv, cvec *argv); + #endif /* _CLIXON_CLI_API_H_ */ diff --git a/configure b/configure index bc0eed46..ba501256 100755 --- a/configure +++ b/configure @@ -4417,6 +4417,54 @@ _ACEOF fi +# This is for Libxml2 code which we can use for XSD regexp +# AC_CHECK_HEADERS([libxml2/libxml/xmlexports.h]) +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for xmlRegexpCompile in -lxml2" >&5 +$as_echo_n "checking for xmlRegexpCompile in -lxml2... " >&6; } +if ${ac_cv_lib_xml2_xmlRegexpCompile+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lxml2 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char xmlRegexpCompile (); +int +main () +{ +return xmlRegexpCompile (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_xml2_xmlRegexpCompile=yes +else + ac_cv_lib_xml2_xmlRegexpCompile=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_xml2_xmlRegexpCompile" >&5 +$as_echo "$ac_cv_lib_xml2_xmlRegexpCompile" >&6; } +if test "x$ac_cv_lib_xml2_xmlRegexpCompile" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBXML2 1 +_ACEOF + + LIBS="-lxml2 $LIBS" + +fi + + for ac_func in inet_aton sigaction sigvec strlcpy strsep strndup alphasort versionsort do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` diff --git a/configure.ac b/configure.ac index 47b79aa2..b070e3fa 100644 --- a/configure.ac +++ b/configure.ac @@ -220,6 +220,10 @@ AC_CHECK_LIB(socket, socket) AC_CHECK_LIB(nsl, xdr_char) AC_CHECK_LIB(dl, dlopen) +# This is for Libxml2 code which we can use for XSD regexp +# AC_CHECK_HEADERS([libxml2/libxml/xmlexports.h]) +AC_CHECK_LIB(xml2, xmlRegexpCompile) + AC_CHECK_FUNCS(inet_aton sigaction sigvec strlcpy strsep strndup alphasort versionsort) # CLIXON_DATADIR is where clixon installs the "system" yang files in yang/Makfile diff --git a/example/main/example_backend.c b/example/main/example_backend.c index e132b278..43f7e936 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -264,7 +264,7 @@ example_copy_extra(clicon_handle h, /* Clicon handle */ /*! Called to get state data from plugin * @param[in] h Clicon handle * @param[in] xpath String with XPATH syntax. or NULL for all - * @param[in] xtop XML tree, on entry. + * @param[in] xstate XML tree, on entry. * @retval 0 OK * @retval -1 Error * @see xmldb_get @@ -285,13 +285,34 @@ example_statedata(clicon_handle h, { int retval = -1; cxobj **xvec = NULL; + size_t xlen; + cbuf *cb = cbuf_new(); + int i; + cxobj *xt = NULL; + char *name; if (!_state) goto ok; - /* Example of (static) statedata, real code would poll state - * Note this state needs to be accomanied by yang snippet - * above - */ + + /* Example of statedata, in this case merging state data with + * state information. In this case adding dummy interface operation state + * to configured interfaces. + * Get config according to xpath */ + if (xmldb_get(h, "running", xpath, 1, &xt, NULL) < 0) + goto done; + /* From that subset, only get the names */ + if (xpath_vec(xt, "/interfaces/interface/name", &xvec, &xlen) < 0) + goto done; + if (xlen){ + cprintf(cb, ""); + for (i=0; i%sup", name); + } + cprintf(cb, ""); + if (xml_parse_string(cbuf_get(cb), NULL, &xstate) < 0) + goto done; + } if (xml_parse_string("" "42" "41" @@ -301,6 +322,10 @@ example_statedata(clicon_handle h, ok: retval = 0; done: + if (xt) + xml_free(xt); + if (cb) + cbuf_free(cb); if (xvec) free(xvec); return retval; diff --git a/example/main/example_cli.cli b/example/main/example_cli.cli index 1b31ab30..5ffcd2aa 100644 --- a/example/main/example_cli.cli +++ b/example/main/example_cli.cli @@ -24,7 +24,7 @@ debug("Debugging parts of the system"), cli_debug_cli((int32)1);{ } copy("Copy and create a new object") { interface("Copy interface"){ - ("name of interface to copy from") to("Copy to interface") ("Name of interface to copy to"), cli_copy_config("candidate","//interface[%s='%s']","name","name","toname"); + (|("name of interface to copy from")) to("Copy to interface") ("Name of interface to copy to"), cli_copy_config("candidate","//interface[%s='%s']","name","name","toname"); } } discard("Discard edits (rollback 0)"), discard_changes(); @@ -37,6 +37,11 @@ show("Show a particular state of the system"){ xml("Show comparison in xml"), compare_dbs((int32)0); text("Show comparison in text"), compare_dbs((int32)1); } + state("Show configuration and state"), cli_show_config_state("running", "text", "/"){ + xml("Show configuration and state as XML"), cli_show_config_state("candidate", "xml", "/");{ + @datamodel, cli_show_auto_state("running", "xml"); + } + } configuration("Show configuration"), cli_show_config("candidate", "text", "/");{ xml("Show configuration as XML"), cli_show_config("candidate", "xml", "/");{ @datamodel, cli_show_auto("candidate", "xml"); diff --git a/include/clixon_config.h.in b/include/clixon_config.h.in index a3f841ca..7bd27d76 100644 --- a/include/clixon_config.h.in +++ b/include/clixon_config.h.in @@ -57,6 +57,9 @@ /* Define to 1 if you have the `socket' library (-lsocket). */ #undef HAVE_LIBSOCKET +/* Define to 1 if you have the `xml2' library (-lxml2). */ +#undef HAVE_LIBXML2 + /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_IF_VLAN_H diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index c176fc12..7a8013aa 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -69,7 +69,6 @@ #include #include #include -#include /* cligen */ #include @@ -3323,6 +3322,12 @@ ys_parse_sub(yang_stmt *ys, cv_uint32_set(ys->ys_cv, minmax); } break; + case Y_MODIFIER: + if (strcmp(yang_argument_get(ys), "invert-match")){ + clicon_err(OE_YANG, EINVAL, "modifier %s, expected invert-match", yang_argument_get(ys)); + goto done; + } + break; case Y_UNKNOWN: /* XXX This code assumes ymod already loaded but it may not be */ if (extra == NULL) diff --git a/lib/src/clixon_yang_module.c b/lib/src/clixon_yang_module.c index d9303061..4d71c828 100644 --- a/lib/src/clixon_yang_module.c +++ b/lib/src/clixon_yang_module.c @@ -75,6 +75,7 @@ #include "clixon_data.h" #include "clixon_plugin.h" #include "clixon_netconf_lib.h" +#include "clixon_xml_map.h" #include "clixon_yang_module.h" #include "clixon_yang_internal.h" /* internal */ @@ -289,11 +290,16 @@ yang_modules_state_get(clicon_handle h, cxobj *x1; cbuf *cb = NULL; int ret; + cxobj **xvec = NULL; + size_t xlen; + int i; msid = clicon_option_str(h, "CLICON_MODULE_SET_ID"); if ((x = clicon_modst_cache_get(h, brief)) != NULL){ /* x is here: ... * and x is original tree, need to copy */ + if ((x = xml_wrap(x, "top")) < 0) + goto done; if (xpath_first(x, "%s", xpath)){ if ((x1 = xml_dup(x)) == NULL) goto done; @@ -301,6 +307,9 @@ yang_modules_state_get(clicon_handle h, } else x = NULL; + /* unwrap */ + if (x && xml_rootchild(x, 0, &x) < 0) + goto done; } else { /* No cache -> build the tree */ if ((cb = cbuf_new()) == NULL){ @@ -322,10 +331,21 @@ yang_modules_state_get(clicon_handle h, if (clicon_modst_cache_set(h, brief, x) < 0) /* move to fn above? */ goto done; } - if (x){ - /* Wrap x (again) with new top-level node "top" which merge wants */ + if (x){ /* x is here a copy (This code is ugly and I think wrong) */ + /* Wrap x (again) with new top-level node "top" which xpath wants */ if ((x = xml_wrap(x, "top")) < 0) goto done; + /* extract xpath part of module-state tree */ + if (xpath_vec(x, "%s", &xvec, &xlen, xpath?xpath:"/") < 0) + goto done; + if (xvec != NULL){ + for (i=0; i $cfg $dir /usr/local/share/clixon $IETFRFC - scaling + $fyang /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/$APPNAME/$APPNAME.pidfile false @@ -61,26 +61,27 @@ EOF sudo callgrind_control -i off -new "test params: -f $cfg -y $fyang" +new "test params: -f $cfg if [ $BE -ne 0 ]; then new "kill old backend" - sudo clixon_backend -zf $cfg -y $fyang + sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg -y $fyang" - start_backend -s init -f $cfg -y $fyang + new "start backend -s init -f $cfg + start_backend -s init -f $cfg fi new "kill old restconf daemon" sudo pkill -u www-data -f "/www-data/clixon_restconf" new "start restconf daemon" -start_restconf -f $cfg -y $fyang +start_restconf -f $cfg new "waiting" -sleep $RCWAIT +wait_backend +wait_restconf new "generate 'large' config with $perfnr list entries" echo -n "" > $fconfig @@ -91,16 +92,18 @@ echo "]]>]]>" >> $fconfig # Now take large config file and write it via netconf to candidate new "netconf write large config" -expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg" "$fconfig" "^]]>]]>$" # Now commit it from candidate to running new "netconf commit large config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" # Zero all event counters sudo callgrind_control -i on sudo callgrind_control -z + + while [ 1 ] ; do new "restconf add $perfreq small config" diff --git a/test/test_datastore.sh b/test/test_datastore.sh index 1978c973..cd054751 100755 --- a/test/test_datastore.sh +++ b/test/test_datastore.sh @@ -7,7 +7,6 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi fyang=$dir/ietf-ip.yang -# If set, enable debugging (of backend) : ${clixon_util_datastore:=clixon_util_datastore} cat < $fyang diff --git a/test/test_feature.sh b/test/test_feature.sh index d441b3e7..6db7c439 100755 --- a/test/test_feature.sh +++ b/test/test_feature.sh @@ -33,7 +33,7 @@ cat < $cfg $cfg /usr/local/share/clixon $IETFRFC - $APPNAME + $fyang /usr/local/lib/$APPNAME/clispec /usr/local/lib/$APPNAME/cli $APPNAME @@ -73,52 +73,54 @@ module example{ } EOF -new "test params: -f $cfg -y $fyang" +new "test params: -f $cfg" if [ $BE -ne 0 ]; then new "kill old backend" - sudo clixon_backend -zf $cfg -y $fyang + sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg -y $fyang" - start_backend -s init -f $cfg -y $fyang + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg new "waiting" sleep $RCWAIT fi new "cli enabled feature" -expectfn "$clixon_cli -1f $cfg -y $fyang set x foo" 0 "" +expectfn "$clixon_cli -1f $cfg set x foo" 0 "" new "cli disabled feature" -expectfn "$clixon_cli -1f $cfg -l o -y $fyang set y foo" 255 "CLI syntax error: \"set y foo\": Unknown command" +expectfn "$clixon_cli -1f $cfg -l o set y foo" 255 "CLI syntax error: \"set y foo\": Unknown command" new "cli enabled feature in other module" -expectfn "$clixon_cli -1f $cfg -y $fyang set routing router-id 1.2.3.4" 0 "" +expectfn "$clixon_cli -1f $cfg set routing router-id 1.2.3.4" 0 "" new "cli disabled feature in other module" -expectfn "$clixon_cli -1f $cfg -l o -y $fyang set routing ribs rib default-rib false" 255 "CLI syntax error: \"set routing ribs rib default-rib false\": Unknown command" +expectfn "$clixon_cli -1f $cfg -l o set routing ribs rib default-rib false" 255 "CLI syntax error: \"set routing ribs rib default-rib false\": Unknown command" new "netconf discard-changes" -expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "netconf enabled feature" -expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 'foo]]>]]>' "^]]>]]>$" +expecteof "$clixon_netconf -qf $cfg" 0 'foo]]>]]>' "^]]>]]>$" new "netconf validate enabled feature" -expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "netconf disabled feature" -expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 'foo]]>]]>' '^applicationunknown-elementyerrorUnassigned yang spec]]>]]>$' +expecteof "$clixon_netconf -qf $cfg" 0 'foo]]>]]>' '^applicationunknown-elementyerrorUnassigned yang spec]]>]]>$' # This test has been broken up into all different modules instead of one large # reply since the modules change so often new "netconf schema resource, RFC 7895" -ret=$($clixon_netconf -qf $cfg -y $fyang<]]>]]> EOF -) + ) +#echo $ret + new "netconf modules-state header" expect='^' match=`echo "$ret" | grep -GZo "$expect"` diff --git a/test/test_nacm_ext.sh b/test/test_nacm_ext.sh index f873b11d..cd66b01c 100755 --- a/test/test_nacm_ext.sh +++ b/test/test_nacm_ext.sh @@ -29,6 +29,7 @@ cat < $cfg /usr/local/lib/$APPNAME/cli $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock + false /usr/local/var/$APPNAME/$APPNAME.pidfile 1 /usr/local/var/$APPNAME @@ -151,7 +152,7 @@ new "waiting" sleep $RCWAIT new "auth get" -expecteq "$(curl -u andy:bar -sS -X GET http://localhost/restconf/data/clixon-example:state)" 0 '{"clixon-example:state": {"op": ["42","41","43"]}} +expecteq "$(curl -u andy:bar -sS -X GET http://localhost/restconf/data)" 0 '{"data": {"clixon-example:state": {"op": ["42","41","43"]}}} ' new "Set x to 0" diff --git a/test/test_nacm_module_write.sh b/test/test_nacm_module_write.sh index 1d4bab13..9457ecf9 100755 --- a/test/test_nacm_module_write.sh +++ b/test/test_nacm_module_write.sh @@ -42,6 +42,7 @@ cat < $cfg /usr/local/lib/$APPNAME/cli $APPNAME /usr/local/var/$APPNAME/$APPNAME.sock + false /usr/local/lib/$APPNAME/backend /usr/local/var/$APPNAME/$APPNAME.pidfile 1 diff --git a/test/test_netconf.sh b/test/test_netconf.sh index 129cb93a..6ec9e127 100755 --- a/test/test_netconf.sh +++ b/test/test_netconf.sh @@ -159,10 +159,10 @@ new "netconf discard-changes" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "netconf edit state operation should fail" -expecteof "$clixon_netconf -qf $cfg" 0 '42]]>]]>' "^protocolinvalid-value" +expecteof "$clixon_netconf -qf $cfg" 0 'e0up]]>]]>' "^protocolinvalid-valueerrorState data not allowed]]>]]>" new "netconf get state operation" -expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^424143]]>]]>$' +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" '^eth1ex:ethtrueup]]>]]>$' new "netconf lock/unlock" expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>]]>]]>" "^]]>]]>]]>]]>$" diff --git a/test/test_perf.sh b/test/test_perf.sh index 7adb1ba4..94bca500 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -1,5 +1,8 @@ #!/bin/bash # Scaling/ performance tests +# CLI/Netconf/Restconf +# Lists (and leaf-lists) +# Add, get and delete entries # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi @@ -8,7 +11,7 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi : ${format:=xml} # Number of list/leaf-list entries in file -: ${perfnr:=10000} +: ${perfnr:=5000} # Number of requests made get/put : ${perfreq:=100} @@ -20,7 +23,6 @@ fyang=$dir/scaling.yang fconfig=$dir/large.xml fconfig2=$dir/large2.xml - cat < $fyang module scaling{ yang-version 1.1; @@ -48,7 +50,7 @@ cat < $cfg $cfg $dir /usr/local/share/clixon - scaling + $fyang /usr/local/var/$APPNAME/$APPNAME.sock /usr/local/var/example/$APPNAME.pidfile false @@ -65,89 +67,27 @@ cat < $cfg EOF +new "test params: -f $cfg" if [ $BE -ne 0 ]; then new "kill old backend" - sudo clixon_backend -zf $cfg -y $fyang - if [ $? -ne 0 ]; then - err - fi -fi - -# First generate large XML file -# Use it latter to generate startup-db in xml, tree formats -tmpx=$dir/tmp.xml -new "generate large startup config ($tmpx) with $perfnr entries" -echo -n "" > $tmpx -for (( i=0; i<$perfnr; i++ )); do - echo -n "$i$i" >> $tmpx -done -echo "" >> $tmpx - -if false; then -# Then generate large JSON file (cant translate namespace - long story) -tmpj=$dir/tmp.json -new "generate large startup config ($tmpj) with $perfnr entries" -echo -n '{"config": {"scaling:x":{"y":[' > $tmpj -for (( i=0; i<$perfnr; i++ )); do - if [ $i -ne 0 ]; then - echo -n ",{\"a\":$i,\"b\":$i}" >> $tmpj - else - echo -n "{\"a\":$i,\"b\":$i}" >> $tmpj - fi - -done -echo "]}}}" >> $tmpj -fi - -# Loop over mode and format -for mode in startup running; do - file=$dir/${mode}_db - for format in tree xml; do # json - something w namespaces - sudo rm -f $file - sudo touch $file - sudo chmod 666 $file - case $format in - xml) - echo "cp $tmpx $file" - cp $tmpx $file - ;; - json) - cp $tmpj $file - ;; - tree) - echo "clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create" - - clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create - ;; - esac - new "Startup format: $format mode:$mode" -# echo "time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format" - # Cannot use start_backend here due to expected error case -{ time -p sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format 2> /dev/null; } 2>&1 | awk '/real/ {print $2}' - - done -done - -new "test params: -f $cfg -y $fyang" -if [ $BE -ne 0 ]; then - new "kill old backend" - sudo clixon_backend -zf $cfg -y $fyang + sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg -y $fyang" - start_backend -s init -f $cfg -y $fyang + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s fi new "kill old restconf daemon" sudo pkill -u www-data -f "/www-data/clixon_restconf" new "start restconf daemon" -start_restconf -f $cfg -y $fyang +start_restconf -f $cfg new "waiting" -sleep $RCWAIT +wait_backend +wait_restconf new "generate 'large' config with $perfnr list entries" echo -n "" > $fconfig @@ -158,63 +98,106 @@ echo "]]>]]>" >> $fconfig # Now take large config file and write it via netconf to candidate new "netconf write large config" -expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg" "$fconfig" "^]]>]]>$" # Here, there are $perfnr entries in candidate new "netconf write large config again" -expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg" "$fconfig" "^]]>]]>$" # Now commit it from candidate to running new "netconf commit large config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" # Now commit it again from candidate (validation takes time when # comparing to existing) new "netconf commit large config again" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" # Having a large db, get and put single entries many times -# Note same entries in the range alreayd there, db has same size -new "netconf add $perfreq small config" -time -p for (( i=0; i<$perfreq; i++ )); do - rnd=$(( ( RANDOM % $perfnr ) )) - echo "$rnd$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null +# Note same entries in the range alreay there, db has same size +# NETCONF get new "netconf get $perfreq small config" -time -p for (( i=0; i<$perfreq; i++ )); do +{ time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) echo "]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null +done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' +# NETCONF add +new "netconf add $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + echo "$rnd$rnd]]>]]>" +done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +# RESTCONF get new "restconf get $perfreq small config" -time -p for (( i=0; i<$perfreq; i++ )); do +{ time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null -done +done } 2>&1 | awk '/real/ {print $2}' +# RESTCONF put # Reference: # i686 format=xml perfnr=10000/100 time: 38/29s 20190425 WITH/OUT startup copying # i686 format=tree perfnr=10000/100 time: 72/64s 20190425 WITH/OUT startup copying new "restconf add $perfreq small config" -time -p for (( i=0; i<$perfreq; i++ )); do +{ time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) curl -s -X PUT http://localhost/restconf/data/scaling:x/y=$rnd -d '{"scaling:y":{"a":"'$rnd'","b":"'$rnd'"}}' -done +done } 2>&1 | awk '/real/ {print $2}' + +# CLI get +new "cli get $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg show conf xml x y $rnd > /dev/null +done } 2>&1 | awk '/real/ {print $2}' + +# CLI add +new "cli add $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg set x y $rnd b $rnd +done } 2>&1 | awk '/real/ {print $2}' # Instead of many small entries, get one large in netconf and restconf +# cli? new "netconf get large config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" '^00112233' +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" '^00112233' new "restconf get large config" -expecteof "/usr/bin/time -f %e curl -sG http://localhost/restconf/data" 0 "]]>]]>" '^{"data": {"scaling:x": {"y": \[{"a": 0,"b": 0},{ "a": 1,"b": 1},{ "a": 2,"b": 2},{ "a": 3,"b": 3},' +/usr/bin/time -f %e curl -sG http://localhost/restconf/data > /dev/null + +new "cli get large config" +/usr/bin/time -f %e $clixon_cli -1f $cfg show config xml> /dev/null + +# Delete entries (last since entries are removed from db) +# netconf +new "cli delete $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg delete x y $rnd +done } 2>&1 | awk '/real/ {print $2}' + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "netconf delete $perfreq small config" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + echo "$rnd]]>]]>" +done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +new "netconf discard-changes" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "restconf delete $perfreq small config" -time -p for (( i=0; i<$perfreq; i++ )); do +{ time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) curl -s -X DELETE http://localhost/restconf/data/scaling:x/y=$rnd -done - +done > /dev/null; } 2>&1 | awk '/real/ {print $2}' +exit # Now do leaf-lists istead of leafs new "generate large leaf-list config" @@ -225,25 +208,25 @@ done echo "]]>]]>" >> $fconfig2 new "netconf replace large list-leaf config" -expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig2" "^]]>]]>$" +expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg" "$fconfig2" "^]]>]]>$" new "netconf commit large leaf-list config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "netconf add $perfreq small leaf-list config" time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) echo "$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null +done | $clixon_netconf -qf $cfg > /dev/null new "netconf add small leaf-list config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 'x]]>]]>' "^]]>]]>$" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 'x]]>]]>' "^]]>]]>$" new "netconf commit small leaf-list config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" new "netconf get large leaf-list config" -expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" '^01' +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" '^01' new "Kill restconf daemon" stop_restconf diff --git a/test/test_perf_startup.sh b/test/test_perf_startup.sh new file mode 100755 index 00000000..c6fbe95c --- /dev/null +++ b/test/test_perf_startup.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Startup performance tests for different formats and startup modes. + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +# Number of list/leaf-list entries in file +: ${perfnr:=10000} + +APPNAME=example + +cfg=$dir/scaling-conf.xml +fyang=$dir/scaling.yang + +cat < $fyang +module scaling{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix ip; + container x { + list y { + key "a"; + leaf a { + type int32; + } + leaf b { + type int32; + } + } + leaf-list c { + type string; + } + } +} +EOF + +cat < $cfg + + $cfg + $dir + /usr/local/share/clixon + $fyang + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/example/$APPNAME.pidfile + false + $dir + false + example + /usr/local/lib/example/cli + /usr/local/lib/example/clispec + 1 + VARS + 0 + ietf-netconf:startup + +EOF + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi +fi + +# First generate large XML file +# Use it latter to generate startup-db in xml, tree formats +tmpx=$dir/tmp.xml +new "generate large startup config ($tmpx) with $perfnr entries" +echo -n "" > $tmpx +for (( i=0; i<$perfnr; i++ )); do + echo -n "$i$i" >> $tmpx +done +echo "" >> $tmpx + +if false; then # XXX JSON dont work as datastore yet +# Then generate large JSON file (cant translate namespace - long story) +tmpj=$dir/tmp.json +new "generate large startup config ($tmpj) with $perfnr entries" +echo -n '{"config": {"scaling:x":{"y":[' > $tmpj +for (( i=0; i<$perfnr; i++ )); do + if [ $i -ne 0 ]; then + echo -n ",{\"a\":$i,\"b\":$i}" >> $tmpj + else + echo -n "{\"a\":$i,\"b\":$i}" >> $tmpj + fi + +done +echo "]}}}" >> $tmpj +fi + +# Loop over mode and format +for mode in startup running; do + file=$dir/${mode}_db + for format in tree xml; do # json - something w namespaces + sudo rm -f $file + sudo touch $file + sudo chmod 666 $file + case $format in + xml) + echo "cp $tmpx $file" + cp $tmpx $file + ;; + json) + cp $tmpj $file + ;; + tree) + echo "clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create" + + clixon_util_datastore -d ${mode} -f tree -y $fyang -b $dir -x $tmpx put create + ;; + esac + new "Startup format: $format mode:$mode" +# echo "time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format" + # Cannot use start_backend here due to expected error case +{ time -p sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang -o CLICON_XMLDB_FORMAT=$format 2> /dev/null; } 2>&1 | awk '/real/ {print $2}' + + done +done + +rm -rf $dir diff --git a/test/test_perf_state.sh b/test/test_perf_state.sh new file mode 100755 index 00000000..8c4d76b7 --- /dev/null +++ b/test/test_perf_state.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# Scaling/ performance tests +# Config + state data, only get +# Restconf/Netconf/CLI +# Use mixed ietf-interfaces config+state + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +# Which format to use as datastore format internally +: ${format:=xml} + +# Number of list/leaf-list entries in file (cant be less than 2) +: ${perfnr:=1000} + +# Number of requests made get/put +: ${perfreq:=100} + +APPNAME=example + +cfg=$dir/config.xml +fconfig=$dir/large.xml + +cat < $cfg + + $cfg + $dir + /usr/local/share/clixon + clixon-example + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/example/$APPNAME.pidfile + /usr/local/lib/$APPNAME/backend + example_backend.so$ + false + $dir + false + $format + example + /usr/local/lib/example/cli + /usr/local/lib/example/clispec + 1 + VARS + 0 + ietf-netconf:startup + +EOF + +new "test params: -f $cfg" +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + + new "start backend -s init -f $cfg -- -s" + start_backend -s init -f $cfg -- -s +fi + +new "kill old restconf daemon" +sudo pkill -u www-data -f "/www-data/clixon_restconf" + +new "start restconf daemon" +start_restconf -f $cfg + +new "waiting" +wait_backend +wait_restconf + +new "generate 'large' config with $perfnr list entries" +echo -n "" > $fconfig +for (( i=0; i<$perfnr; i++ )); do + echo -n "e$iex:eth" >> $fconfig +done +echo "]]>]]>" >> $fconfig + +# Now take large config file and write it via netconf to candidate +new "netconf write large config" +expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg" "$fconfig" "^]]>]]>$" + +# Now commit it from candidate to running +new "netconf commit large config" +expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +# START actual tests + +# Having a large db, get single entries many times +# NETCONF get +new "netconf get test single req" +sel="/ietf-interfaces:interfaces/interface[name='e1']" +msg="]]>]]>" +expecteof "$clixon_netconf -qf $cfg" 0 "$msg" '^e1ex:ethtrueup]]>]]>$' + +new "netconf get $perfreq single reqs" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + sel="/ietf-interfaces:interfaces/interface[name='e$rnd']" + echo "]]>]]>" +done | $clixon_netconf -qf $cfg > /dev/null; } 2>&1 | awk '/real/ {print $2}' + +# RESTCONF get +new "restconf get test single req" +expecteq "$(curl -s -X GET http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e1)" 0 '{"ietf-interfaces:interface": [{"name": "e1","type": "ex:eth","enabled": true,"oper-status": "up"}]} + ' + +new "restconf get $perfreq single reqs" +#echo "curl -sG http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e0" +#curl -sG http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e67 + +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + curl -sG http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e$rnd > /dev/null +done } 2>&1 | awk '/real/ {print $2}' + +# CLI get +new "cli get test single req" +expectfn "$clixon_cli -1 -1f $cfg -l o show state xml interfaces interface e1" 0 '^ + e1 + ex:eth + true + up +$' + +new "cli get $perfreq single reqs" +{ time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + $clixon_cli -1 -f $cfg show state xml interfaces interface e$rnd > /dev/null +done } 2>&1 | awk '/real/ {print $2}' + +# Get config in one large get +new "netconf get large config" +/usr/bin/time -f %e echo " ]]>]]>" | $clixon_netconf -qf $cfg > /tmp/netconf + +new "restconf get large config" +/usr/bin/time -f %e curl -sG http://localhost/restconf/data/ietf-interfaces:interfaces | wc + +new "cli get large config" +/usr/bin/time -f %e $clixon_cli -1f $cfg show state xml interfaces | wc + +new "Kill restconf daemon" +stop_restconf + +if [ $BE -eq 0 ]; then + exit # BE +fi + +new "Kill backend" +# Check if premature kill +pid=`pgrep -u root -f clixon_backend` +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +stop_backend -f $cfg + + +rm -rf $dir diff --git a/test/test_restconf.sh b/test/test_restconf.sh index a2e9ff59..61659a88 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -1,5 +1,6 @@ #!/bin/bash # Restconf basic functionality +# also uri encoding using eth/0/0 # Assume http server setup, such as nginx described in apps/restconf/README.md # Magic line must be first in script (see README.md) @@ -35,7 +36,7 @@ EOF # This is a fixed 'state' implemented in routing_backend. It is assumed to be always there state='{"clixon-example:state": {"op": ["42","41","43"]}} ' -new "test params: -f $cfg" +new "test params: -f $cfg -- -s" if [ $BE -ne 0 ]; then new "kill old backend" sudo clixon_backend -zf $cfg @@ -114,11 +115,7 @@ new "restconf empty rpc with extra args (should fail)" expecteq "$(curl -s -X POST -d {\"clixon-example:input\":{\"extra\":null}} http://localhost/restconf/operations/clixon-example:empty)" 0 '{"ietf-restconf:errors" : {"error": {"error-type": "application","error-tag": "unknown-element","error-info": {"bad-element": "extra"},"error-severity": "error"}}} ' new "restconf get empty config + state json" -expecteq "$(curl -sSG http://localhost/restconf/data/clixon-example:state)" 0 '{"clixon-example:state": {"op": ["42","41","43"]}} - ' - -new "restconf get empty config + state json + module" -expecteq "$(curl -sSG http://localhost/restconf/data/clixon-example:state)" 0 '{"clixon-example:state": {"op": ["42","41","43"]}} +expecteq "$(curl -sS -X GET http://localhost/restconf/data/clixon-example:state)" 0 '{"clixon-example:state": {"op": ["42","41","43"]}} ' new "restconf get empty config + state json with wrong module name" @@ -136,7 +133,7 @@ new "restconf get data/ json" expecteq "$(curl -s -G http://localhost/restconf/data/clixon-example:state/op=42)" 0 '{"clixon-example:op": ["42","41","43"]} ' -new "restconf get state operation eth0 xml" +new "restconf get state operation" # Cant get shell macros to work, inline matching from lib.sh ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/clixon-example:state/op=42) expect='42' @@ -145,11 +142,11 @@ if [ -z "$match" ]; then err "$expect" "$ret" fi -new "restconf get state operation eth0 type json" +new "restconf get state operation type json" expecteq "$(curl -s -G http://localhost/restconf/data/clixon-example:state/op=42)" 0 '{"clixon-example:op": ["42","41","43"]} ' -new "restconf get state operation eth0 type xml" +new "restconf get state operation type xml" # Cant get shell macros to work, inline matching from lib.sh ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/clixon-example:state/op=42) expect='42' @@ -163,17 +160,18 @@ expecteq "$(curl -s -X GET http://localhost/restconf/data/clixon-example:state)" ' # Exact match -new "restconf Add subtree to datastore using POST" +new "restconf Add subtree eth/0/0 to datastore using POST" expectfn 'curl -s -i -X POST -H "Accept: application/yang-data+json" -d {"ietf-interfaces:interfaces":{"interface":{"name":"eth/0/0","type":"ex:eth","enabled":true}}} http://localhost/restconf/data' 0 'HTTP/1.1 200 OK' -new "restconf Re-add subtree which should give error" +new "restconf Re-add subtree eth/0/0 which should give error" expectfn 'curl -s -X POST -d {"ietf-interfaces:interfaces":{"interface":{"name":"eth/0/0","type":"ex:eth","enabled":true}}} http://localhost/restconf/data' 0 '{"ietf-restconf:errors" : {"error": {"error-type": "application","error-tag": "data-exists","error-severity": "error","error-message": "Data already exists; cannot create new resource"}}}' # XXX Cant get this to work #expecteq "$(curl -s -X POST -d {\"interfaces\":{\"interface\":{\"name\":\"eth/0/0\",\"type\":\"ex:eth\",\"enabled\":true}}} http://localhost/restconf/data)" 0 '{"ietf-restconf:errors" : {"error": {"error-type": "application","error-tag": "data-exists","error-severity": "error","error-message": "Data already exists; cannot create new resource"}}}' new "restconf Check interfaces eth/0/0 added" -expectfn "curl -s -G http://localhost/restconf/data" 0 '"ietf-interfaces:interfaces": {"interface": \[{"name": "eth/0/0","type": "ex:eth","enabled": true}\]}' +expectfn "curl -s -X GET http://localhost/restconf/data/ietf-interfaces:interfaces" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","type": "ex:eth","enabled": true,"oper-status": "up"}]}}} + ' new "restconf delete interfaces" expecteq "$(curl -s -X DELETE http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 "" @@ -190,7 +188,7 @@ expectfn 'curl -s -X POST -d {"ietf-interfaces:interface":{"name":"eth/0/0","typ #expecteq "$(curl -s -X POST -d '{"interface":{"name":"eth/0/0","type\":"ex:eth","enabled":true}}' http://localhost/restconf/data/interfaces)" 0 "" new "restconf Check eth/0/0 added config" -expecteq "$(curl -s -G http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","type": "ex:eth","enabled": true}]}} +expecteq "$(curl -s -G http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","type": "ex:eth","enabled": true,"oper-status": "up"}]}} ' new "restconf Check eth/0/0 added state" @@ -207,7 +205,7 @@ new "Add nothing using POST" expectfn 'curl -s -X POST http://localhost/restconf/data/ietf-interfaces:interfaces/interface=eth%2f0%2f0' 0 '"ietf-restconf:errors" : {"error": {"error-type": "rpc","error-tag": "malformed-message","error-severity": "error","error-message": " on line 1: syntax error at or before:' new "restconf Check description added" -expecteq "$(curl -s -G http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","description": "The-first-interface","type": "ex:eth","enabled": true}]}} +expecteq "$(curl -s -G http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","description": "The-first-interface","type": "ex:eth","enabled": true,"oper-status": "up"}]}} ' new "restconf delete eth/0/0" @@ -223,7 +221,7 @@ new "restconf Add subtree eth/0/0 using PUT" expecteq "$(curl -s -X PUT -d '{"ietf-interfaces:interface":{"name":"eth/0/0","type":"ex:eth","enabled":true}}' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=eth%2f0%2f0)" 0 "" new "restconf get subtree" -expecteq "$(curl -s -G http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","type": "ex:eth","enabled": true}]}} +expecteq "$(curl -s -G http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 '{"ietf-interfaces:interfaces": {"interface": [{"name": "eth/0/0","type": "ex:eth","enabled": true,"oper-status": "up"}]}} ' new "restconf rpc using POST json" diff --git a/util/Makefile.in b/util/Makefile.in index a7eacd4c..f8422df5 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -44,6 +44,7 @@ localstatedir = @localstatedir@ sysconfdir = @sysconfdir@ HOST_VENDOR = @host_vendor@ with_restconf = @with_restconf@ +HAVE_LIBXML2 = @HAVE_LIBXML2@ SH_SUFFIX = @SH_SUFFIX@ @@ -59,7 +60,6 @@ INSTALL_LIB = @INSTALL@ INSTALLFLAGS = @INSTALLFLAGS@ LDFLAGS = @LDFLAGS@ LIBS = @LIBS@ - CPPFLAGS = @CPPFLAGS@ INCLUDES = -I. @INCLUDES@ -I$(top_srcdir)/lib -I$(top_srcdir)/include @@ -74,6 +74,7 @@ APPSRC += clixon_util_yang.c APPSRC += clixon_util_xpath.c APPSRC += clixon_util_datastore.c APPSRC += clixon_util_insert.c +APPSRC += clixon_util_regexp.c ifeq ($(with_restconf),yes) APPSRC += clixon_util_stream.c # Needs curl endif @@ -102,15 +103,20 @@ clixon_util_yang: clixon_util_yang.c $(LIBDEPS) clixon_util_xpath: clixon_util_xpath.c $(LIBDEPS) $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ -clixon_util_stream: clixon_util_stream.c $(LIBDEPS) - $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -lcurl -o $@ - clixon_util_datastore: clixon_util_datastore.c $(LIBDEPS) $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ clixon_util_insert: clixon_util_insert.c $(LIBDEPS) $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ +clixon_util_regexp: clixon_util_regexp.c $(LIBDEPS) + $(CC) $(INCLUDES) -I /usr/include/libxml2 $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ + +ifeq ($(with_restconf),yes) +clixon_util_stream: clixon_util_stream.c $(LIBDEPS) + $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -lcurl -o $@ +endif + distclean: clean rm -f Makefile *~ .depend diff --git a/util/clixon_util_insert.c b/util/clixon_util_insert.c index 0278ada8..4dc8ba70 100644 --- a/util/clixon_util_insert.c +++ b/util/clixon_util_insert.c @@ -31,16 +31,8 @@ ***** END LICENSE BLOCK ***** -See https://www.w3.org/TR/xpath/ - - * Turn this on to get an xpath test program - * Usage: xpath [] - * read xpath on first line and xml on rest of lines from input - * Example compile: - gcc -g -o xpath -I. -I../clixon ./clixon_xsl.c -lclixon -lcligen - * Example run: -echo "a\n" | xpath -*/ + * Utility for Inserting element in list + */ #ifdef HAVE_CONFIG_H #include "clixon_config.h" /* generated by config & autoconf */ diff --git a/util/clixon_util_regexp.c b/util/clixon_util_regexp.c new file mode 100644 index 00000000..7a547ed8 --- /dev/null +++ b/util/clixon_util_regexp.c @@ -0,0 +1,214 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * Utility for compiling regexp and checking validity + * gcc -I /usr/include/libxml2 regex.c -o regex -lxml2 + */ +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include /* unistd */ +#include +#include /* posix regex */ +#include + +#ifdef HAVE_LIBXML2 /* Actually it should check for a header file */ +#include +#endif + +/* cligen */ +#include + +/* clixon */ +#include "clixon/clixon.h" + +/*! libxml2 regex implementation + * @see http://www.w3.org/TR/2004/REC-xmlschema-2-20041028 + * @retval -1 Error + * @retval 0 Not match + * @retval 1 Match + */ +static int +regex_libxml2(char *regexp0, + char *content0, + int nr) +{ + int retval = -1; +#ifdef HAVE_LIBXML2 + xmlChar *regexp = (xmlChar*)regexp0; + xmlChar *content = (xmlChar*)content0; + xmlRegexp *xrp = NULL; + int ret; + int i; + + if ((xrp = xmlRegexpCompile(regexp)) == NULL) + goto done; + if (nr==0) + return 1; + for (i=0; i sizeof(pattern)-5){ + fprintf(stderr, "pattern too long\n"); + return -1; + } + strncpy(pattern, "^(", 2); + strncpy(pattern+2, posix, sizeof(pattern)-2); + strncat(pattern, ")$", sizeof(pattern)-len0-1); + if (regcomp(&re, pattern, REG_NOSUB|REG_EXTENDED) != 0) + return(0); /* report error */ + if (nr==0) + return 1; + for (i=0; i\tDebug\n" + "\t-p \txsd->posix translation regexp\n" + "\t-x \tlibxml2 regexp\n" + "\t-n \tIterate content match (0 means only compile)\n" + "\t-r \tregexp (mandatory)\n" + "\t-c \tValue content string(mandatory)\n", + argv0 + ); + exit(0); +} + +int +main(int argc, + char **argv) +{ + int retval = -1; + char *argv0 = argv[0]; + int c; + char *regexp = NULL; + char *content = NULL; + int posix = 0; + int libxml2 = 0; + int ret; + int nr = 1; + + optind = 1; + opterr = 0; + while ((c = getopt(argc, argv, "hD:pxn:r:c:")) != -1) + switch (c) { + case 'h': + usage(argv0); + break; + case 'D': + if (sscanf(optarg, "%d", &debug) != 1) + usage(argv0); + break; + case 'p': /* xsd->posix */ + posix++; + break; + case 'n': /* Number of iterations */ + if ((nr = atoi(optarg)) < 0) + usage(argv0); + break; + case 'x': /* libxml2 */ + libxml2++; + break; + case 'r': /* regexp */ + regexp = optarg; + break; + case 'c': /* value content string */ + content = optarg; + break; + default: + usage(argv[0]); + break; + } + clicon_log_init(__FILE__, debug?LOG_DEBUG:LOG_INFO, CLICON_LOG_STDERR); + if (regexp == NULL || content == NULL) + usage(argv0); + if (posix == libxml2) + usage(argv0); + clicon_debug(1, "regexp:%s", regexp); + clicon_debug(1, "content:%s", content); + if (libxml2){ + if ((ret = regex_libxml2(regexp, content, nr)) < 0) + goto done; + } + else if (posix){ + if ((ret = regex_posix(regexp, content, nr)) < 0) + goto done; + } + else + goto done; + fprintf(stdout, "%d\n", ret); + exit(ret); + retval = 0; + done: + return retval; +}