diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af8c90b5..df128ad7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - test-actions + - datastore-split pull_request: branches: [ master ] diff --git a/CHANGELOG.md b/CHANGELOG.md index f18794af..19c05100 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,17 +15,26 @@ Expected: June 2024 ### Features +* New: Split configure datastore multiple sub-files on mount-point boundaries + * Avoid writung sub-files without new data (dirty cache) +* Added: Code for SHA digests. * New: [Autolock](https://github.com/clicon/clixon/issues/508) * CLI configurable format: [Default format should be configurable](https://github.com/clicon/clixon-controller/issues/87) * CLI support for multiple inline commands separated by semi-colon * New `clixon-config@2024-04-01.yang` revision * Added options: - - `CLICON_NETCONF_DUPLICATE_ALLOW` - Disable duplicate check in NETCONF messages - - `CLICON_CLI_OUTPUT_FORMAT` - Default CLI output format - - `CLICON_AUTOLOCK` - Implicit locks + - `CLICON_XMLDB_MULTIPLE`: Split datastore into multiple sub files + - `CLICON_NETCONF_DUPLICATE_ALLOW`: Disable duplicate check in NETCONF messages + - `CLICON_CLI_OUTPUT_FORMAT`: Default CLI output format + - `CLICON_AUTOLOCK`: Implicit locks * New `clixon-lib@2024-04-01.yang` revision - Added: Default format +### API changes on existing protocol/config features +Users may have to change how they access the system + +* Openssl mandatory for all configs, not only restconf + ### Corrected Bugs * Fixed: Fail on return errors when reading from datastore diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index fadd7e1d..66a15754 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -251,7 +251,10 @@ startup_common(clixon_handle h, /* clear XML tree of defaults */ if (xml_tree_prune_flagged(xt, XML_FLAG_DEFAULT, 1) < 0) goto done; - if (xmldb_dump(h, stdout, xt, WITHDEFAULTS_REPORT_ALL) < 0) + + if (xmldb_dump(h, stdout, xt, FORMAT_XML, + clicon_option_bool(h, "CLICON_XMLDB_PRETTY"), + WITHDEFAULTS_REPORT_ALL, 0, NULL) < 0) goto done; if (xt) xml_free(xt); @@ -295,8 +298,7 @@ startup_common(clixon_handle h, xt = NULL; x = NULL; while ((x = xml_child_each(td->td_target, x, CX_ELMNT)) != NULL){ - xml_flag_set(x, XML_FLAG_ADD); /* Also down */ - xml_apply(x, CX_ELMNT, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_ADD); + xml_apply0(x, CX_ELMNT, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_ADD); if (cxvec_append(x, &td->td_avec, &td->td_alen) < 0) goto done; } diff --git a/configure b/configure index fda5ea2f..e0d6c478 100755 --- a/configure +++ b/configure @@ -6278,96 +6278,6 @@ fi printf "%s\n" "#define WITH_RESTCONF_FCGI 1" >>confdefs.h # For c-code that cant use strings elif test "x${with_restconf}" = xnative; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for OPENSSL_init_ssl in -lssl" >&5 -printf %s "checking for OPENSSL_init_ssl in -lssl... " >&6; } -if test ${ac_cv_lib_ssl_OPENSSL_init_ssl_+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lssl $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char OPENSSL_init_ssl (); -int -main (void) -{ -return OPENSSL_init_ssl (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_ssl_OPENSSL_init_ssl_=yes -else $as_nop - ac_cv_lib_ssl_OPENSSL_init_ssl_=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_OPENSSL_init_ssl_" >&5 -printf "%s\n" "$ac_cv_lib_ssl_OPENSSL_init_ssl_" >&6; } -if test "x$ac_cv_lib_ssl_OPENSSL_init_ssl_" = xyes -then : - printf "%s\n" "#define HAVE_LIBSSL 1" >>confdefs.h - - LIBS="-lssl $LIBS" - -else $as_nop - as_fn_error $? "libssl missing" "$LINENO" 5 -fi - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CRYPTO_new_ex_data in -lcrypto" >&5 -printf %s "checking for CRYPTO_new_ex_data in -lcrypto... " >&6; } -if test ${ac_cv_lib_crypto_CRYPTO_new_ex_data+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lcrypto $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char CRYPTO_new_ex_data (); -int -main (void) -{ -return CRYPTO_new_ex_data (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_crypto_CRYPTO_new_ex_data=yes -else $as_nop - ac_cv_lib_crypto_CRYPTO_new_ex_data=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_crypto_CRYPTO_new_ex_data" >&5 -printf "%s\n" "$ac_cv_lib_crypto_CRYPTO_new_ex_data" >&6; } -if test "x$ac_cv_lib_crypto_CRYPTO_new_ex_data" = xyes -then : - printf "%s\n" "#define HAVE_LIBCRYPTO 1" >>confdefs.h - - LIBS="-lcrypto $LIBS" - -else $as_nop - as_fn_error $? "libcrypto missing" "$LINENO" 5 -fi - # Check if http/1 enabled # Check whether --enable-http1 was given. if test ${enable_http1+y} @@ -6755,6 +6665,98 @@ then : fi +# This is for digest / restconf +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for OPENSSL_init_ssl in -lssl" >&5 +printf %s "checking for OPENSSL_init_ssl in -lssl... " >&6; } +if test ${ac_cv_lib_ssl_OPENSSL_init_ssl_+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-lssl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char OPENSSL_init_ssl (); +int +main (void) +{ +return OPENSSL_init_ssl (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_ssl_OPENSSL_init_ssl_=yes +else $as_nop + ac_cv_lib_ssl_OPENSSL_init_ssl_=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_OPENSSL_init_ssl_" >&5 +printf "%s\n" "$ac_cv_lib_ssl_OPENSSL_init_ssl_" >&6; } +if test "x$ac_cv_lib_ssl_OPENSSL_init_ssl_" = xyes +then : + printf "%s\n" "#define HAVE_LIBSSL 1" >>confdefs.h + + LIBS="-lssl $LIBS" + +else $as_nop + as_fn_error $? "libssl missing" "$LINENO" 5 +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CRYPTO_new_ex_data in -lcrypto" >&5 +printf %s "checking for CRYPTO_new_ex_data in -lcrypto... " >&6; } +if test ${ac_cv_lib_crypto_CRYPTO_new_ex_data+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-lcrypto $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char CRYPTO_new_ex_data (); +int +main (void) +{ +return CRYPTO_new_ex_data (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_crypto_CRYPTO_new_ex_data=yes +else $as_nop + ac_cv_lib_crypto_CRYPTO_new_ex_data=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_crypto_CRYPTO_new_ex_data" >&5 +printf "%s\n" "$ac_cv_lib_crypto_CRYPTO_new_ex_data" >&6; } +if test "x$ac_cv_lib_crypto_CRYPTO_new_ex_data" = xyes +then : + printf "%s\n" "#define HAVE_LIBCRYPTO 1" >>confdefs.h + + LIBS="-lcrypto $LIBS" + +else $as_nop + as_fn_error $? "libcrypto missing" "$LINENO" 5 +fi + + # This is for libxml2 XSD regex engine # Note this only enables the compiling of the code. In order to actually # use it you need to set Clixon config option CLICON_YANG_REGEXP to libxml2 diff --git a/configure.ac b/configure.ac index 77820bb4..0fda29d9 100644 --- a/configure.ac +++ b/configure.ac @@ -275,8 +275,6 @@ if test "x${with_restconf}" = xfcgi; then AC_CHECK_LIB(fcgi, FCGX_Init,, AC_MSG_ERROR([libfcgi-dev missing])) AC_DEFINE(WITH_RESTCONF_FCGI, 1, [Use fcgi restconf mode]) # For c-code that cant use strings elif test "x${with_restconf}" = xnative; then - AC_CHECK_LIB(ssl, OPENSSL_init_ssl ,, AC_MSG_ERROR([libssl missing])) - AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, , AC_MSG_ERROR([libcrypto missing])) # Check if http/1 enabled AC_ARG_ENABLE(http1, AS_HELP_STRING([--disable-http1],[Disable http1 for native restconf http/1, ie http/2 only]),[ if test "$enableval" = no; then @@ -373,6 +371,10 @@ AC_DEFINE_UNQUOTED(CLIXON_CONFIG_SYSCONFDIR, "${SYSCONFDIR}", [Pass-through $sys AC_CHECK_LIB(socket, socket) AC_CHECK_LIB(dl, dlopen) +# This is for digest / restconf +AC_CHECK_LIB(ssl, OPENSSL_init_ssl ,, AC_MSG_ERROR([libssl missing])) +AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, , AC_MSG_ERROR([libcrypto missing])) + # This is for libxml2 XSD regex engine # Note this only enables the compiling of the code. In order to actually # use it you need to set Clixon config option CLICON_YANG_REGEXP to libxml2 diff --git a/include/clixon_custom.h b/include/clixon_custom.h index 3d06724b..dcf970da 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -206,3 +206,8 @@ */ #define COMPAT_6_5 +/*! Use SHA256 (32 bytes) instead of SHA1 (20 bytes) + * + * Digest use is not cryptographic use, so SHA1 is enough for now + */ +#undef USE_SHA256 diff --git a/lib/clixon/clixon.h.in b/lib/clixon/clixon.h.in index f9329e18..daf621be 100644 --- a/lib/clixon/clixon.h.in +++ b/lib/clixon/clixon.h.in @@ -70,6 +70,7 @@ extern "C" { #include #include #include +#include #include #include #include diff --git a/lib/clixon/clixon_datastore.h b/lib/clixon/clixon_datastore.h index e5562adc..807407a2 100644 --- a/lib/clixon/clixon_datastore.h +++ b/lib/clixon/clixon_datastore.h @@ -55,7 +55,7 @@ struct db_elmnt { * reset by commit, discard */ int de_empty; /* Empty on read from file, xmldb_readfile and xmldb_put sets it */ - int de_volatile; /* Do not sync to disk on every update (ie xmldb_put) */ + int de_volatile; /* Disable auto-sync of cache to disk on every update (ie xmldb_put) */ }; typedef struct db_elmnt db_elmnt; @@ -66,16 +66,21 @@ typedef struct db_elmnt db_elmnt; db_elmnt *clicon_db_elmnt_get(clixon_handle h, const char *db); int clicon_db_elmnt_set(clixon_handle h, const char *db, db_elmnt *xc); int xmldb_db2file(clixon_handle h, const char *db, char **filename); +int xmldb_db2subdir(clixon_handle h, const char *db, char **dir); /* API */ int xmldb_connect(clixon_handle h); int xmldb_disconnect(clixon_handle h); - /* in clixon_datastore_read.[ch] */ + /* in clixon_datastore_read.[ch]: */ int xmldb_get(clixon_handle h, const char *db, cvec *nsc, char *xpath, cxobj **xret); int xmldb_get0(clixon_handle h, const char *db, yang_bind yb, cvec *nsc, const char *xpath, int copy, withdefaults_type wdef, cxobj **xret, modstate_diff_t *msd, cxobj **xerr); -int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret); /* in clixon_datastore_write.[ch] */ +/* in clixon_datastore_write.[ch]: */ +int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret); +int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, enum format_enum format, int pretty, withdefaults_type wdef, int multi, const char *multidb); +int xmldb_write_cache2file(clixon_handle h, const char *db); + int xmldb_copy(clixon_handle h, const char *from, const char *to); int xmldb_lock(clixon_handle h, const char *db, uint32_t id); int xmldb_unlock(clixon_handle h, const char *db); @@ -98,7 +103,5 @@ int xmldb_volatile_set(clixon_handle h, const char *db, int value); int xmldb_print(clixon_handle h, FILE *f); int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix); int xmldb_populate(clixon_handle h, const char *db); -int xmldb_write_cache2file(clixon_handle h, const char *db); -int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, withdefaults_type wdef); #endif /* _CLIXON_DATASTORE_H */ diff --git a/lib/clixon/clixon_digest.h b/lib/clixon/clixon_digest.h new file mode 100644 index 00000000..7c2bb47b --- /dev/null +++ b/lib/clixon/clixon_digest.h @@ -0,0 +1,43 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren + Copyright (C) 2017-2019 Olof Hagsand + Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC (Netgate) + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + */ + +#ifndef _CLIXON_DIGEST_H_ +#define _CLIXON_DIGEST_H_ + +int clixon_digest_hex(const char *string, char **hexstrp); + +#endif /* _CLIXON_DIGEST_H_ */ diff --git a/lib/clixon/clixon_file.h b/lib/clixon/clixon_file.h index 1c7a6fa3..5d73a199 100644 --- a/lib/clixon/clixon_file.h +++ b/lib/clixon/clixon_file.h @@ -43,6 +43,7 @@ int clicon_file_dirent(const char *dir, struct dirent **ent, const char *regexp, mode_t type); int clicon_files_recursive(const char *dir, const char *regexp, cvec *cvv); int clicon_file_copy(char *src, char *target); +int clicon_dir_copy(char *src, char *target); int clicon_file_cbuf(const char *filename, cbuf *cb); #endif /* _CLIXON_FILE_H_ */ diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 35455c60..9fcb8147 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -207,6 +207,7 @@ enum format_enum{ #define XML_FLAG_TOP 0x80 /* Top datastore symbol */ #define XML_FLAG_BODYKEY 0x100 /* Text parsing key to be translated from body to key */ #define XML_FLAG_ANYDATA 0x200 /* Treat as anydata, eg mount-points before bound */ +#define XML_FLAG_CACHE_DIRTY 0x400 /* This part of XML tree is not synced to disk */ /* * Prototypes diff --git a/lib/clixon/clixon_xml_io.h b/lib/clixon/clixon_xml_io.h index c44a5668..fba224c3 100644 --- a/lib/clixon/clixon_xml_io.h +++ b/lib/clixon/clixon_xml_io.h @@ -44,8 +44,12 @@ * Prototypes */ int clixon_xml2file1(FILE *f, cxobj *xn, int level, int pretty, char *prefix, - clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef); + clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef, + int multi); int clixon_xml2file(FILE *f, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext); +int clixon_xml2file_multi(clixon_handle h, const char *db, cxobj *xn, int level, int pretty, + char *prefix, clicon_output_cb *fn, int skiptop, int autocliext, + withdefaults_type wdef); int xml_print(FILE *f, cxobj *xn); int xml_dump(FILE *f, cxobj *x); int clixon_xml2cbuf1(cbuf *cb, cxobj *x, int level, int prettyprint, char *prefix, diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index 66acdab1..62abf160 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -87,7 +87,7 @@ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_debug.c clixon_err.c cli clixon_yang_cardinality.c clixon_yang_schema_mount.c \ clixon_xml_changelog.c clixon_xml_nsctx.c \ clixon_path.c clixon_validate.c clixon_validate_minmax.c \ - clixon_hash.c clixon_options.c clixon_data.c clixon_plugin.c \ + clixon_hash.c clixon_digest.c clixon_options.c clixon_data.c clixon_plugin.c \ clixon_proto.c clixon_proto_client.c \ clixon_xpath.c clixon_xpath_ctx.c clixon_xpath_eval.c clixon_xpath_function.c \ clixon_xpath_optimize.c clixon_xpath_yang.c \ diff --git a/lib/src/clixon_datastore.c b/lib/src/clixon_datastore.c index d23c406c..2174bbf0 100644 --- a/lib/src/clixon_datastore.c +++ b/lib/src/clixon_datastore.c @@ -127,7 +127,7 @@ clicon_db_elmnt_set(clixon_handle h, /*! Translate from symbolic database name to actual filename in file-system * - * @param[in] th text handle handle + * @param[in] h Clixon handle * @param[in] db Symbolic database name, eg "candidate", "running" * @param[out] filename Filename. Unallocate after use with free() * @retval 0 OK @@ -138,7 +138,7 @@ clicon_db_elmnt_set(clixon_handle h, * The filename reside in CLICON_XMLDB_DIR option */ int -xmldb_db2file(clixon_handle h, +xmldb_db2file(clixon_handle h, const char *db, char **filename) { @@ -151,7 +151,7 @@ xmldb_db2file(clixon_handle h, goto done; } if ((dir = clicon_xmldb_dir(h)) == NULL){ - clixon_err(OE_XML, errno, "dbdir not set"); + clixon_err(OE_XML, errno, "CLICON_XMLDB_DIR not set"); goto done; } cprintf(cb, "%s/%s_db", dir, db); @@ -166,6 +166,50 @@ xmldb_db2file(clixon_handle h, return retval; } +/*! Translate from symbolic database name to sub-directory of configure sub-files, no checks + * + * @param[in] h Clixon handle + * @param[in] db Symbolic database name, eg "candidate", "running" + * @param[out] subdirp Sub-directory name. Unallocate after use with free() + * @retval 0 OK + * @retval -1 Error + * The dir is subdir to CLICON_XMLDB_DIR option + * @see xmldb_db2file For top-level config file + */ +int +xmldb_db2subdir(clixon_handle h, + const char *db, + char **subdirp) +{ + int retval = -1; + cbuf *cb = NULL; + char *dir; + char *subdir = NULL; + + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + if ((dir = clicon_xmldb_dir(h)) == NULL){ + clixon_err(OE_XML, errno, "CLICON_XMLDB_DIR not set"); + goto done; + } + cprintf(cb, "%s/%s.d", dir, db); + if ((subdir = strdup4(cbuf_get(cb))) == NULL){ + clixon_err(OE_UNIX, errno, "strdup"); + goto done; + } + *subdirp = subdir; + subdir = NULL; + retval = 0; + done: + if (subdir) + free(subdir); + if (cb) + cbuf_free(cb); + return retval; +} + /*! Connect to a datastore plugin, allocate resources to be used in API calls * * @param[in] h Clixon handle @@ -209,11 +253,12 @@ xmldb_disconnect(clixon_handle h) return retval; } -/*! Copy database from db1 to db2 +/*! Copy datastore from db1 to db2 * + * May include copying datastore directory structure * @param[in] h Clixon handle - * @param[in] from Source database - * @param[in] to Destination database + * @param[in] from Source datastore + * @param[in] to Destination datastore * @retval 0 OK * @retval -1 Error */ @@ -267,8 +312,22 @@ xmldb_copy(clixon_handle h, if (de2) de0 = *de2; de0.de_xml = x2; /* The new tree */ - clicon_db_elmnt_set(h, to, &de0); + if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){ + char *subdir = NULL; + struct stat st = {0,}; + if (xmldb_db2subdir(h, to, &subdir) < 0) + goto done; + if (stat(subdir, &st) < 0){ + if (mkdir(subdir, S_IRWXU|S_IRGRP|S_IWGRP|S_IROTH|S_IXOTH) < 0){ + clixon_err(OE_UNIX, errno, "mkdir(%s)", subdir); + goto done; + } + } + if (subdir) + free(subdir); + } + clicon_db_elmnt_set(h, to, &de0); /* Copy the files themselves (above only in-memory cache) */ if (xmldb_db2file(h, from, &fromfile) < 0) goto done; @@ -276,6 +335,21 @@ xmldb_copy(clixon_handle h, goto done; if (clicon_file_copy(fromfile, tofile) < 0) goto done; + if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")) { + char *fromdir = NULL; + char *todir = NULL; + + if (xmldb_db2subdir(h, from, &fromdir) < 0) + goto done; + if (xmldb_db2subdir(h, to, &todir) < 0) + goto done; + if (clicon_dir_copy(fromdir, todir) < 0) + goto done; + if (fromdir) + free(fromdir); + if (todir) + free(todir); + } retval = 0; done: clixon_debug(CLIXON_DBG_DATASTORE, "retval:%d", retval); @@ -467,7 +541,7 @@ xmldb_clear(clixon_handle h, return 0; } -/*! Delete database, clear cache if any. Remove file +/*! Delete database, clear cache if any. Remove file and dir * * @param[in] h Clixon handle * @param[in] db Database @@ -480,23 +554,61 @@ int xmldb_delete(clixon_handle h, const char *db) { - int retval = -1; - char *filename = NULL; - struct stat sb; + int retval = -1; + char *filename = NULL; + struct stat st = {0,}; + cbuf *cb = NULL; + char *subdir = NULL; + struct dirent *dp = NULL; + int ndp; + int i; + char *regexp = NULL; clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "%s", db); if (xmldb_clear(h, db) < 0) goto done; if (xmldb_db2file(h, db, &filename) < 0) goto done; - if (lstat(filename, &sb) == 0) + if (lstat(filename, &st) == 0) if (truncate(filename, 0) < 0){ clixon_err(OE_DB, errno, "truncate %s", filename); goto done; } + if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){ + if (xmldb_db2subdir(h, db, &subdir) < 0) + goto done; + if (stat(subdir, &st) == 0){ + if ((ndp = clicon_file_dirent(subdir, &dp, regexp, S_IFREG)) < 0) + goto done; + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + for (i = 0; i < ndp; i++){ + cbuf_reset(cb); + cprintf(cb, "%s/%s", subdir, dp[i].d_name); + if (unlink(cbuf_get(cb)) < 0){ + clixon_err(OE_DB, errno, "unlink(%s)", cbuf_get(cb)); + goto done; + } + } + if (rmdir(subdir) < 0){ +#if 0 /* Ignore this for now, there are some cornercases where this is problamatic, see confirmed-commit */ + clixon_err(OE_DB, errno, "rmdir(%s)", subdir); + goto done; +#endif + } + } + } retval = 0; done: clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval); + if (dp) + free(dp); + if (cb) + cbuf_free(cb); + if (subdir) + free(subdir); if (filename) free(filename); return retval; @@ -518,6 +630,8 @@ xmldb_create(clixon_handle h, int fd = -1; db_elmnt *de = NULL; cxobj *xt = NULL; + char *subdir = NULL; + struct stat st = {0,}; clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "%s", db); if ((de = clicon_db_elmnt_get(h, db)) != NULL){ @@ -532,9 +646,21 @@ xmldb_create(clixon_handle h, clixon_err(OE_UNIX, errno, "open(%s)", filename); goto done; } + if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){ + if (xmldb_db2subdir(h, db, &subdir) < 0) + goto done; + if (stat(subdir, &st) < 0){ + if (mkdir(subdir, S_IRWXU|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH) < 0){ + clixon_err(OE_UNIX, errno, "mkdir(%s)", subdir); + goto done; + } + } + } retval = 0; done: clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval); + if (subdir) + free(subdir); if (filename) free(filename); if (fd != -1) @@ -671,9 +797,9 @@ xmldb_empty_set(clixon_handle h, return 0; } -/*! Get volatile flag of datastore +/*! Get volatile flag of datastore cache * - * Whether to sync to disk on every update (ie xmldb_put) + * Whether to sync cache to disk on every update (ie xmldb_put) * @param[in] h Clixon handle * @param[in] db Database name * @retval 1 Db was empty on load @@ -693,9 +819,9 @@ xmldb_volatile_get(clixon_handle h, return de->de_volatile; } -/*! Set datastore status of datastore +/*! Set datastore status of datastore cache * - * Whether to sync to disk on every update (ie xmldb_put) + * Whether to sync cache to disk on every update (ie xmldb_put) * @param[in] h Clixon handle * @param[in] db Database name * @param[in] value 0 or 1 @@ -830,104 +956,3 @@ xmldb_populate(clixon_handle h, done: return retval; } - -/* Dump a datastore to file, add modstate - * - * @param[in] h Clixon handle - * @param[in] db Name of database to search in (filename including dir path - * @param[in] xt Top of XML tree - * @param[in] wdef With-defaults parameter - * @retval 0 OK - * @retval -1 Error - */ -int -xmldb_dump(clixon_handle h, - FILE *f, - cxobj *xt, - withdefaults_type wdef) -{ - int retval = -1; - cxobj *xm; - cxobj *xmodst = NULL; - char *formatstr; - enum format_enum format = FORMAT_XML; - int pretty; - - /* Add modstate first */ - if ((xm = clicon_modst_cache_get(h, 1)) != NULL){ - if ((xmodst = xml_dup(xm)) == NULL) - goto done; - if (xml_child_insert_pos(xt, xmodst, 0) < 0) - goto done; - xml_parent_set(xmodst, xt); - } - pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY"); - if ((formatstr = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) != NULL){ - if ((format = format_str2int(formatstr)) < 0){ - clixon_err(OE_XML, 0, "Format %s invalid", formatstr); - goto done; - } - } - switch (format){ - case FORMAT_JSON: - if (clixon_json2file(f, xt, pretty, fprintf, 0, 0) < 0) - goto done; - break; - case FORMAT_XML: - if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef) < 0) - goto done; - break; - default: - clixon_err(OE_XML, 0, "Format %s not supported", format_int2str(format)); - goto done; - break; - } - /* Remove modules state after writing to file */ - if (xmodst && xml_purge(xmodst) < 0) - goto done; - retval = 0; - done: - return retval; -} - -/*! Given a datastore, write the cache to file - * - * Also add mod-state if applicable - * @param[in] h Clixon handle - * @param[in] db Name of database to search in (filename including dir path - * @retval 0 OK - * @retval -1 Error - */ -int -xmldb_write_cache2file(clixon_handle h, - const char *db) -{ - int retval = -1; - cxobj *xt; - FILE *f = NULL; - char *dbfile = NULL; - - if (xmldb_db2file(h, db, &dbfile) < 0) - goto done; - if (dbfile==NULL){ - clixon_err(OE_XML, 0, "dbfile NULL"); - goto done; - } - if ((xt = xmldb_cache_get(h, db)) == NULL){ - clixon_err(OE_XML, 0, "XML cache not found"); - goto done; - } - if ((f = fopen(dbfile, "w")) == NULL){ - clixon_err(OE_CFG, errno, "Creating file %s", dbfile); - goto done; - } - if (xmldb_dump(h, f, xt, WITHDEFAULTS_EXPLICIT) < 0) - goto done; - retval = 0; - done: - if (dbfile) - free(dbfile); - if (f) - fclose(f); - return retval; -} diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index 078d22ff..7bc3b783 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -86,6 +86,17 @@ #define handle(xh) (assert(text_handle_check(xh)==0),(struct text_handle *)(xh)) +/* Local types */ +/* Argument to apply for recursive call to xmldb_multi calls + * @see xmldb_multi_write_arg + */ +struct xmldb_multi_read_arg { + char *mr_subdir; + yang_stmt *mr_yspec; + enum format_enum mr_format; + cxobj **mr_xerr; +}; + /*! Ensure that xt only has a single sub-element and that is "config" * * @retval 0 There exists a single "config" sub-element @@ -446,6 +457,68 @@ disable_nacm_on_empty(cxobj *xt, return retval; } +/*! Callback function for xmldb-multi read + * + * Look for link attribute in XML, and if found open the linked file for parsing + * @param[in] x XML node + * @param[in] arg + * @retval 2 Locally abort this subtree, continue with others + * @retval 1 Abort, dont continue with others, return 1 to end user + * @retval 0 OK, continue + * @retval -1 Error, aborted at first error encounter, return -1 to end user + */ +static int +xmldb_multi_read_applyfn(cxobj *x, + void *arg) +{ + struct xmldb_multi_read_arg *mr = (struct xmldb_multi_read_arg *) arg; + int retval = -1; + cxobj *xa; + char *filename; + cbuf *cb = NULL; + char *dbfile; + FILE *fp = NULL; + + if ((xa = xml_find_type(x, CLIXON_LIB_PREFIX, "link", CX_ATTR)) != NULL && + (filename = xml_value(xa)) != NULL){ + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + cprintf(cb, "%s/%s", mr->mr_subdir, filename); + xml_purge(xa); + if ((xa = xml_find_type(x, "xmlns", CLIXON_LIB_PREFIX, CX_ATTR)) != NULL) + xml_purge(xa); + dbfile = cbuf_get(cb); + clixon_debug(CLIXON_DBG_DATASTORE, "Parsing: %s", dbfile); + if ((fp = fopen(dbfile, "r")) == NULL){ + clixon_err(OE_CFG, errno, "fdopen(%s)", dbfile); + goto done; + } + switch (mr->mr_format){ + case FORMAT_JSON: + if (clixon_json_parse_file(fp, 1, YB_NONE, mr->mr_yspec, &x, mr->mr_xerr) < 0) + goto done; + break; + case FORMAT_XML: + if (clixon_xml_parse_file(fp, YB_NONE, mr->mr_yspec, &x, mr->mr_xerr) < 0) + goto done; + break; + default: + clixon_err(OE_DB, 0, "Format not supported"); + goto done; + break; + } + } + retval = 0; + done: + if (cb) + cbuf_free(cb); + if (fp) + fclose(fp); + return retval; +} + /*! Common read function that reads an XML tree from file * * @param[in] th Datastore text handle @@ -477,7 +550,8 @@ xmldb_readfile(clixon_handle h, cxobj *x0 = NULL; char *dbfile = NULL; FILE *fp = NULL; - char *format; + char *formatstr; + enum format_enum format; int ret; modstate_diff_t *msdiff = NULL; cxobj *xmsd; /* XML module state diff */ @@ -489,6 +563,7 @@ xmldb_readfile(clixon_handle h, cxobj *xmodfile = NULL; cxobj *x; yang_stmt *yspec1 = NULL; + struct xmldb_multi_read_arg mr = {0, }; if (yb != YB_MODULE && yb != YB_NONE){ clixon_err(OE_XML, EINVAL, "yb is %d but should be module or none", yb); @@ -500,11 +575,15 @@ xmldb_readfile(clixon_handle h, clixon_err(OE_XML, 0, "dbfile NULL"); goto done; } - if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){ + if ((formatstr = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){ clixon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT"); goto done; } - clixon_debug(CLIXON_DBG_DATASTORE, "Reading datastore %s using %s", dbfile, format); + if ((format = format_str2int(formatstr)) < 0){ + clixon_err(OE_XML, 0, "format not found %s", formatstr); + goto done; + } + clixon_debug(CLIXON_DBG_DATASTORE, "Reading datastore %s using %s", dbfile, formatstr); /* Parse file into internal XML tree from different formats */ if ((fp = fopen(dbfile, "r")) == NULL) { clixon_err(OE_UNIX, errno, "open(%s)", dbfile); @@ -516,14 +595,28 @@ xmldb_readfile(clixon_handle h, * config* * * ret == 0 should not happen with YB_NONE. Binding is done later */ - if (strcmp(format, "json")==0){ + switch (format){ + case FORMAT_JSON: if (clixon_json_parse_file(fp, 1, YB_NONE, yspec, &x0, xerr) < 0) goto done; - } - else { - if (clixon_xml_parse_file(fp, YB_NONE, yspec, &x0, xerr) < 0){ + break; + case FORMAT_XML: + if (clixon_xml_parse_file(fp, YB_NONE, yspec, &x0, xerr) < 0) + goto done; + break; + default: + clixon_err(OE_DB, 0, "Format %s not supported", formatstr); + goto done; + break; + } + if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){ + if (xmldb_db2subdir(h, db, &mr.mr_subdir) < 0) + goto done; + mr.mr_format = format; + mr.mr_yspec = yspec; + mr.mr_xerr = xerr; + if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xmldb_multi_read_applyfn, &mr) < 0) goto done; - } } /* Always assert a top-level called "config". * To ensure that, deal with two cases: @@ -646,6 +739,8 @@ xmldb_readfile(clixon_handle h, } retval = 1; done: + if (mr.mr_subdir) + free(mr.mr_subdir); if (yspec1) ys_free1(yspec1, 1); if (xmodfile) diff --git a/lib/src/clixon_datastore_write.c b/lib/src/clixon_datastore_write.c index e41d8567..89e0ba7e 100644 --- a/lib/src/clixon_datastore_write.c +++ b/lib/src/clixon_datastore_write.c @@ -45,11 +45,12 @@ #include #include #include -#include #include #include #include #include +#include +#include /* cligen */ #include @@ -58,6 +59,7 @@ #include "clixon_string.h" #include "clixon_queue.h" #include "clixon_hash.h" +#include "clixon_digest.h" #include "clixon_handle.h" #include "clixon_yang.h" #include "clixon_xml.h" @@ -85,6 +87,18 @@ #include "clixon_datastore_write.h" #include "clixon_datastore_read.h" +/* Local types */ +/* Argument to apply for recursive call to xmldb_multi write calls + * @see xmldb_multi_read_arg +*/ +struct xmldb_multi_write_arg { + clixon_handle *mw_h; + const char *mw_db; + int mw_pretty; + withdefaults_type mw_wdef; + enum format_enum mw_format; +}; + /*! Given an attribute name and its expected namespace, find its value * * An attribute may have a prefix(or NULL). The routine finds the associated @@ -691,6 +705,7 @@ text_modify(clixon_handle h, } if (xml_value_set(x0b, x1bstr) < 0) goto done; + xml_flag_set(x0, XML_FLAG_ADD); /* If a default value ies replaced, then reset default flag */ if (xml_flag(x0, XML_FLAG_DEFAULT)) xml_flag_reset(x0, XML_FLAG_DEFAULT); @@ -1167,7 +1182,7 @@ text_modify_top(clixon_handle h, goto done; } /* text_modify_top */ -/*! Callback function type for xml_apply +/*! Mark ancestor if any changes to children. Also mark changed xml as cache dirty * * @param[in] x XML node * @param[in] arg General-purpose argument @@ -1185,12 +1200,27 @@ xml_mark_added_ancestors(cxobj *x, int flags = (intptr_t)arg; if (xml_flag(x, flags)){ - xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE); + xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)(XML_FLAG_CHANGE)); return 2; } return 0; } +static int +xml_mark_cache_dirty(cxobj *x, + void *arg) +{ + if (xml_flag(x, XML_FLAG_CHANGE)){ + xml_flag_set(x, XML_FLAG_CACHE_DIRTY); + return 0; + } + else if (xml_flag(x, XML_FLAG_ADD|XML_FLAG_DEL)){ + if (xml_apply0(x, CX_ELMNT, (xml_applyfn_t*)xml_flag_set, (void*)(XML_FLAG_CACHE_DIRTY)) < 0) + return -1; + } + return 2; +} + /*! Modify database given an xml tree and an operation * * @param[in] h CLICON handle @@ -1291,8 +1321,12 @@ xmldb_put(clixon_handle h, /* Remove NONE nodes if all subs recursively are also NONE */ if (xml_tree_prune_flagged_sub(x0, XML_FLAG_NONE, 0, NULL) <0) goto done; + /* Mark ancestor if any changes to children. */ if (xml_apply(x0, CX_ELMNT, xml_mark_added_ancestors, (void*)(XML_FLAG_ADD|XML_FLAG_DEL)) < 0) goto done; + /* Mark changed xml as cache dirty */ + if (xml_apply(x0, CX_ELMNT, xml_mark_cache_dirty, NULL) < 0) + goto done; /* Remove empty non-presence containers recursively. */ if (xml_default_nopresence(x0, 3, XML_FLAG_ADD|XML_FLAG_DEL) < 0) @@ -1304,10 +1338,6 @@ xmldb_put(clixon_handle h, /* Add default recursive values */ if (xml_default_recurse(x0, 0, XML_FLAG_ADD|XML_FLAG_DEL) < 0) goto done; - /* Clear flags from previous steps */ - if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, - (void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE)) < 0) - goto done; /* Write back to datastore cache if first time */ if (de != NULL) de0 = *de; @@ -1315,10 +1345,21 @@ xmldb_put(clixon_handle h, de0.de_xml = x0; de0.de_empty = (xml_child_nr(de0.de_xml) == 0); clicon_db_elmnt_set(h, db, &de0); - /* Write cache to file unless volatile */ - if (xmldb_volatile_get(h, db) == 0) + /* Write cache to file unless volatile (ie stop syncing to store) */ + if (xmldb_volatile_get(h, db) == 0){ if (xmldb_write_cache2file(h, db) < 0) goto done; + /* Clear flags from previous steps + dirty */ + if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, + (void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE|XML_FLAG_CACHE_DIRTY)) < 0) + goto done; + } + else { + /* Clear flags from previous steps */ + if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, + (void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE)) < 0) + goto done; + } retval = 1; done: clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval); @@ -1331,3 +1372,203 @@ xmldb_put(clixon_handle h, retval = 0; goto done; } + +/*! Callback function for xmldb-multi write + * + * Look for link attribute in XML, and if found open the linked file for parsing + * @param[in] x XML node + * @param[in] arg + * @retval 2 Locally abort this subtree, continue with others + * @retval 1 Abort, dont continue with others, return 1 to end user + * @retval 0 OK, continue + * @retval -1 Error, aborted at first error encounter, return -1 to end user + */ +static int +xmldb_multi_write_applyfn(cxobj *x, + void *arg) +{ + struct xmldb_multi_write_arg *mw = (struct xmldb_multi_write_arg *) arg; + int retval = -1; + clixon_handle h = mw->mw_h; + yang_stmt *y; + int exist = 0; + char *xpath = NULL; + char *hexstr = NULL; + cbuf *cb = NULL; + char *subdir = NULL; + char *dbfile; + struct stat st = {0,}; + int fd = -1; + FILE *fsub = NULL; + + if (xml_child_nr_type(x, CX_ELMNT) > 0 && + (y = xml_spec(x)) != NULL){ + if (yang_extension_value(y, "xmldb-split", CLIXON_LIB_NS, &exist, NULL) < 0) + goto done; + if (exist){ + if (xml2xpath(x, NULL, 1, 0, &xpath) < 0) + goto done; + if (clixon_digest_hex(xpath, &hexstr) < 0) + goto done; + if (xmldb_db2subdir(h, mw->mw_db, &subdir) < 0) + goto done; + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + cprintf(cb, "%s/%s.xml", subdir, hexstr); + dbfile = cbuf_get(cb); + if (xml_flag(x, XML_FLAG_CACHE_DIRTY) || + lstat(dbfile, &st) < 0){ + clixon_debug(CLIXON_DBG_DATASTORE, "Open: %s for writing", dbfile); + if ((fd = open(dbfile, O_CREAT|O_WRONLY|O_TRUNC, S_IRWXU)) < 0) { + clixon_err(OE_UNIX, errno, "open(%s)", dbfile); + goto done; + } + if ((fsub = fdopen(fd, "w")) == NULL){ + clixon_err(OE_CFG, errno, "fdopen(%s)", dbfile); + goto done; + } + /* Dont recurse multi-file yet */ + if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0) < 0) + goto done; + } + } + } + retval = 0; + done: + if (fsub != NULL) + fclose(fsub); + if (cb) + cbuf_free(cb); + if (subdir) + free(subdir); + if (xpath) + free(xpath); + if (hexstr) + free(hexstr); + return retval; +} + +/* Given open file, xml-tree, and wdef, add modstate, get format and write to file + * + * @param[in] h Clixon handle + * @param[in] f Output file + * @param[in] xt Top of XML tree + * @param[in] format Output format + * @param[in] pretty Pretty-print + * @param[in] wdef With-defaults parameter + * @param[in] multi If split into multiple files + * @param[in] multidb Database name (only if multi) + * @retval 0 OK + * @retval -1 Error + */ +int +xmldb_dump(clixon_handle h, + FILE *f, + cxobj *xt, + enum format_enum format, + int pretty, + withdefaults_type wdef, + int multi, + const char *multidb) +{ + int retval = -1; + struct xmldb_multi_write_arg mw = {0,}; + cxobj *xm; + cxobj *xmodst = NULL; + + /* Add modstate */ + if ((xm = clicon_modst_cache_get(h, 1)) != NULL){ + if ((xmodst = xml_dup(xm)) == NULL) + goto done; + if (xml_child_insert_pos(xt, xmodst, 0) < 0) + goto done; + xml_parent_set(xmodst, xt); + } + switch (format){ + case FORMAT_XML: + if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi) < 0) + goto done; + if (multi){ + mw.mw_h = h; + mw.mw_db = multidb; + mw.mw_pretty = pretty; + mw.mw_wdef = wdef; + mw.mw_format = format; + if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xmldb_multi_write_applyfn, &mw) < 0) + goto done; + } + break; + case FORMAT_JSON: + if (multi){ + clixon_err(OE_CFG, errno, "JSON+multi not supported"); + goto done; + } + if (clixon_json2file(f, xt, pretty, fprintf, 0, 0) < 0) + goto done; + break; + default: + clixon_err(OE_XML, 0, "Format %s not supported", format_int2str(format)); + goto done; + break; + } + /* Remove modules state after writing to file */ + if (xmodst && xml_purge(xmodst) < 0) + goto done; + retval = 0; + done: + return retval; +} + + +/*! Given datastore, get cache and format, set wdef, add modstate and print to multiple files + * + * Also add mod-state if applicable + * @param[in] h Clixon handle + * @param[in] db Name of database to search in (filename including dir path + * @retval 0 OK + * @retval -1 Error + */ +int +xmldb_write_cache2file(clixon_handle h, + const char *db) +{ + int retval = -1; + cxobj *xt; + char *formatstr; + enum format_enum format = FORMAT_XML; + withdefaults_type wdef = WITHDEFAULTS_EXPLICIT; + int pretty; + int multi; + FILE *f = NULL; + char *dbfile = NULL; + + if ((xt = xmldb_cache_get(h, db)) == NULL){ + clixon_err(OE_XML, 0, "XML cache not found"); + goto done; + } + pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY"); + multi = clicon_option_bool(h, "CLICON_XMLDB_MULTI"); + if ((formatstr = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) != NULL){ + if ((format = format_str2int(formatstr)) < 0){ + clixon_err(OE_XML, 0, "Format %s invalid", formatstr); + goto done; + } + } + if (xmldb_db2file(h, db, &dbfile) < 0) + goto done; + if ((f = fopen(dbfile, "w")) == NULL){ + clixon_err(OE_CFG, errno, "fopen(%s)", dbfile); + goto done; + } + if (xmldb_dump(h, f, xt, format, pretty, wdef, multi, db) < 0) + goto done; + retval = 0; + done: + if (dbfile) + free(dbfile); + if (f) + fclose(f); + return retval; +} diff --git a/lib/src/clixon_datastore_write.h b/lib/src/clixon_datastore_write.h index b7bfd9f5..419d436d 100644 --- a/lib/src/clixon_datastore_write.h +++ b/lib/src/clixon_datastore_write.h @@ -46,5 +46,7 @@ * Prototypes */ int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret); +int xmldb_write_cache2file(clixon_handle h, const char *db); +int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, enum format_enum format, int pretty, withdefaults_type wdef, int multi, const char *multidb); #endif /* _CLIXON_DATASTORE_WRITE_H */ diff --git a/lib/src/clixon_digest.c b/lib/src/clixon_digest.c new file mode 100644 index 00000000..615be0a4 --- /dev/null +++ b/lib/src/clixon_digest.c @@ -0,0 +1,112 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2024 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include + +/* cligen */ +#include + +/* clixon */ +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_yang.h" +#include "clixon_xml.h" +#include "clixon_err.h" +#include "clixon_digest.h" + +/*! Given null-terminated string, return malloced NULL_terinated SHA digest string in hex + * + * @param[in] str Input null-terminated string + * @param[out] hexstrp Output null-terminated digest HEX string + * @retval 0 OK + * @retval -1 Error + */ +int +clixon_digest_hex(const char *str, + char **hexstrp) +{ + int retval = -1; + unsigned char *md = NULL; + cbuf *cb = NULL; + int i; + uint32_t len; + +#ifdef USE_SHA256 + len = SHA256_DIGEST_LENGTH; +#else + len = SHA_DIGEST_LENGTH; +#endif + if (str == NULL || hexstrp == NULL){ + clixon_err(OE_UNIX, EINVAL, "str or hexstr is NULL"); + return -1; + } + if ((md = calloc(len, sizeof(unsigned char))) == NULL){ + clixon_err(OE_UNIX, errno, "calloc"); + goto done; + } +#ifdef USE_SHA256 + if (SHA256((unsigned char *)str, strlen(str), md) == NULL) +#else + if (SHA1((unsigned char *)str, strlen(str), md) == NULL) +#endif + { + clixon_err(OE_UNIX, 0, "SHA256 error"); + goto done; + } + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + for (i = 0; i < len; i++) + cprintf(cb, "%02x", md[i]&0xff); + if ((*hexstrp = strdup(cbuf_get(cb))) == NULL){ + clixon_err(OE_UNIX, errno, "strdup"); + goto done; + } + retval = 0; + done: + if (cb) + cbuf_free(cb); + if (md) + free(md); + return retval; +} diff --git a/lib/src/clixon_file.c b/lib/src/clixon_file.c index 6ac54307..6df7931b 100644 --- a/lib/src/clixon_file.c +++ b/lib/src/clixon_file.c @@ -176,6 +176,7 @@ clicon_files_recursive(const char *dir, * @code * char *dir = "/root/fs"; * struct dirent *dp; + * int ndp; * if ((ndp = clicon_file_dirent(dir, &dp, "\\.so$", S_IFREG)) < 0) * return -1; * for (i = 0; i < ndp; i++) @@ -221,6 +222,8 @@ clicon_file_dirent(const char *dir, goto quit; } while((dent = readdir(dirp)) != NULL) { + if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0) + continue; /* Filename matching */ if (regexp) { if (regexec(&re, dent->d_name, (size_t) 0, NULL, 0) != 0) @@ -250,7 +253,6 @@ clicon_file_dirent(const char *dir, memcpy(&new[nent], dent, direntStructSize); nent++; } /* while */ - qsort((void *)new, nent, sizeof(*new), clicon_file_dirent_sort); *ent = new; new = NULL; @@ -271,6 +273,7 @@ quit: * @param[out] target Destination filename * @retval 0 OK * @retval -1 Error + * XXX This is probably faster with stdio? */ int clicon_file_copy(char *src, @@ -312,6 +315,43 @@ clicon_file_copy(char *src, return retval; } +/*! Make a copy of directory non-recursive + * + * @param[in] srcdir Source dirname + * @param[out] dstdir Destination dirname + * @retval 0 OK + * @retval -1 Error + */ +int +clicon_dir_copy(char *srcdir, + char *dstdir) +{ + int retval = -1; + struct dirent *dent = NULL; + DIR *dirp = NULL; + char srcfile[MAXPATHLEN]; + char dstfile[MAXPATHLEN]; + + if (srcdir == NULL || dstdir == NULL){ + clixon_err(OE_UNIX, EINVAL, "Requires src and dst dir != NULL"); + goto done; + } + if ((dirp = opendir(srcdir)) != NULL) + while ((dent = readdir(dirp)) != NULL) { + if (dent->d_type != DT_REG) + continue; + snprintf(srcfile, MAXPATHLEN-1, "%s/%s", srcdir, dent->d_name); + snprintf(dstfile, MAXPATHLEN-1, "%s/%s", dstdir, dent->d_name); + if (clicon_file_copy(srcfile, dstfile) < 0) + goto done; + } + retval = 0; + done: + if (dirp) + closedir(dirp); + return retval; +} + /*! Read content of file into cbuf * * @param[in] filename diff --git a/lib/src/clixon_hash.c b/lib/src/clixon_hash.c index eba99f13..fdf938bb 100644 --- a/lib/src/clixon_hash.c +++ b/lib/src/clixon_hash.c @@ -186,6 +186,7 @@ clicon_hash_lookup(clicon_hash_t *hash, } /*! Get value of hash + * * @param[in] hash Hash table * @param[in] key Variable name * @param[out] vlen Length of value (as returned by function if != NULL) diff --git a/lib/src/clixon_path.c b/lib/src/clixon_path.c index 808fe06a..01f60a5d 100644 --- a/lib/src/clixon_path.c +++ b/lib/src/clixon_path.c @@ -1896,8 +1896,10 @@ clixon_xml_find_instance_id(cxobj *xt, va_end(ap); if (instance_id_parse(path, &cplist) < 0) goto done; +#if 0 if (clixon_debug_get()) clixon_path_print(stderr, cplist); +#endif /* Resolve module:name to pointer to yang-stmt, fail if not successful */ if ((ret = instance_id_resolve(cplist, yt)) < 0) goto done; diff --git a/lib/src/clixon_stream.c b/lib/src/clixon_stream.c index 4f708bbc..01b48d3d 100644 --- a/lib/src/clixon_stream.c +++ b/lib/src/clixon_stream.c @@ -282,7 +282,7 @@ stream_timer_setup(int fd, struct stream_replay *r; struct stream_replay *r1; - clixon_debug(CLIXON_DBG_STREAM, ""); + clixon_debug(CLIXON_DBG_STREAM|CLIXON_DBG_DETAIL, ""); /* Go thru callbacks and see if any have timed out, if so remove them * Could also be done by a separate timer. */ diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index 3f9121b2..3ff576bc 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -81,7 +81,7 @@ * Constants */ /* How many XML children to start with if any. Then add quadratic until threshold when - * add lineraly + * add linearly * Heurestics: if child is body only single child is expected, but element children may * have siblings */ @@ -1234,7 +1234,6 @@ xml_new_body(char *name, return new_node; } - /*! Return yang spec of node. * * Not necessarily set. Either has not been set yet (by xml_spec_set( or anyxml. @@ -2025,7 +2024,7 @@ xml_copy_one(cxobj *x0, default: break; } - xml_flag_set(x1, xml_flag(x0, XML_FLAG_DEFAULT | XML_FLAG_TOP | XML_FLAG_ANYDATA)); /* Maybe more flags */ + xml_flag_set(x1, xml_flag(x0, XML_FLAG_DEFAULT | XML_FLAG_TOP | XML_FLAG_ANYDATA | XML_FLAG_CACHE_DIRTY)); /* Maybe more flags */ retval = 0; done: return retval; diff --git a/lib/src/clixon_xml_io.c b/lib/src/clixon_xml_io.c index 020cb0db..31e022e2 100644 --- a/lib/src/clixon_xml_io.c +++ b/lib/src/clixon_xml_io.c @@ -52,6 +52,9 @@ #include #include #include +#include +#include +#include /* cligen */ #include @@ -60,6 +63,7 @@ #include "clixon_string.h" #include "clixon_queue.h" #include "clixon_hash.h" +#include "clixon_digest.h" #include "clixon_handle.h" #include "clixon_yang.h" #include "clixon_xml.h" @@ -68,6 +72,7 @@ #include "clixon_debug.h" #include "clixon_options.h" #include "clixon_yang_module.h" +#include "clixon_yang_schema_mount.h" #include "clixon_xml_bind.h" #include "clixon_xml_vec.h" #include "clixon_xml_sort.h" @@ -75,6 +80,9 @@ #include "clixon_xml_parse.h" #include "clixon_netconf_lib.h" #include "clixon_xml_default.h" +#include "clixon_xpath_ctx.h" +#include "clixon_xpath.h" +#include "clixon_datastore.h" #include "clixon_xml_io.h" /* @@ -193,6 +201,7 @@ xml2output_wdef(cxobj *x, * @param[in] fn Callback to make print function * @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow * @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL + * @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI * @retval 0 OK * @retval -1 Error * One can use clixon_xml2cbuf to get common code, but using fprintf is @@ -212,7 +221,8 @@ xml2file_recurse(FILE *f, char *prefix, clicon_output_cb *fn, int autocliext, - withdefaults_type wdef) + withdefaults_type wdef, + int multi) { int retval = -1; char *name; @@ -227,6 +237,9 @@ xml2file_recurse(FILE *f, int level1; int tag = 0; int ret; + int subfile = 0; /* File is split into subfile */ + char *xpath = NULL; + char *hexstr = NULL; if (x == NULL) goto ok; @@ -277,7 +290,7 @@ xml2file_recurse(FILE *f, while ((xc = xml_child_each(x, xc, -1)) != NULL) { switch (xml_type(xc)){ case CX_ATTR: - if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef) <0) + if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi) < 0) goto done; break; case CX_BODY: @@ -296,9 +309,26 @@ xml2file_recurse(FILE *f, if (hasbody==0 && haselement==0) (*fn)(f, "/>"); else{ - (*fn)(f, ">"); - if (pretty && hasbody == 0){ - (*fn)(f, "\n"); + /* Check if this is a multi-file split-point */ + if (multi && (y = xml_spec(x)) != NULL){ + if (yang_extension_value(y, "xmldb-split", CLIXON_LIB_NS, &exist, NULL) < 0) + goto done; + if (exist){ + subfile++; + if (xml2xpath(x, NULL, 1, 0, &xpath) < 0) + goto done; + if (clixon_digest_hex(xpath, &hexstr) < 0) + goto done; + (*fn)(f, " xmlns:%s=\"%s\"", CLIXON_LIB_PREFIX, CLIXON_LIB_NS); + (*fn)(f, " %s:link=\"%s.xml\"", CLIXON_LIB_PREFIX, hexstr); + (*fn)(f, "/>"); + } + } + if (!subfile) { + (*fn)(f, ">"); + if (pretty && hasbody == 0){ + (*fn)(f, "\n"); + } } xc = NULL; while ((xc = xml_child_each(x, xc, -1)) != NULL) { @@ -316,23 +346,26 @@ xml2file_recurse(FILE *f, xa = xml_find_type(xc, IETF_NETCONF_WITH_DEFAULTS_ATTR_PREFIX, IETF_NETCONF_WITH_DEFAULTS_ATTR_NAMESPACE, CX_ATTR); } } - if (xml_type(xc) != CX_ATTR) - if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef) <0) - goto done; + if (xml_type(xc) != CX_ATTR && !subfile) + if (xml2file_recurse(f, xc, level+1, pretty, prefix, + fn, autocliext, wdef, multi) <0) + goto done; if (xa){ if (xml_purge(xa) < 0) goto done; } } - if (pretty && hasbody==0){ - if (pretty && prefix) - (*fn)(f, "%s", prefix); - (*fn)(f, "%*s", level1, ""); + if (subfile == 0){ + if (pretty && hasbody==0){ + if (pretty && prefix) + (*fn)(f, "%s", prefix); + (*fn)(f, "%*s", level1, ""); + } + (*fn)(f, "", name); } - (*fn)(f, "", name); } if (pretty){ (*fn)(f, "\n"); @@ -346,6 +379,10 @@ xml2file_recurse(FILE *f, done: if (encstr) free(encstr); + if (xpath) + free(xpath); + if (hexstr) + free(hexstr); return retval; } @@ -362,6 +399,7 @@ xml2file_recurse(FILE *f, * @param[in] skiptop 0: Include top object 1: Skip top-object, only children, * @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow * @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL + * @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI * @retval 0 OK * @retval -1 Error * @see clixon_xml2cbuf print to a cbuf string @@ -377,7 +415,8 @@ clixon_xml2file1(FILE *f, clicon_output_cb *fn, int skiptop, int autocliext, - withdefaults_type wdef) + withdefaults_type wdef, + int multi) { int retval = 1; cxobj *xc; @@ -387,11 +426,11 @@ clixon_xml2file1(FILE *f, if (skiptop){ xc = NULL; while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL) - if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef) < 0) + if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi) < 0) goto done; } else { - if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef) < 0) + if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi) < 0) goto done; } retval = 0; @@ -427,7 +466,7 @@ clixon_xml2file(FILE *f, int skiptop, int autocliext) { - return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0); + return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0); } /*! Print an XML tree structure to an output stream @@ -442,7 +481,7 @@ int xml_print(FILE *f, cxobj *x) { - return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL); + return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0); } /*! Dump cxobj structure with pointers and flags for debugging, internal function @@ -456,20 +495,20 @@ xml_dump1(FILE *f, if (xml_type(x) != CX_ELMNT) return 0; - fprintf(stderr, "%*s %s(%s)", + fprintf(f, "%*s %s(%s)", indent*PRETTYPRINT_INDENT, "", // x, xml_name(x), xml_type2str(xml_type(x))); if (xml_flag(x, XML_FLAG_ADD)) - fprintf(stderr, " add"); + fprintf(f, " add"); if (xml_flag(x, XML_FLAG_DEL)) - fprintf(stderr, " delete"); + fprintf(f, " delete"); if (xml_flag(x, XML_FLAG_CHANGE)) - fprintf(stderr, " change"); + fprintf(f, " change"); if (xml_flag(x, XML_FLAG_MARK)) - fprintf(stderr, " mark"); - fprintf(stderr, "\n"); + fprintf(f, " mark"); + fprintf(f, "\n"); xc = NULL; while ((xc = xml_child_each(x, xc, -1)) != NULL) { xml_dump1(f, xc, indent+1); @@ -949,11 +988,13 @@ clixon_xml_parse_file(FILE *fp, int xmlbuflen = BUFLEN; /* start size */ int oldxmlbuflen; int failed = 0; + int xtempty; /* empty on entry */ - if (xt==NULL || fp == NULL){ + if (xt == NULL || fp == NULL){ clixon_err(OE_XML, EINVAL, "arg is NULL"); return -1; } + xtempty = (*xt == NULL); if (yb == YB_MODULE && yspec == NULL){ clixon_err(OE_XML, EINVAL, "yspec is required if yb == YB_MODULE"); return -1; @@ -995,7 +1036,7 @@ clixon_xml_parse_file(FILE *fp, } /* while */ retval = (failed==0) ? 1 : 0; done: - if (retval < 0 && *xt){ + if (retval < 0 && *xt && xtempty){ free(*xt); *xt = NULL; } diff --git a/lib/src/clixon_yang_schema_mount.c b/lib/src/clixon_yang_schema_mount.c index 700c6b63..f35fdb76 100644 --- a/lib/src/clixon_yang_schema_mount.c +++ b/lib/src/clixon_yang_schema_mount.c @@ -138,7 +138,11 @@ yang_schema_mount_point0(yang_stmt *y) goto done; } -/*! Cached variant of yang_schema_mount_point +/*! Cached variant of yang_schema_mount_point0 + * + * @param[in] y Yang node + * @retval 1 Yes, node is potential mountpoint + * @retval 0 No, node is not potential mountpoint */ int yang_schema_mount_point(yang_stmt *y) @@ -250,9 +254,7 @@ xml_yang_mount_get(clixon_handle h, if ((y = xml_spec(xt)) == NULL) goto fail; - if ((ret = yang_schema_mount_point(y)) < 0) - goto done; - if (ret == 0) + if (yang_schema_mount_point(y) == 0) goto fail; /* Check validate level */ if (vl && clixon_plugin_yang_mount_all(h, xt, NULL, vl, NULL) < 0) diff --git a/test/test_datastore_multi.sh b/test/test_datastore_multi.sh new file mode 100755 index 00000000..56750a56 --- /dev/null +++ b/test/test_datastore_multi.sh @@ -0,0 +1,368 @@ +#!/usr/bin/env bash +# Datastore split test, eg x_db has x.d/ directory with subdirs# ALso test cache bevahour, that unmodified +# subteres are not touched +# For now subdirs only enabled for mointpoints, so this test is with mountpoints as well + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +# include err() and new() functions and creates $dir + +cfg=$dir/conf_yang.xml +clispec=$dir/automode.cli + +fyang=$dir/clixon-example.yang +fyang1=$dir/clixon-mount1.yang + +CFD=$dir/conf.d +test -d $CFD || mkdir -p $CFD + +AUTOCLI=$(autocli_config clixon-\* kw-nokey false) + +# Well-known digest of mount-point xpath +subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml + +cat < $cfg + + $cfg + $CFD + ${YANG_INSTALLDIR} + ${dir} + $fyang + true + $dir + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/run/$APPNAME.sock + /usr/local/lib/$APPNAME/backend + /usr/local/var/run/$APPNAME.pidfile + $dir + true + true + true + true + true + +EOF + +cat < $CFD/autocli.xml + + + false + kw-nokey + true + + include clixon + enable + clixon-* + + + +EOF + +cat < $fyang +module clixon-example{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix ex; + import ietf-yang-schema-mount { + prefix yangmnt; + } + import clixon-lib { + prefix cl; + } + container top{ + list mylist{ + key name; + leaf name{ + type string; + } + container root{ + presence "Otherwise root is not visible"; + yangmnt:mount-point "mylabel"{ + description "Root for other yang models"; + } + cl:xmldb-split; /* Multi-XMLDB: split datastore here */ + } + } + } +} +EOF + +cat < $fyang1 +module clixon-mount1{ + yang-version 1.1; + namespace "urn:example:mount1"; + prefix m1; + container mount1{ + list mylist1{ + key name1; + leaf name1{ + type string; + } + leaf value1 { + type string; + } + } + } + container extra{ + leaf extraval{ + type string; + } + } +} +EOF + +cat < $clispec +CLICON_MODE="example"; +CLICON_PROMPT="%U@%H %W> "; +CLICON_PLUGIN="example_cli"; + +# Autocli syntax tree operations +set @datamodel, cli_auto_set(); +merge @datamodel, cli_auto_merge(); +create @datamodel, cli_auto_create(); +delete("Delete a configuration item") @datamodel, cli_auto_del(); +validate("Validate changes"), cli_validate(); +commit("Commit the changes"), cli_commit(); +quit("Quit"), cli_quit(); +show("Show a particular state of the system"){ + configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{ + xml("Show configuration as XML"), cli_show_auto_mode("candidate", "xml", false, false); + cli("Show configuration as CLI commands"), cli_show_auto_mode("candidate", "cli", false, false, "report-all", "set "); + netconf("Show configuration as netconf edit-config operation"), cli_show_auto_mode("candidate", "netconf", false, false); + text("Show configuration as text"), cli_show_auto_mode("candidate", "text", false, false); + json("Show configuration as JSON"), cli_show_auto_mode("candidate", "json", false, false); + } + state("Show configuration and state"), cli_show_auto_mode("running", "xml", false, true); + compare("Compare candidate and running databases"), compare_dbs("running", "candidate", "xml"); +} +EOF + +# Check content of db +# Args: +# 0: dbname +# 1: subfile +function check_db() +{ + dbname=$1 + subfile=$2 + + sudo chmod o+r $dir/${dbname}_db + sudo chmod o+r $dir/${dbname}.d/$subfile + + sudo rm -f $dir/x_db + cat < $dir/x_db + + + + x + + + + +EOF + new "Check ${dbname}_db" + ret=$(diff $dir/x_db $dir/${dbname}_db) + if [ $? -ne 0 ]; then + err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)" + fi + cat < $dir/x_subfile + + + x1 + + + + foo + +EOF + new "Check ${dbname}.d/$subfile" + ret=$(diff $dir/x_subfile $dir/${dbname}.d/$subfile) + if [ $? -ne 0 ]; then + err "$(cat $dir/x_subfile)" "$(cat $dir/${dbname}.d/$subfile)" + fi +} + +new "test params: -f $cfg" + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg -- -m clixon-mount1 -M urn:example:mount1" + start_backend -s init -f $cfg -- -m clixon-mount1 -M urn:example:mount1 +fi + +new "wait backend" +wait_backend + +new "Add mountpoint x " +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "x" "" "" + +new "netconf commit" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" "" + +new "Add data to mount x" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx1foo" "" "" + +new "Check candidate after edit" +check_db candidate ${subfilename} + +s0=$(stat -c "%Y" $dir/candidate.d/${subfilename}) +sleep 1 + +new "Add 2nd data to mount x" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx2x2value" "" "" + +new "Check candidate subfile changed" +s1=$(stat -c "%Y" $dir/candidate.d/${subfilename}) +if [ $s0 -eq $s1 ]; then + err "Timestamp changed" "$s0 = $s1" +fi + +sleep 1 + +new "Change existing value in mount x" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx2x2new" "" "" + +new "Check candidate subfile changed" +s2=$(stat -c "%Y" $dir/candidate.d/${subfilename}) +if [ $s1 -eq $s2 ]; then + err "Timestamp changed" "$s1 = $s2" +fi + +sleep 1 + +new "Add data to top-level (not mount)" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "y" "" "" + +new "Check candidate subfile not changed" +s3=$(stat -c "%Y" $dir/candidate.d/${subfilename}) +if [ $s2 -ne $s3 ]; then + err "Timestamp not changed" "$s2 != $s3" +fi + +sleep 1 + +new "Delete leaf" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx2x2newnone" "" "" + +new "Check candidate subfile changed" +s4=$(stat -c "%Y" $dir/candidate.d/${subfilename}) +if [ $s4 -eq $s3 ]; then + err "Timestamp changed" "$s4 = $s3" +fi + +sleep 1 + +new "Delete node" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx2none" "" "" + +new "Check candidate subfile changed" +s4=$(stat -c "%Y" $dir/candidate.d/${subfilename}) +if [ $s4 -eq $s3 ]; then + err "Timestamp changed" "$s4 = $s3" +fi + +new "Reset secondary adds" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx1fooreplace" "" "" + +new "netconf commit 2" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" "" + +new "Check candidate after commit" +check_db candidate ${subfilename} + +new "Check running after commit" +check_db running ${subfilename} + +new "cli show config" +expectpart "$($clixon_cli -1 -f $cfg show config xml -- -m clixon-mount1 -M urn:example:mount1)" 0 "xx1foo" + +s0=$(stat -c "%Y" $dir/running.d/${subfilename}) +new "Change mount data" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx1foo" "" "" + +sleep 1 + +new "netconf commit 3" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" "" + +new "Check running subfile changed" +s1=$(stat -c "%Y" $dir/running.d/${subfilename}) +if [ $s0 -eq $s1 ]; then + err "Timestamp changed" "$s0 = $s1" +fi + +new "Add data to top-level (not mount)" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "y" "" "" + +new "netconf commit 4" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" "" + +sleep 1 + +new "Check running subfile not changed" +s2=$(stat -c "%Y" $dir/running.d/${subfilename}) +if [ $s1 -ne $s2 ]; then + err "Timestamp not changed" "$s1 != $s2" +fi + +new "Reset secondary adds" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "xx1fooreplace" "" "" + +new "netconf commit 5" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" "" + +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg +fi + +new "Check running before restart" +check_db running ${subfilename} + +echo "-s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1" + +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1" + start_backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1 +fi + +new "Check running after restart" +check_db running ${subfilename} + +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg +fi + +sudo rm -rf $dir + +unset dbname +unset filename + +new "endtest" +endtest diff --git a/test/test_yang_schema_mount.sh b/test/test_yang_schema_mount.sh index 7689c4a7..712d15b4 100755 --- a/test/test_yang_schema_mount.sh +++ b/test/test_yang_schema_mount.sh @@ -254,7 +254,7 @@ if [ $BE -ne 0 ]; then stop_backend -f $cfg fi -rm -rf $dir +sudo rm -rf $dir new "endtest" endtest diff --git a/yang/clixon/clixon-config@2024-04-01.yang b/yang/clixon/clixon-config@2024-04-01.yang index 44f5ade8..c958433c 100644 --- a/yang/clixon/clixon-config@2024-04-01.yang +++ b/yang/clixon/clixon-config@2024-04-01.yang @@ -52,9 +52,10 @@ module clixon-config { revision 2024-04-01 { description "Added options: - CLICON_NETCONF_DUPLICATE_ALLOW - Disable duplicate check in NETCONF messages. - CLICON_CLI_OUTPUT_FORMAT - Default CLI output format - CLICON_AUTOLOCK - Implicit locks + CLICON_XMLDB_MULTI: Split datastore into multiple sub files + CLICON_NETCONF_DUPLICATE_ALLOW: Disable duplicate check in NETCONF messages. + CLICON_CLI_OUTPUT_FORMAT: Default CLI output format + CLICON_AUTOLOCK: Implicit locks Released in Clixon 7.1"; } revision 2024-01-01 { @@ -993,6 +994,7 @@ module clixon-config { description "Directory where \"running\", \"candidate\" and \"startup\" are placed."; } + leaf CLICON_DATASTORE_CACHE { type datastore_cache; default cache; @@ -1035,6 +1037,17 @@ module clixon-config { Will fail startup if old yang not found or if old config does not match. If not set, no yang check of old config is made until it is upgraded to new yang."; } + leaf CLICON_XMLDB_MULTI { + type boolean; + default false; + description + "Split configure datastore into multiple sub files + Keep track of which part of the XML tree are dirty, + Do not sync sub files if cache is not dirty. + Uses .d/ directory structure with digest.xml/.json + Splits are marked in YANG made on mountpoints only (may be generalized) + See CLICON_YANG_SCHEMA_MOUNT"; + } leaf CLICON_XML_CHANGELOG { type boolean; default false; diff --git a/yang/clixon/clixon-lib@2024-04-01.yang b/yang/clixon/clixon-lib@2024-04-01.yang index ac5d977a..ce934c60 100644 --- a/yang/clixon/clixon-lib@2024-04-01.yang +++ b/yang/clixon/clixon-lib@2024-04-01.yang @@ -66,11 +66,13 @@ module clixon-lib { - source-host (see RFC6022) - objectcreate - objectexisted + - link # For split multiple XML files "; revision 2024-04-01 { description - "Added: Default format + "Added: xmldb-split extension + Added: Default format Released in Clixon 7.1"; } revision 2024-01-01 { @@ -212,7 +214,17 @@ module clixon-lib { "The object should be ignored when comparing device configs for equality. The object should never be added, modified, or deleted on target. Essentially a read-only object - One example is auto-created objects by the , such as uid."; + One example is auto-created objects by the controller, such as uid."; + } + extension xmldb-split { + description + "When split configuration stores are used, ie CLICON_XMLDB_MULTI is set, + This extension marks where in the configuration tree, one file terminates + and a new sub-file is written. + A designer adds the 'xmldb-split' extension to a YANG node which should be split. + For example, a split could be made at mountpoints. + See also the 'link 'attribute. + "; } md:annotation creator { type string; diff --git a/yang/clixon/clixon-restconf@2022-08-01.yang b/yang/clixon/clixon-restconf@2022-08-01.yang index 909bd594..9930c75b 100644 --- a/yang/clixon/clixon-restconf@2022-08-01.yang +++ b/yang/clixon/clixon-restconf@2022-08-01.yang @@ -293,7 +293,7 @@ module clixon-restconf { for an interval longer than this number of seconds. If set to zero, then the server will never drop a session because it is idle."; - } + } } } }