experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Split config into multiple sub-files on mount-point boundaries and dont write clean subfiles

Browse source 
Added CLICON_XMLDB_MULTI option, added cl:xmldb-split extension
This commit is contained in:
Olof hagsand 2024-04-16 12:19:15 +02:00
parent bd290e4594
commit f511cb0030
30 changed files with 1311 additions and 285 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View file

@ -4,7 +4,7 @@ on:
push:
branches:
- master
- test-actions
- datastore-split
pull_request:
branches: [ master ]

15
CHANGELOG.md
View file

@ -15,17 +15,26 @@ Expected: June 2024
### Features
* New: Split configure datastore multiple sub-files on mount-point boundaries
* Avoid writung sub-files without new data (dirty cache)
* Added: Code for SHA digests.
* New: [Autolock](https://github.com/clicon/clixon/issues/508)
* CLI configurable format: [Default format should be configurable](https://github.com/clicon/clixon-controller/issues/87)
* CLI support for multiple inline commands separated by semi-colon
* New `clixon-config@2024-04-01.yang` revision
* Added options:
- `CLICON_NETCONF_DUPLICATE_ALLOW` - Disable duplicate check in NETCONF messages
- `CLICON_CLI_OUTPUT_FORMAT` - Default CLI output format
- `CLICON_AUTOLOCK` - Implicit locks
- `CLICON_XMLDB_MULTIPLE`: Split datastore into multiple sub files
- `CLICON_NETCONF_DUPLICATE_ALLOW`: Disable duplicate check in NETCONF messages
- `CLICON_CLI_OUTPUT_FORMAT`: Default CLI output format
- `CLICON_AUTOLOCK`: Implicit locks
* New `clixon-lib@2024-04-01.yang` revision
- Added: Default format
### API changes on existing protocol/config features
Users may have to change how they access the system
* Openssl mandatory for all configs, not only restconf
### Corrected Bugs
* Fixed: Fail on return errors when reading from datastore

8
apps/backend/backend_commit.c
View file

@ -251,7 +251,10 @@ startup_common(clixon_handle h,
/* clear XML tree of defaults */
if (xml_tree_prune_flagged(xt, XML_FLAG_DEFAULT, 1) < 0)
goto done;
if (xmldb_dump(h, stdout, xt, WITHDEFAULTS_REPORT_ALL) < 0)
if (xmldb_dump(h, stdout, xt, FORMAT_XML,
clicon_option_bool(h, "CLICON_XMLDB_PRETTY"),
WITHDEFAULTS_REPORT_ALL, 0, NULL) < 0)
goto done;
if (xt)
xml_free(xt);
@ -295,8 +298,7 @@ startup_common(clixon_handle h,
xt = NULL;
x = NULL;
while ((x = xml_child_each(td->td_target, x, CX_ELMNT)) != NULL){
xml_flag_set(x, XML_FLAG_ADD); /* Also down */
xml_apply(x, CX_ELMNT, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_ADD);
xml_apply0(x, CX_ELMNT, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_ADD);
if (cxvec_append(x, &td->td_avec, &td->td_alen) < 0)
goto done;
}

182
configure vendored
View file

@ -6278,96 +6278,6 @@ fi
printf "%s\n" "#define WITH_RESTCONF_FCGI 1" >>confdefs.h
# For c-code that cant use strings
elif test "x${with_restconf}" = xnative; then
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for OPENSSL_init_ssl in -lssl" >&5
printf %s "checking for OPENSSL_init_ssl in -lssl... " >&6; }
if test ${ac_cv_lib_ssl_OPENSSL_init_ssl_+y}
then :
printf %s "(cached) " >&6
else $as_nop
ac_check_lib_save_LIBS=$LIBS
LIBS="-lssl $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
char OPENSSL_init_ssl ();
int
main (void)
{
return OPENSSL_init_ssl ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"
then :
ac_cv_lib_ssl_OPENSSL_init_ssl_=yes
else $as_nop
ac_cv_lib_ssl_OPENSSL_init_ssl_=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_OPENSSL_init_ssl_" >&5
printf "%s\n" "$ac_cv_lib_ssl_OPENSSL_init_ssl_" >&6; }
if test "x$ac_cv_lib_ssl_OPENSSL_init_ssl_" = xyes
then :
printf "%s\n" "#define HAVE_LIBSSL 1" >>confdefs.h
LIBS="-lssl $LIBS"
else $as_nop
as_fn_error $? "libssl missing" "$LINENO" 5
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CRYPTO_new_ex_data in -lcrypto" >&5
printf %s "checking for CRYPTO_new_ex_data in -lcrypto... " >&6; }
if test ${ac_cv_lib_crypto_CRYPTO_new_ex_data+y}
then :
printf %s "(cached) " >&6
else $as_nop
ac_check_lib_save_LIBS=$LIBS
LIBS="-lcrypto $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
char CRYPTO_new_ex_data ();
int
main (void)
{
return CRYPTO_new_ex_data ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"
then :
ac_cv_lib_crypto_CRYPTO_new_ex_data=yes
else $as_nop
ac_cv_lib_crypto_CRYPTO_new_ex_data=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_crypto_CRYPTO_new_ex_data" >&5
printf "%s\n" "$ac_cv_lib_crypto_CRYPTO_new_ex_data" >&6; }
if test "x$ac_cv_lib_crypto_CRYPTO_new_ex_data" = xyes
then :
printf "%s\n" "#define HAVE_LIBCRYPTO 1" >>confdefs.h
LIBS="-lcrypto $LIBS"
else $as_nop
as_fn_error $? "libcrypto missing" "$LINENO" 5
fi
# Check if http/1 enabled
# Check whether --enable-http1 was given.
if test ${enable_http1+y}
@ -6755,6 +6665,98 @@ then :
fi
# This is for digest / restconf
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for OPENSSL_init_ssl in -lssl" >&5
printf %s "checking for OPENSSL_init_ssl in -lssl... " >&6; }
if test ${ac_cv_lib_ssl_OPENSSL_init_ssl_+y}
then :
printf %s "(cached) " >&6
else $as_nop
ac_check_lib_save_LIBS=$LIBS
LIBS="-lssl $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
char OPENSSL_init_ssl ();
int
main (void)
{
return OPENSSL_init_ssl ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"
then :
ac_cv_lib_ssl_OPENSSL_init_ssl_=yes
else $as_nop
ac_cv_lib_ssl_OPENSSL_init_ssl_=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ssl_OPENSSL_init_ssl_" >&5
printf "%s\n" "$ac_cv_lib_ssl_OPENSSL_init_ssl_" >&6; }
if test "x$ac_cv_lib_ssl_OPENSSL_init_ssl_" = xyes
then :
printf "%s\n" "#define HAVE_LIBSSL 1" >>confdefs.h
LIBS="-lssl $LIBS"
else $as_nop
as_fn_error $? "libssl missing" "$LINENO" 5
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for CRYPTO_new_ex_data in -lcrypto" >&5
printf %s "checking for CRYPTO_new_ex_data in -lcrypto... " >&6; }
if test ${ac_cv_lib_crypto_CRYPTO_new_ex_data+y}
then :
printf %s "(cached) " >&6
else $as_nop
ac_check_lib_save_LIBS=$LIBS
LIBS="-lcrypto $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
char CRYPTO_new_ex_data ();
int
main (void)
{
return CRYPTO_new_ex_data ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"
then :
ac_cv_lib_crypto_CRYPTO_new_ex_data=yes
else $as_nop
ac_cv_lib_crypto_CRYPTO_new_ex_data=no
fi
rm -f core conftest.err conftest.$ac_objext conftest.beam \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_crypto_CRYPTO_new_ex_data" >&5
printf "%s\n" "$ac_cv_lib_crypto_CRYPTO_new_ex_data" >&6; }
if test "x$ac_cv_lib_crypto_CRYPTO_new_ex_data" = xyes
then :
printf "%s\n" "#define HAVE_LIBCRYPTO 1" >>confdefs.h
LIBS="-lcrypto $LIBS"
else $as_nop
as_fn_error $? "libcrypto missing" "$LINENO" 5
fi
# This is for libxml2 XSD regex engine
# Note this only enables the compiling of the code. In order to actually
# use it you need to set Clixon config option CLICON_YANG_REGEXP to libxml2

6
configure.ac
View file

@ -275,8 +275,6 @@ if test "x${with_restconf}" = xfcgi; then
AC_CHECK_LIB(fcgi, FCGX_Init,, AC_MSG_ERROR([libfcgi-dev missing]))
AC_DEFINE(WITH_RESTCONF_FCGI, 1, [Use fcgi restconf mode]) # For c-code that cant use strings
elif test "x${with_restconf}" = xnative; then
AC_CHECK_LIB(ssl, OPENSSL_init_ssl ,, AC_MSG_ERROR([libssl missing]))
AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, , AC_MSG_ERROR([libcrypto missing]))
# Check if http/1 enabled
AC_ARG_ENABLE(http1, AS_HELP_STRING([--disable-http1],[Disable http1 for native restconf http/1, ie http/2 only]),[
if test "$enableval" = no; then
@ -373,6 +371,10 @@ AC_DEFINE_UNQUOTED(CLIXON_CONFIG_SYSCONFDIR, "${SYSCONFDIR}", [Pass-through $sys
AC_CHECK_LIB(socket, socket)
AC_CHECK_LIB(dl, dlopen)
# This is for digest / restconf
AC_CHECK_LIB(ssl, OPENSSL_init_ssl ,, AC_MSG_ERROR([libssl missing]))
AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, , AC_MSG_ERROR([libcrypto missing]))
# This is for libxml2 XSD regex engine
# Note this only enables the compiling of the code. In order to actually
# use it you need to set Clixon config option CLICON_YANG_REGEXP to libxml2

5
include/clixon_custom.h
View file

@ -206,3 +206,8 @@
*/
#define COMPAT_6_5
/*! Use SHA256 (32 bytes) instead of SHA1 (20 bytes)
*
* Digest use is not cryptographic use, so SHA1 is enough for now
*/
#undef USE_SHA256

1
lib/clixon/clixon.h.in
View file

@ -70,6 +70,7 @@ extern "C" {
#include <clixon/clixon_uid.h>
#include <clixon/clixon_queue.h>
#include <clixon/clixon_hash.h>
#include <clixon/clixon_digest.h>
#include <clixon/clixon_handle.h>
#include <clixon/clixon_yang.h>
#include <clixon/clixon_xml.h>

13
lib/clixon/clixon_datastore.h
View file

@ -55,7 +55,7 @@ struct db_elmnt {
* reset by commit, discard
*/
int de_empty; /* Empty on read from file, xmldb_readfile and xmldb_put sets it */
int de_volatile; /* Do not sync to disk on every update (ie xmldb_put) */
int de_volatile; /* Disable auto-sync of cache to disk on every update (ie xmldb_put) */
};
typedef struct db_elmnt db_elmnt;
@ -66,16 +66,21 @@ typedef struct db_elmnt db_elmnt;
db_elmnt *clicon_db_elmnt_get(clixon_handle h, const char *db);
int clicon_db_elmnt_set(clixon_handle h, const char *db, db_elmnt *xc);
int xmldb_db2file(clixon_handle h, const char *db, char **filename);
int xmldb_db2subdir(clixon_handle h, const char *db, char **dir);
/* API */
int xmldb_connect(clixon_handle h);
int xmldb_disconnect(clixon_handle h);
/* in clixon_datastore_read.[ch] */
/* in clixon_datastore_read.[ch]: */
int xmldb_get(clixon_handle h, const char *db, cvec *nsc, char *xpath, cxobj **xret);
int xmldb_get0(clixon_handle h, const char *db, yang_bind yb,
cvec *nsc, const char *xpath, int copy, withdefaults_type wdef,
cxobj **xret, modstate_diff_t *msd, cxobj **xerr);
int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret); /* in clixon_datastore_write.[ch] */
/* in clixon_datastore_write.[ch]: */
int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret);
int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, enum format_enum format, int pretty, withdefaults_type wdef, int multi, const char *multidb);
int xmldb_write_cache2file(clixon_handle h, const char *db);
int xmldb_copy(clixon_handle h, const char *from, const char *to);
int xmldb_lock(clixon_handle h, const char *db, uint32_t id);
int xmldb_unlock(clixon_handle h, const char *db);
@ -98,7 +103,5 @@ int xmldb_volatile_set(clixon_handle h, const char *db, int value);
int xmldb_print(clixon_handle h, FILE *f);
int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix);
int xmldb_populate(clixon_handle h, const char *db);
int xmldb_write_cache2file(clixon_handle h, const char *db);
int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, withdefaults_type wdef);
#endif /* _CLIXON_DATASTORE_H */

43
lib/clixon/clixon_digest.h Normal file
View file

@ -0,0 +1,43 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
Copyright (C) 2017-2019 Olof Hagsand
Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC (Netgate)
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*/
#ifndef _CLIXON_DIGEST_H_
#define _CLIXON_DIGEST_H_
int clixon_digest_hex(const char *string, char **hexstrp);
#endif /* _CLIXON_DIGEST_H_ */

1
lib/clixon/clixon_file.h
View file

@ -43,6 +43,7 @@ int clicon_file_dirent(const char *dir, struct dirent **ent,
const char *regexp, mode_t type);
int clicon_files_recursive(const char *dir, const char *regexp, cvec *cvv);
int clicon_file_copy(char *src, char *target);
int clicon_dir_copy(char *src, char *target);
int clicon_file_cbuf(const char *filename, cbuf *cb);
#endif /* _CLIXON_FILE_H_ */

1
lib/clixon/clixon_xml.h
View file

@ -207,6 +207,7 @@ enum format_enum{
#define XML_FLAG_TOP 0x80 /* Top datastore symbol */
#define XML_FLAG_BODYKEY 0x100 /* Text parsing key to be translated from body to key */
#define XML_FLAG_ANYDATA 0x200 /* Treat as anydata, eg mount-points before bound */
#define XML_FLAG_CACHE_DIRTY 0x400 /* This part of XML tree is not synced to disk */
/*
* Prototypes

6
lib/clixon/clixon_xml_io.h
View file

@ -44,8 +44,12 @@
* Prototypes
*/
int clixon_xml2file1(FILE *f, cxobj *xn, int level, int pretty, char *prefix,
clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef);
clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef,
int multi);
int clixon_xml2file(FILE *f, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext);
int clixon_xml2file_multi(clixon_handle h, const char *db, cxobj *xn, int level, int pretty,
char *prefix, clicon_output_cb *fn, int skiptop, int autocliext,
withdefaults_type wdef);
int xml_print(FILE *f, cxobj *xn);
int xml_dump(FILE *f, cxobj *x);
int clixon_xml2cbuf1(cbuf *cb, cxobj *x, int level, int prettyprint, char *prefix,

2
lib/src/Makefile.in
View file

@ -87,7 +87,7 @@ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_debug.c clixon_err.c cli
clixon_yang_cardinality.c clixon_yang_schema_mount.c \
clixon_xml_changelog.c clixon_xml_nsctx.c \
clixon_path.c clixon_validate.c clixon_validate_minmax.c \
clixon_hash.c clixon_options.c clixon_data.c clixon_plugin.c \
clixon_hash.c clixon_digest.c clixon_options.c clixon_data.c clixon_plugin.c \
clixon_proto.c clixon_proto_client.c \
clixon_xpath.c clixon_xpath_ctx.c clixon_xpath_eval.c clixon_xpath_function.c \
clixon_xpath_optimize.c clixon_xpath_yang.c \

253
lib/src/clixon_datastore.c
View file

@ -127,7 +127,7 @@ clicon_db_elmnt_set(clixon_handle h,
/*! Translate from symbolic database name to actual filename in file-system
*
* @param[in] th text handle handle
* @param[in] h Clixon handle
* @param[in] db Symbolic database name, eg "candidate", "running"
* @param[out] filename Filename. Unallocate after use with free()
* @retval 0 OK
@ -151,7 +151,7 @@ xmldb_db2file(clixon_handle h,
goto done;
}
if ((dir = clicon_xmldb_dir(h)) == NULL){
clixon_err(OE_XML, errno, "dbdir not set");
clixon_err(OE_XML, errno, "CLICON_XMLDB_DIR not set");
goto done;
}
cprintf(cb, "%s/%s_db", dir, db);
@ -166,6 +166,50 @@ xmldb_db2file(clixon_handle h,
return retval;
}
/*! Translate from symbolic database name to sub-directory of configure sub-files, no checks
*
* @param[in] h Clixon handle
* @param[in] db Symbolic database name, eg "candidate", "running"
* @param[out] subdirp Sub-directory name. Unallocate after use with free()
* @retval 0 OK
* @retval -1 Error
* The dir is subdir to CLICON_XMLDB_DIR option
* @see xmldb_db2file For top-level config file
*/
int
xmldb_db2subdir(clixon_handle h,
const char *db,
char **subdirp)
{
int retval = -1;
cbuf *cb = NULL;
char *dir;
char *subdir = NULL;
if ((cb = cbuf_new()) == NULL){
clixon_err(OE_XML, errno, "cbuf_new");
goto done;
}
if ((dir = clicon_xmldb_dir(h)) == NULL){
clixon_err(OE_XML, errno, "CLICON_XMLDB_DIR not set");
goto done;
}
cprintf(cb, "%s/%s.d", dir, db);
if ((subdir = strdup4(cbuf_get(cb))) == NULL){
clixon_err(OE_UNIX, errno, "strdup");
goto done;
}
*subdirp = subdir;
subdir = NULL;
retval = 0;
done:
if (subdir)
free(subdir);
if (cb)
cbuf_free(cb);
return retval;
}
/*! Connect to a datastore plugin, allocate resources to be used in API calls
*
* @param[in] h Clixon handle
@ -209,11 +253,12 @@ xmldb_disconnect(clixon_handle h)
return retval;
}
/*! Copy database from db1 to db2
/*! Copy datastore from db1 to db2
*
* May include copying datastore directory structure
* @param[in] h Clixon handle
* @param[in] from Source database
* @param[in] to Destination database
* @param[in] from Source datastore
* @param[in] to Destination datastore
* @retval 0 OK
* @retval -1 Error
*/
@ -267,8 +312,22 @@ xmldb_copy(clixon_handle h,
if (de2)
de0 = *de2;
de0.de_xml = x2; /* The new tree */
clicon_db_elmnt_set(h, to, &de0);
if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){
char *subdir = NULL;
struct stat st = {0,};
if (xmldb_db2subdir(h, to, &subdir) < 0)
goto done;
if (stat(subdir, &st) < 0){
if (mkdir(subdir, S_IRWXU|S_IRGRP|S_IWGRP|S_IROTH|S_IXOTH) < 0){
clixon_err(OE_UNIX, errno, "mkdir(%s)", subdir);
goto done;
}
}
if (subdir)
free(subdir);
}
clicon_db_elmnt_set(h, to, &de0);
/* Copy the files themselves (above only in-memory cache) */
if (xmldb_db2file(h, from, &fromfile) < 0)
goto done;
@ -276,6 +335,21 @@ xmldb_copy(clixon_handle h,
goto done;
if (clicon_file_copy(fromfile, tofile) < 0)
goto done;
if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")) {
char *fromdir = NULL;
char *todir = NULL;
if (xmldb_db2subdir(h, from, &fromdir) < 0)
goto done;
if (xmldb_db2subdir(h, to, &todir) < 0)
goto done;
if (clicon_dir_copy(fromdir, todir) < 0)
goto done;
if (fromdir)
free(fromdir);
if (todir)
free(todir);
}
retval = 0;
done:
clixon_debug(CLIXON_DBG_DATASTORE, "retval:%d", retval);
@ -467,7 +541,7 @@ xmldb_clear(clixon_handle h,
return 0;
}
/*! Delete database, clear cache if any. Remove file
/*! Delete database, clear cache if any. Remove file and dir
*
* @param[in] h Clixon handle
* @param[in] db Database
@ -482,21 +556,59 @@ xmldb_delete(clixon_handle h,
{
int retval = -1;
char *filename = NULL;
struct stat sb;
struct stat st = {0,};
cbuf *cb = NULL;
char *subdir = NULL;
struct dirent *dp = NULL;
int ndp;
int i;
char *regexp = NULL;
clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "%s", db);
if (xmldb_clear(h, db) < 0)
goto done;
if (xmldb_db2file(h, db, &filename) < 0)
goto done;
if (lstat(filename, &sb) == 0)
if (lstat(filename, &st) == 0)
if (truncate(filename, 0) < 0){
clixon_err(OE_DB, errno, "truncate %s", filename);
goto done;
}
if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){
if (xmldb_db2subdir(h, db, &subdir) < 0)
goto done;
if (stat(subdir, &st) == 0){
if ((ndp = clicon_file_dirent(subdir, &dp, regexp, S_IFREG)) < 0)
goto done;
if ((cb = cbuf_new()) == NULL){
clixon_err(OE_XML, errno, "cbuf_new");
goto done;
}
for (i = 0; i < ndp; i++){
cbuf_reset(cb);
cprintf(cb, "%s/%s", subdir, dp[i].d_name);
if (unlink(cbuf_get(cb)) < 0){
clixon_err(OE_DB, errno, "unlink(%s)", cbuf_get(cb));
goto done;
}
}
if (rmdir(subdir) < 0){
#if 0 /* Ignore this for now, there are some cornercases where this is problamatic, see confirmed-commit */
clixon_err(OE_DB, errno, "rmdir(%s)", subdir);
goto done;
#endif
}
}
}
retval = 0;
done:
clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval);
if (dp)
free(dp);
if (cb)
cbuf_free(cb);
if (subdir)
free(subdir);
if (filename)
free(filename);
return retval;
@ -518,6 +630,8 @@ xmldb_create(clixon_handle h,
int fd = -1;
db_elmnt *de = NULL;
cxobj *xt = NULL;
char *subdir = NULL;
struct stat st = {0,};
clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "%s", db);
if ((de = clicon_db_elmnt_get(h, db)) != NULL){
@ -532,9 +646,21 @@ xmldb_create(clixon_handle h,
clixon_err(OE_UNIX, errno, "open(%s)", filename);
goto done;
}
if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){
if (xmldb_db2subdir(h, db, &subdir) < 0)
goto done;
if (stat(subdir, &st) < 0){
if (mkdir(subdir, S_IRWXU|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH) < 0){
clixon_err(OE_UNIX, errno, "mkdir(%s)", subdir);
goto done;
}
}
}
retval = 0;
done:
clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval);
if (subdir)
free(subdir);
if (filename)
free(filename);
if (fd != -1)
@ -671,9 +797,9 @@ xmldb_empty_set(clixon_handle h,
return 0;
}
/*! Get volatile flag of datastore
/*! Get volatile flag of datastore cache
*
* Whether to sync to disk on every update (ie xmldb_put)
* Whether to sync cache to disk on every update (ie xmldb_put)
* @param[in] h Clixon handle
* @param[in] db Database name
* @retval 1 Db was empty on load
@ -693,9 +819,9 @@ xmldb_volatile_get(clixon_handle h,
return de->de_volatile;
}
/*! Set datastore status of datastore
/*! Set datastore status of datastore cache
*
* Whether to sync to disk on every update (ie xmldb_put)
* Whether to sync cache to disk on every update (ie xmldb_put)
* @param[in] h Clixon handle
* @param[in] db Database name
* @param[in] value 0 or 1
@ -830,104 +956,3 @@ xmldb_populate(clixon_handle h,
done:
return retval;
}
/* Dump a datastore to file, add modstate
*
* @param[in] h Clixon handle
* @param[in] db Name of database to search in (filename including dir path
* @param[in] xt Top of XML tree
* @param[in] wdef With-defaults parameter
* @retval 0 OK
* @retval -1 Error
*/
int
xmldb_dump(clixon_handle h,
FILE *f,
cxobj *xt,
withdefaults_type wdef)
{
int retval = -1;
cxobj *xm;
cxobj *xmodst = NULL;
char *formatstr;
enum format_enum format = FORMAT_XML;
int pretty;
/* Add modstate first */
if ((xm = clicon_modst_cache_get(h, 1)) != NULL){
if ((xmodst = xml_dup(xm)) == NULL)
goto done;
if (xml_child_insert_pos(xt, xmodst, 0) < 0)
goto done;
xml_parent_set(xmodst, xt);
}
pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY");
if ((formatstr = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) != NULL){
if ((format = format_str2int(formatstr)) < 0){
clixon_err(OE_XML, 0, "Format %s invalid", formatstr);
goto done;
}
}
switch (format){
case FORMAT_JSON:
if (clixon_json2file(f, xt, pretty, fprintf, 0, 0) < 0)
goto done;
break;
case FORMAT_XML:
if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef) < 0)
goto done;
break;
default:
clixon_err(OE_XML, 0, "Format %s not supported", format_int2str(format));
goto done;
break;
}
/* Remove modules state after writing to file */
if (xmodst && xml_purge(xmodst) < 0)
goto done;
retval = 0;
done:
return retval;
}
/*! Given a datastore, write the cache to file
*
* Also add mod-state if applicable
* @param[in] h Clixon handle
* @param[in] db Name of database to search in (filename including dir path
* @retval 0 OK
* @retval -1 Error
*/
int
xmldb_write_cache2file(clixon_handle h,
const char *db)
{
int retval = -1;
cxobj *xt;
FILE *f = NULL;
char *dbfile = NULL;
if (xmldb_db2file(h, db, &dbfile) < 0)
goto done;
if (dbfile==NULL){
clixon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
if ((xt = xmldb_cache_get(h, db)) == NULL){
clixon_err(OE_XML, 0, "XML cache not found");
goto done;
}
if ((f = fopen(dbfile, "w")) == NULL){
clixon_err(OE_CFG, errno, "Creating file %s", dbfile);
goto done;
}
if (xmldb_dump(h, f, xt, WITHDEFAULTS_EXPLICIT) < 0)
goto done;
retval = 0;
done:
if (dbfile)
free(dbfile);
if (f)
fclose(f);
return retval;
}

109
lib/src/clixon_datastore_read.c
View file

@ -86,6 +86,17 @@
#define handle(xh) (assert(text_handle_check(xh)==0),(struct text_handle *)(xh))
/* Local types */
/* Argument to apply for recursive call to xmldb_multi calls
* @see xmldb_multi_write_arg
*/
struct xmldb_multi_read_arg {
char *mr_subdir;
yang_stmt *mr_yspec;
enum format_enum mr_format;
cxobj **mr_xerr;
};
/*! Ensure that xt only has a single sub-element and that is "config"
*
* @retval 0 There exists a single "config" sub-element
@ -446,6 +457,68 @@ disable_nacm_on_empty(cxobj *xt,
return retval;
}
/*! Callback function for xmldb-multi read
*
* Look for link attribute in XML, and if found open the linked file for parsing
* @param[in] x XML node
* @param[in] arg
* @retval 2 Locally abort this subtree, continue with others
* @retval 1 Abort, dont continue with others, return 1 to end user
* @retval 0 OK, continue
* @retval -1 Error, aborted at first error encounter, return -1 to end user
*/
static int
xmldb_multi_read_applyfn(cxobj *x,
void *arg)
{
struct xmldb_multi_read_arg *mr = (struct xmldb_multi_read_arg *) arg;
int retval = -1;
cxobj *xa;
char *filename;
cbuf *cb = NULL;
char *dbfile;
FILE *fp = NULL;
if ((xa = xml_find_type(x, CLIXON_LIB_PREFIX, "link", CX_ATTR)) != NULL &&
(filename = xml_value(xa)) != NULL){
if ((cb = cbuf_new()) == NULL){
clixon_err(OE_XML, errno, "cbuf_new");
goto done;
}
cprintf(cb, "%s/%s", mr->mr_subdir, filename);
xml_purge(xa);
if ((xa = xml_find_type(x, "xmlns", CLIXON_LIB_PREFIX, CX_ATTR)) != NULL)
xml_purge(xa);
dbfile = cbuf_get(cb);
clixon_debug(CLIXON_DBG_DATASTORE, "Parsing: %s", dbfile);
if ((fp = fopen(dbfile, "r")) == NULL){
clixon_err(OE_CFG, errno, "fdopen(%s)", dbfile);
goto done;
}
switch (mr->mr_format){
case FORMAT_JSON:
if (clixon_json_parse_file(fp, 1, YB_NONE, mr->mr_yspec, &x, mr->mr_xerr) < 0)
goto done;
break;
case FORMAT_XML:
if (clixon_xml_parse_file(fp, YB_NONE, mr->mr_yspec, &x, mr->mr_xerr) < 0)
goto done;
break;
default:
clixon_err(OE_DB, 0, "Format not supported");
goto done;
break;
}
}
retval = 0;
done:
if (cb)
cbuf_free(cb);
if (fp)
fclose(fp);
return retval;
}
/*! Common read function that reads an XML tree from file
*
* @param[in] th Datastore text handle
@ -477,7 +550,8 @@ xmldb_readfile(clixon_handle h,
cxobj *x0 = NULL;
char *dbfile = NULL;
FILE *fp = NULL;
char *format;
char *formatstr;
enum format_enum format;
int ret;
modstate_diff_t *msdiff = NULL;
cxobj *xmsd; /* XML module state diff */
@ -489,6 +563,7 @@ xmldb_readfile(clixon_handle h,
cxobj *xmodfile = NULL;
cxobj *x;
yang_stmt *yspec1 = NULL;
struct xmldb_multi_read_arg mr = {0, };
if (yb != YB_MODULE && yb != YB_NONE){
clixon_err(OE_XML, EINVAL, "yb is %d but should be module or none", yb);
@ -500,11 +575,15 @@ xmldb_readfile(clixon_handle h,
clixon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
if ((formatstr = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
clixon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
}
clixon_debug(CLIXON_DBG_DATASTORE, "Reading datastore %s using %s", dbfile, format);
if ((format = format_str2int(formatstr)) < 0){
clixon_err(OE_XML, 0, "format not found %s", formatstr);
goto done;
}
clixon_debug(CLIXON_DBG_DATASTORE, "Reading datastore %s using %s", dbfile, formatstr);
/* Parse file into internal XML tree from different formats */
if ((fp = fopen(dbfile, "r")) == NULL) {
clixon_err(OE_UNIX, errno, "open(%s)", dbfile);
@ -516,14 +595,28 @@ xmldb_readfile(clixon_handle h,
* config*
* </config>
* ret == 0 should not happen with YB_NONE. Binding is done later */
if (strcmp(format, "json")==0){
switch (format){
case FORMAT_JSON:
if (clixon_json_parse_file(fp, 1, YB_NONE, yspec, &x0, xerr) < 0)
goto done;
}
else {
if (clixon_xml_parse_file(fp, YB_NONE, yspec, &x0, xerr) < 0){
break;
case FORMAT_XML:
if (clixon_xml_parse_file(fp, YB_NONE, yspec, &x0, xerr) < 0)
goto done;
break;
default:
clixon_err(OE_DB, 0, "Format %s not supported", formatstr);
goto done;
break;
}
if (clicon_option_bool(h, "CLICON_XMLDB_MULTI")){
if (xmldb_db2subdir(h, db, &mr.mr_subdir) < 0)
goto done;
mr.mr_format = format;
mr.mr_yspec = yspec;
mr.mr_xerr = xerr;
if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xmldb_multi_read_applyfn, &mr) < 0)
goto done;
}
/* Always assert a top-level called "config".
* To ensure that, deal with two cases:
@ -646,6 +739,8 @@ xmldb_readfile(clixon_handle h,
}
retval = 1;
done:
if (mr.mr_subdir)
free(mr.mr_subdir);
if (yspec1)
ys_free1(yspec1, 1);
if (xmodfile)

259
lib/src/clixon_datastore_write.c
View file

@ -45,11 +45,12 @@
#include <string.h>
#include <limits.h>
#include <stdint.h>
#include <sys/types.h>
#include <unistd.h>
#include <dirent.h>
#include <syslog.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
/* cligen */
#include <cligen/cligen.h>
@ -58,6 +59,7 @@
#include "clixon_string.h"
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_digest.h"
#include "clixon_handle.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
@ -85,6 +87,18 @@
#include "clixon_datastore_write.h"
#include "clixon_datastore_read.h"
/* Local types */
/* Argument to apply for recursive call to xmldb_multi write calls
* @see xmldb_multi_read_arg
*/
struct xmldb_multi_write_arg {
clixon_handle *mw_h;
const char *mw_db;
int mw_pretty;
withdefaults_type mw_wdef;
enum format_enum mw_format;
};
/*! Given an attribute name and its expected namespace, find its value
*
* An attribute may have a prefix(or NULL). The routine finds the associated
@ -691,6 +705,7 @@ text_modify(clixon_handle h,
}
if (xml_value_set(x0b, x1bstr) < 0)
goto done;
xml_flag_set(x0, XML_FLAG_ADD);
/* If a default value ies replaced, then reset default flag */
if (xml_flag(x0, XML_FLAG_DEFAULT))
xml_flag_reset(x0, XML_FLAG_DEFAULT);
@ -1167,7 +1182,7 @@ text_modify_top(clixon_handle h,
goto done;
} /* text_modify_top */
/*! Callback function type for xml_apply
/*! Mark ancestor if any changes to children. Also mark changed xml as cache dirty
*
* @param[in] x XML node
* @param[in] arg General-purpose argument
@ -1185,12 +1200,27 @@ xml_mark_added_ancestors(cxobj *x,
int flags = (intptr_t)arg;
if (xml_flag(x, flags)){
xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)(XML_FLAG_CHANGE));
return 2;
}
return 0;
}
static int
xml_mark_cache_dirty(cxobj *x,
void *arg)
{
if (xml_flag(x, XML_FLAG_CHANGE)){
xml_flag_set(x, XML_FLAG_CACHE_DIRTY);
return 0;
}
else if (xml_flag(x, XML_FLAG_ADD|XML_FLAG_DEL)){
if (xml_apply0(x, CX_ELMNT, (xml_applyfn_t*)xml_flag_set, (void*)(XML_FLAG_CACHE_DIRTY)) < 0)
return -1;
}
return 2;
}
/*! Modify database given an xml tree and an operation
*
* @param[in] h CLICON handle
@ -1291,8 +1321,12 @@ xmldb_put(clixon_handle h,
/* Remove NONE nodes if all subs recursively are also NONE */
if (xml_tree_prune_flagged_sub(x0, XML_FLAG_NONE, 0, NULL) <0)
goto done;
/* Mark ancestor if any changes to children. */
if (xml_apply(x0, CX_ELMNT, xml_mark_added_ancestors, (void*)(XML_FLAG_ADD|XML_FLAG_DEL)) < 0)
goto done;
/* Mark changed xml as cache dirty */
if (xml_apply(x0, CX_ELMNT, xml_mark_cache_dirty, NULL) < 0)
goto done;
/* Remove empty non-presence containers recursively.
*/
if (xml_default_nopresence(x0, 3, XML_FLAG_ADD|XML_FLAG_DEL) < 0)
@ -1304,10 +1338,6 @@ xmldb_put(clixon_handle h,
/* Add default recursive values */
if (xml_default_recurse(x0, 0, XML_FLAG_ADD|XML_FLAG_DEL) < 0)
goto done;
/* Clear flags from previous steps */
if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
(void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE)) < 0)
goto done;
/* Write back to datastore cache if first time */
if (de != NULL)
de0 = *de;
@ -1315,10 +1345,21 @@ xmldb_put(clixon_handle h,
de0.de_xml = x0;
de0.de_empty = (xml_child_nr(de0.de_xml) == 0);
clicon_db_elmnt_set(h, db, &de0);
/* Write cache to file unless volatile */
if (xmldb_volatile_get(h, db) == 0)
/* Write cache to file unless volatile (ie stop syncing to store) */
if (xmldb_volatile_get(h, db) == 0){
if (xmldb_write_cache2file(h, db) < 0)
goto done;
/* Clear flags from previous steps + dirty */
if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
(void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE|XML_FLAG_CACHE_DIRTY)) < 0)
goto done;
}
else {
/* Clear flags from previous steps */
if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
(void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE)) < 0)
goto done;
}
retval = 1;
done:
clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval);
@ -1331,3 +1372,203 @@ xmldb_put(clixon_handle h,
retval = 0;
goto done;
}
/*! Callback function for xmldb-multi write
*
* Look for link attribute in XML, and if found open the linked file for parsing
* @param[in] x XML node
* @param[in] arg
* @retval 2 Locally abort this subtree, continue with others
* @retval 1 Abort, dont continue with others, return 1 to end user
* @retval 0 OK, continue
* @retval -1 Error, aborted at first error encounter, return -1 to end user
*/
static int
xmldb_multi_write_applyfn(cxobj *x,
void *arg)
{
struct xmldb_multi_write_arg *mw = (struct xmldb_multi_write_arg *) arg;
int retval = -1;
clixon_handle h = mw->mw_h;
yang_stmt *y;
int exist = 0;
char *xpath = NULL;
char *hexstr = NULL;
cbuf *cb = NULL;
char *subdir = NULL;
char *dbfile;
struct stat st = {0,};
int fd = -1;
FILE *fsub = NULL;
if (xml_child_nr_type(x, CX_ELMNT) > 0 &&
(y = xml_spec(x)) != NULL){
if (yang_extension_value(y, "xmldb-split", CLIXON_LIB_NS, &exist, NULL) < 0)
goto done;
if (exist){
if (xml2xpath(x, NULL, 1, 0, &xpath) < 0)
goto done;
if (clixon_digest_hex(xpath, &hexstr) < 0)
goto done;
if (xmldb_db2subdir(h, mw->mw_db, &subdir) < 0)
goto done;
if ((cb = cbuf_new()) == NULL){
clixon_err(OE_XML, errno, "cbuf_new");
goto done;
}
cprintf(cb, "%s/%s.xml", subdir, hexstr);
dbfile = cbuf_get(cb);
if (xml_flag(x, XML_FLAG_CACHE_DIRTY) ||
lstat(dbfile, &st) < 0){
clixon_debug(CLIXON_DBG_DATASTORE, "Open: %s for writing", dbfile);
if ((fd = open(dbfile, O_CREAT|O_WRONLY|O_TRUNC, S_IRWXU)) < 0) {
clixon_err(OE_UNIX, errno, "open(%s)", dbfile);
goto done;
}
if ((fsub = fdopen(fd, "w")) == NULL){
clixon_err(OE_CFG, errno, "fdopen(%s)", dbfile);
goto done;
}
/* Dont recurse multi-file yet */
if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0) < 0)
goto done;
}
}
}
retval = 0;
done:
if (fsub != NULL)
fclose(fsub);
if (cb)
cbuf_free(cb);
if (subdir)
free(subdir);
if (xpath)
free(xpath);
if (hexstr)
free(hexstr);
return retval;
}
/* Given open file, xml-tree, and wdef, add modstate, get format and write to file
*
* @param[in] h Clixon handle
* @param[in] f Output file
* @param[in] xt Top of XML tree
* @param[in] format Output format
* @param[in] pretty Pretty-print
* @param[in] wdef With-defaults parameter
* @param[in] multi If split into multiple files
* @param[in] multidb Database name (only if multi)
* @retval 0 OK
* @retval -1 Error
*/
int
xmldb_dump(clixon_handle h,
FILE *f,
cxobj *xt,
enum format_enum format,
int pretty,
withdefaults_type wdef,
int multi,
const char *multidb)
{
int retval = -1;
struct xmldb_multi_write_arg mw = {0,};
cxobj *xm;
cxobj *xmodst = NULL;
/* Add modstate */
if ((xm = clicon_modst_cache_get(h, 1)) != NULL){
if ((xmodst = xml_dup(xm)) == NULL)
goto done;
if (xml_child_insert_pos(xt, xmodst, 0) < 0)
goto done;
xml_parent_set(xmodst, xt);
}
switch (format){
case FORMAT_XML:
if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi) < 0)
goto done;
if (multi){
mw.mw_h = h;
mw.mw_db = multidb;
mw.mw_pretty = pretty;
mw.mw_wdef = wdef;
mw.mw_format = format;
if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xmldb_multi_write_applyfn, &mw) < 0)
goto done;
}
break;
case FORMAT_JSON:
if (multi){
clixon_err(OE_CFG, errno, "JSON+multi not supported");
goto done;
}
if (clixon_json2file(f, xt, pretty, fprintf, 0, 0) < 0)
goto done;
break;
default:
clixon_err(OE_XML, 0, "Format %s not supported", format_int2str(format));
goto done;
break;
}
/* Remove modules state after writing to file */
if (xmodst && xml_purge(xmodst) < 0)
goto done;
retval = 0;
done:
return retval;
}
/*! Given datastore, get cache and format, set wdef, add modstate and print to multiple files
*
* Also add mod-state if applicable
* @param[in] h Clixon handle
* @param[in] db Name of database to search in (filename including dir path
* @retval 0 OK
* @retval -1 Error
*/
int
xmldb_write_cache2file(clixon_handle h,
const char *db)
{
int retval = -1;
cxobj *xt;
char *formatstr;
enum format_enum format = FORMAT_XML;
withdefaults_type wdef = WITHDEFAULTS_EXPLICIT;
int pretty;
int multi;
FILE *f = NULL;
char *dbfile = NULL;
if ((xt = xmldb_cache_get(h, db)) == NULL){
clixon_err(OE_XML, 0, "XML cache not found");
goto done;
}
pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY");
multi = clicon_option_bool(h, "CLICON_XMLDB_MULTI");
if ((formatstr = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) != NULL){
if ((format = format_str2int(formatstr)) < 0){
clixon_err(OE_XML, 0, "Format %s invalid", formatstr);
goto done;
}
}
if (xmldb_db2file(h, db, &dbfile) < 0)
goto done;
if ((f = fopen(dbfile, "w")) == NULL){
clixon_err(OE_CFG, errno, "fopen(%s)", dbfile);
goto done;
}
if (xmldb_dump(h, f, xt, format, pretty, wdef, multi, db) < 0)
goto done;
retval = 0;
done:
if (dbfile)
free(dbfile);
if (f)
fclose(f);
return retval;
}

2
lib/src/clixon_datastore_write.h
View file

@ -46,5 +46,7 @@
* Prototypes
*/
int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret);
int xmldb_write_cache2file(clixon_handle h, const char *db);
int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, enum format_enum format, int pretty, withdefaults_type wdef, int multi, const char *multidb);
#endif /* _CLIXON_DATASTORE_WRITE_H */

112
lib/src/clixon_digest.c Normal file
View file

@ -0,0 +1,112 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2024 Olof Hagsand
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*/
#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <openssl/sha.h>
/* cligen */
#include <cligen/cligen.h>
/* clixon */
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_handle.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_err.h"
#include "clixon_digest.h"
/*! Given null-terminated string, return malloced NULL_terinated SHA digest string in hex
*
* @param[in] str Input null-terminated string
* @param[out] hexstrp Output null-terminated digest HEX string
* @retval 0 OK
* @retval -1 Error
*/
int
clixon_digest_hex(const char *str,
char **hexstrp)
{
int retval = -1;
unsigned char *md = NULL;
cbuf *cb = NULL;
int i;
uint32_t len;
#ifdef USE_SHA256
len = SHA256_DIGEST_LENGTH;
#else
len = SHA_DIGEST_LENGTH;
#endif
if (str == NULL || hexstrp == NULL){
clixon_err(OE_UNIX, EINVAL, "str or hexstr is NULL");
return -1;
}
if ((md = calloc(len, sizeof(unsigned char))) == NULL){
clixon_err(OE_UNIX, errno, "calloc");
goto done;
}
#ifdef USE_SHA256
if (SHA256((unsigned char *)str, strlen(str), md) == NULL)
#else
if (SHA1((unsigned char *)str, strlen(str), md) == NULL)
#endif
{
clixon_err(OE_UNIX, 0, "SHA256 error");
goto done;
}
if ((cb = cbuf_new()) == NULL){
clixon_err(OE_XML, errno, "cbuf_new");
goto done;
}
for (i = 0; i < len; i++)
cprintf(cb, "%02x", md[i]&0xff);
if ((*hexstrp = strdup(cbuf_get(cb))) == NULL){
clixon_err(OE_UNIX, errno, "strdup");
goto done;
}
retval = 0;
done:
if (cb)
cbuf_free(cb);
if (md)
free(md);
return retval;
}

42
lib/src/clixon_file.c
View file

@ -176,6 +176,7 @@ clicon_files_recursive(const char *dir,
* @code
* char *dir = "/root/fs";
* struct dirent *dp;
* int ndp;
* if ((ndp = clicon_file_dirent(dir, &dp, "\\.so$", S_IFREG)) < 0)
* return -1;
* for (i = 0; i < ndp; i++)
@ -221,6 +222,8 @@ clicon_file_dirent(const char *dir,
goto quit;
}
while((dent = readdir(dirp)) != NULL) {
if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)
continue;
/* Filename matching */
if (regexp) {
if (regexec(&re, dent->d_name, (size_t) 0, NULL, 0) != 0)
@ -250,7 +253,6 @@ clicon_file_dirent(const char *dir,
memcpy(&new[nent], dent, direntStructSize);
nent++;
} /* while */
qsort((void *)new, nent, sizeof(*new), clicon_file_dirent_sort);
*ent = new;
new = NULL;
@ -271,6 +273,7 @@ quit:
* @param[out] target Destination filename
* @retval 0 OK
* @retval -1 Error
* XXX This is probably faster with stdio?
*/
int
clicon_file_copy(char *src,
@ -312,6 +315,43 @@ clicon_file_copy(char *src,
return retval;
}
/*! Make a copy of directory non-recursive
*
* @param[in] srcdir Source dirname
* @param[out] dstdir Destination dirname
* @retval 0 OK
* @retval -1 Error
*/
int
clicon_dir_copy(char *srcdir,
char *dstdir)
{
int retval = -1;
struct dirent *dent = NULL;
DIR *dirp = NULL;
char srcfile[MAXPATHLEN];
char dstfile[MAXPATHLEN];
if (srcdir == NULL || dstdir == NULL){
clixon_err(OE_UNIX, EINVAL, "Requires src and dst dir != NULL");
goto done;
}
if ((dirp = opendir(srcdir)) != NULL)
while ((dent = readdir(dirp)) != NULL) {
if (dent->d_type != DT_REG)
continue;
snprintf(srcfile, MAXPATHLEN-1, "%s/%s", srcdir, dent->d_name);
snprintf(dstfile, MAXPATHLEN-1, "%s/%s", dstdir, dent->d_name);
if (clicon_file_copy(srcfile, dstfile) < 0)
goto done;
}
retval = 0;
done:
if (dirp)
closedir(dirp);
return retval;
}
/*! Read content of file into cbuf
*
* @param[in] filename

1
lib/src/clixon_hash.c
View file

@ -186,6 +186,7 @@ clicon_hash_lookup(clicon_hash_t *hash,
}
/*! Get value of hash
*
* @param[in] hash Hash table
* @param[in] key Variable name
* @param[out] vlen Length of value (as returned by function if != NULL)

2
lib/src/clixon_path.c
View file

@ -1896,8 +1896,10 @@ clixon_xml_find_instance_id(cxobj *xt,
va_end(ap);
if (instance_id_parse(path, &cplist) < 0)
goto done;
#if 0
if (clixon_debug_get())
clixon_path_print(stderr, cplist);
#endif
/* Resolve module:name to pointer to yang-stmt, fail if not successful */
if ((ret = instance_id_resolve(cplist, yt)) < 0)
goto done;

2
lib/src/clixon_stream.c
View file

@ -282,7 +282,7 @@ stream_timer_setup(int fd,
struct stream_replay *r;
struct stream_replay *r1;
clixon_debug(CLIXON_DBG_STREAM, "");
clixon_debug(CLIXON_DBG_STREAM|CLIXON_DBG_DETAIL, "");
/* Go thru callbacks and see if any have timed out, if so remove them
* Could also be done by a separate timer.
*/

5
lib/src/clixon_xml.c
View file

@ -81,7 +81,7 @@
* Constants
*/
/* How many XML children to start with if any. Then add quadratic until threshold when
* add lineraly
* add linearly
* Heurestics: if child is body only single child is expected, but element children may
* have siblings
*/
@ -1234,7 +1234,6 @@ xml_new_body(char *name,
return new_node;
}
/*! Return yang spec of node.
*
* Not necessarily set. Either has not been set yet (by xml_spec_set( or anyxml.
@ -2025,7 +2024,7 @@ xml_copy_one(cxobj *x0,
default:
break;
}
xml_flag_set(x1, xml_flag(x0, XML_FLAG_DEFAULT | XML_FLAG_TOP | XML_FLAG_ANYDATA)); /* Maybe more flags */
xml_flag_set(x1, xml_flag(x0, XML_FLAG_DEFAULT | XML_FLAG_TOP | XML_FLAG_ANYDATA | XML_FLAG_CACHE_DIRTY)); /* Maybe more flags */
retval = 0;
done:
return retval;

75
lib/src/clixon_xml_io.c
View file

@ -52,6 +52,9 @@
#include <errno.h>
#include <string.h>
#include <limits.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
/* cligen */
#include <cligen/cligen.h>
@ -60,6 +63,7 @@
#include "clixon_string.h"
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_digest.h"
#include "clixon_handle.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
@ -68,6 +72,7 @@
#include "clixon_debug.h"
#include "clixon_options.h"
#include "clixon_yang_module.h"
#include "clixon_yang_schema_mount.h"
#include "clixon_xml_bind.h"
#include "clixon_xml_vec.h"
#include "clixon_xml_sort.h"
@ -75,6 +80,9 @@
#include "clixon_xml_parse.h"
#include "clixon_netconf_lib.h"
#include "clixon_xml_default.h"
#include "clixon_xpath_ctx.h"
#include "clixon_xpath.h"
#include "clixon_datastore.h"
#include "clixon_xml_io.h"
/*
@ -193,6 +201,7 @@ xml2output_wdef(cxobj *x,
* @param[in] fn Callback to make print function
* @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow
* @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL
* @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI
* @retval 0 OK
* @retval -1 Error
* One can use clixon_xml2cbuf to get common code, but using fprintf is
@ -212,7 +221,8 @@ xml2file_recurse(FILE *f,
char *prefix,
clicon_output_cb *fn,
int autocliext,
withdefaults_type wdef)
withdefaults_type wdef,
int multi)
{
int retval = -1;
char *name;
@ -227,6 +237,9 @@ xml2file_recurse(FILE *f,
int level1;
int tag = 0;
int ret;
int subfile = 0; /* File is split into subfile */
char *xpath = NULL;
char *hexstr = NULL;
if (x == NULL)
goto ok;
@ -277,7 +290,7 @@ xml2file_recurse(FILE *f,
while ((xc = xml_child_each(x, xc, -1)) != NULL) {
switch (xml_type(xc)){
case CX_ATTR:
if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef) <0)
if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi) < 0)
goto done;
break;
case CX_BODY:
@ -296,10 +309,27 @@ xml2file_recurse(FILE *f,
if (hasbody==0 && haselement==0)
(*fn)(f, "/>");
else{
/* Check if this is a multi-file split-point */
if (multi && (y = xml_spec(x)) != NULL){
if (yang_extension_value(y, "xmldb-split", CLIXON_LIB_NS, &exist, NULL) < 0)
goto done;
if (exist){
subfile++;
if (xml2xpath(x, NULL, 1, 0, &xpath) < 0)
goto done;
if (clixon_digest_hex(xpath, &hexstr) < 0)
goto done;
(*fn)(f, " xmlns:%s=\"%s\"", CLIXON_LIB_PREFIX, CLIXON_LIB_NS);
(*fn)(f, " %s:link=\"%s.xml\"", CLIXON_LIB_PREFIX, hexstr);
(*fn)(f, "/>");
}
}
if (!subfile) {
(*fn)(f, ">");
if (pretty && hasbody == 0){
(*fn)(f, "\n");
}
}
xc = NULL;
while ((xc = xml_child_each(x, xc, -1)) != NULL) {
cxobj *xa = NULL;
@ -316,14 +346,16 @@ xml2file_recurse(FILE *f,
xa = xml_find_type(xc, IETF_NETCONF_WITH_DEFAULTS_ATTR_PREFIX, IETF_NETCONF_WITH_DEFAULTS_ATTR_NAMESPACE, CX_ATTR);
}
}
if (xml_type(xc) != CX_ATTR)
if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef) <0)
if (xml_type(xc) != CX_ATTR && !subfile)
if (xml2file_recurse(f, xc, level+1, pretty, prefix,
fn, autocliext, wdef, multi) <0)
goto done;
if (xa){
if (xml_purge(xa) < 0)
goto done;
}
}
if (subfile == 0){
if (pretty && hasbody==0){
if (pretty && prefix)
(*fn)(f, "%s", prefix);
@ -334,6 +366,7 @@ xml2file_recurse(FILE *f,
(*fn)(f, "%s:", namespace);
(*fn)(f, "%s>", name);
}
}
if (pretty){
(*fn)(f, "\n");
}
@ -346,6 +379,10 @@ xml2file_recurse(FILE *f,
done:
if (encstr)
free(encstr);
if (xpath)
free(xpath);
if (hexstr)
free(hexstr);
return retval;
}
@ -362,6 +399,7 @@ xml2file_recurse(FILE *f,
* @param[in] skiptop 0: Include top object 1: Skip top-object, only children,
* @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow
* @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL
* @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI
* @retval 0 OK
* @retval -1 Error
* @see clixon_xml2cbuf print to a cbuf string
@ -377,7 +415,8 @@ clixon_xml2file1(FILE *f,
clicon_output_cb *fn,
int skiptop,
int autocliext,
withdefaults_type wdef)
withdefaults_type wdef,
int multi)
{
int retval = 1;
cxobj *xc;
@ -387,11 +426,11 @@ clixon_xml2file1(FILE *f,
if (skiptop){
xc = NULL;
while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL)
if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef) < 0)
if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi) < 0)
goto done;
}
else {
if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef) < 0)
if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi) < 0)
goto done;
}
retval = 0;
@ -427,7 +466,7 @@ clixon_xml2file(FILE *f,
int skiptop,
int autocliext)
{
return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0);
return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0);
}
/*! Print an XML tree structure to an output stream
@ -442,7 +481,7 @@ int
xml_print(FILE *f,
cxobj *x)
{
return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL);
return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0);
}
/*! Dump cxobj structure with pointers and flags for debugging, internal function
@ -456,20 +495,20 @@ xml_dump1(FILE *f,
if (xml_type(x) != CX_ELMNT)
return 0;
fprintf(stderr, "%*s %s(%s)",
fprintf(f, "%*s %s(%s)",
indent*PRETTYPRINT_INDENT, "",
// x,
xml_name(x),
xml_type2str(xml_type(x)));
if (xml_flag(x, XML_FLAG_ADD))
fprintf(stderr, " add");
fprintf(f, " add");
if (xml_flag(x, XML_FLAG_DEL))
fprintf(stderr, " delete");
fprintf(f, " delete");
if (xml_flag(x, XML_FLAG_CHANGE))
fprintf(stderr, " change");
fprintf(f, " change");
if (xml_flag(x, XML_FLAG_MARK))
fprintf(stderr, " mark");
fprintf(stderr, "\n");
fprintf(f, " mark");
fprintf(f, "\n");
xc = NULL;
while ((xc = xml_child_each(x, xc, -1)) != NULL) {
xml_dump1(f, xc, indent+1);
@ -949,11 +988,13 @@ clixon_xml_parse_file(FILE *fp,
int xmlbuflen = BUFLEN; /* start size */
int oldxmlbuflen;
int failed = 0;
int xtempty; /* empty on entry */
if (xt==NULL || fp == NULL){
if (xt == NULL || fp == NULL){
clixon_err(OE_XML, EINVAL, "arg is NULL");
return -1;
}
xtempty = (*xt == NULL);
if (yb == YB_MODULE && yspec == NULL){
clixon_err(OE_XML, EINVAL, "yspec is required if yb == YB_MODULE");
return -1;
@ -995,7 +1036,7 @@ clixon_xml_parse_file(FILE *fp,
} /* while */
retval = (failed==0) ? 1 : 0;
done:
if (retval < 0 && *xt){
if (retval < 0 && *xt && xtempty){
free(*xt);
*xt = NULL;
}

10
lib/src/clixon_yang_schema_mount.c
View file

@ -138,7 +138,11 @@ yang_schema_mount_point0(yang_stmt *y)
goto done;
}
/*! Cached variant of yang_schema_mount_point
/*! Cached variant of yang_schema_mount_point0
*
* @param[in] y Yang node
* @retval 1 Yes, node is potential mountpoint
* @retval 0 No, node is not potential mountpoint
*/
int
yang_schema_mount_point(yang_stmt *y)
@ -250,9 +254,7 @@ xml_yang_mount_get(clixon_handle h,
if ((y = xml_spec(xt)) == NULL)
goto fail;
if ((ret = yang_schema_mount_point(y)) < 0)
goto done;
if (ret == 0)
if (yang_schema_mount_point(y) == 0)
goto fail;
/* Check validate level */
if (vl && clixon_plugin_yang_mount_all(h, xt, NULL, vl, NULL) < 0)

368
test/test_datastore_multi.sh Executable file
View file

@ -0,0 +1,368 @@
#!/usr/bin/env bash
# Datastore split test, eg x_db has x.d/ directory with subdirs# ALso test cache bevahour, that unmodified
# subteres are not touched
# For now subdirs only enabled for mointpoints, so this test is with mountpoints as well
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
APPNAME=example
# include err() and new() functions and creates $dir
cfg=$dir/conf_yang.xml
clispec=$dir/automode.cli
fyang=$dir/clixon-example.yang
fyang1=$dir/clixon-mount1.yang
CFD=$dir/conf.d
test -d $CFD || mkdir -p $CFD
AUTOCLI=$(autocli_config clixon-\* kw-nokey false)
# Well-known digest of mount-point xpath
subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_CONFIGDIR>$CFD</CLICON_CONFIGDIR>
<CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
<CLICON_YANG_DIR>${dir}</CLICON_YANG_DIR>
<CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
<CLICON_YANG_LIBRARY>true</CLICON_YANG_LIBRARY>
<CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
<CLICON_SOCK>/usr/local/var/run/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
<CLICON_BACKEND_PIDFILE>/usr/local/var/run/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_MULTI>true</CLICON_XMLDB_MULTI>
<CLICON_NETCONF_MONITORING>true</CLICON_NETCONF_MONITORING>
<CLICON_VALIDATE_STATE_XML>true</CLICON_VALIDATE_STATE_XML>
<CLICON_STREAM_DISCOVERY_RFC5277>true</CLICON_STREAM_DISCOVERY_RFC5277>
<CLICON_YANG_SCHEMA_MOUNT>true</CLICON_YANG_SCHEMA_MOUNT>
</clixon-config>
EOF
cat <<EOF > $CFD/autocli.xml
<clixon-config xmlns="http://clicon.org/config">
<autocli>
<module-default>false</module-default>
<list-keyword-default>kw-nokey</list-keyword-default>
<grouping-treeref>true</grouping-treeref>
<rule>
<name>include clixon</name>
<operation>enable</operation>
<module-name>clixon-*</module-name>
</rule>
</autocli>
</clixon-config>
EOF
cat <<EOF > $fyang
module clixon-example{
yang-version 1.1;
namespace "urn:example:clixon";
prefix ex;
import ietf-yang-schema-mount {
prefix yangmnt;
}
import clixon-lib {
prefix cl;
}
container top{
list mylist{
key name;
leaf name{
type string;
}
container root{
presence "Otherwise root is not visible";
yangmnt:mount-point "mylabel"{
description "Root for other yang models";
}
cl:xmldb-split; /* Multi-XMLDB: split datastore here */
}
}
}
}
EOF
cat <<EOF > $fyang1
module clixon-mount1{
yang-version 1.1;
namespace "urn:example:mount1";
prefix m1;
container mount1{
list mylist1{
key name1;
leaf name1{
type string;
}
leaf value1 {
type string;
}
}
}
container extra{
leaf extraval{
type string;
}
}
}
EOF
cat <<EOF > $clispec
CLICON_MODE="example";
CLICON_PROMPT="%U@%H %W> ";
CLICON_PLUGIN="example_cli";
# Autocli syntax tree operations
set @datamodel, cli_auto_set();
merge @datamodel, cli_auto_merge();
create @datamodel, cli_auto_create();
delete("Delete a configuration item") @datamodel, cli_auto_del();
validate("Validate changes"), cli_validate();
commit("Commit the changes"), cli_commit();
quit("Quit"), cli_quit();
show("Show a particular state of the system"){
configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{
xml("Show configuration as XML"), cli_show_auto_mode("candidate", "xml", false, false);
cli("Show configuration as CLI commands"), cli_show_auto_mode("candidate", "cli", false, false, "report-all", "set ");
netconf("Show configuration as netconf edit-config operation"), cli_show_auto_mode("candidate", "netconf", false, false);
text("Show configuration as text"), cli_show_auto_mode("candidate", "text", false, false);
json("Show configuration as JSON"), cli_show_auto_mode("candidate", "json", false, false);
}
state("Show configuration and state"), cli_show_auto_mode("running", "xml", false, true);
compare("Compare candidate and running databases"), compare_dbs("running", "candidate", "xml");
}
EOF
# Check content of db
# Args:
# 0: dbname
# 1: subfile
function check_db()
{
dbname=$1
subfile=$2
sudo chmod o+r $dir/${dbname}_db
sudo chmod o+r $dir/${dbname}.d/$subfile
sudo rm -f $dir/x_db
cat <<EOF > $dir/x_db
<config>
<top xmlns="urn:example:clixon">
<mylist>
<name>x</name>
<root xmlns:cl="http://clicon.org/lib" cl:link="$subfile"/>
</mylist>
</top>
</config>
EOF
new "Check ${dbname}_db"
ret=$(diff $dir/x_db $dir/${dbname}_db)
if [ $? -ne 0 ]; then
err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
fi
cat <<EOF > $dir/x_subfile
<mount1 xmlns="urn:example:mount1">
<mylist1>
<name1>x1</name1>
</mylist1>
</mount1>
<extra xmlns="urn:example:mount1">
<extraval>foo</extraval>
</extra>
EOF
new "Check ${dbname}.d/$subfile"
ret=$(diff $dir/x_subfile $dir/${dbname}.d/$subfile)
if [ $? -ne 0 ]; then
err "$(cat $dir/x_subfile)" "$(cat $dir/${dbname}.d/$subfile)"
fi
}
new "test params: -f $cfg"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s init -f $cfg -- -m clixon-mount1 -M urn:example:mount1"
start_backend -s init -f $cfg -- -m clixon-mount1 -M urn:example:mount1
fi
new "wait backend"
wait_backend
new "Add mountpoint x "
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root/></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "netconf commit"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Add data to mount x"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x1</name1></mylist1></mount1><extra xmlns=\"urn:example:mount1\"><extraval>foo</extraval></extra></root></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate after edit"
check_db candidate ${subfilename}
s0=$(stat -c "%Y" $dir/candidate.d/${subfilename})
sleep 1
new "Add 2nd data to mount x"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x2</name1><value1>x2value</value1></mylist1></mount1></root></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate subfile changed"
s1=$(stat -c "%Y" $dir/candidate.d/${subfilename})
if [ $s0 -eq $s1 ]; then
err "Timestamp changed" "$s0 = $s1"
fi
sleep 1
new "Change existing value in mount x"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x2</name1><value1>x2new</value1></mylist1></mount1></root></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate subfile changed"
s2=$(stat -c "%Y" $dir/candidate.d/${subfilename})
if [ $s1 -eq $s2 ]; then
err "Timestamp changed" "$s1 = $s2"
fi
sleep 1
new "Add data to top-level (not mount)"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>y</name></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate subfile not changed"
s3=$(stat -c "%Y" $dir/candidate.d/${subfilename})
if [ $s2 -ne $s3 ]; then
err "Timestamp not changed" "$s2 != $s3"
fi
sleep 1
new "Delete leaf"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\" xmlns:nc=\"${BASENS}\"><mylist1><name1>x2</name1><value1 nc:operation=\"delete\">x2new</value1></mylist1></mount1></root></mylist></top></config><default-operation>none</default-operation></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate subfile changed"
s4=$(stat -c "%Y" $dir/candidate.d/${subfilename})
if [ $s4 -eq $s3 ]; then
err "Timestamp changed" "$s4 = $s3"
fi
sleep 1
new "Delete node"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\" xmlns:nc=\"${BASENS}\"><mylist1 nc:operation=\"delete\"><name1>x2</name1></mylist1></mount1></root></mylist></top></config><default-operation>none</default-operation></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate subfile changed"
s4=$(stat -c "%Y" $dir/candidate.d/${subfilename})
if [ $s4 -eq $s3 ]; then
err "Timestamp changed" "$s4 = $s3"
fi
new "Reset secondary adds"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x1</name1></mylist1></mount1><extra xmlns=\"urn:example:mount1\"><extraval>foo</extraval></extra></root></mylist></top></config><default-operation>replace</default-operation></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "netconf commit 2"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check candidate after commit"
check_db candidate ${subfilename}
new "Check running after commit"
check_db running ${subfilename}
new "cli show config"
expectpart "$($clixon_cli -1 -f $cfg show config xml -- -m clixon-mount1 -M urn:example:mount1)" 0 "<top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x1</name1></mylist1></mount1><extra xmlns=\"urn:example:mount1\"><extraval>foo</extraval></extra></root></mylist></top>"
s0=$(stat -c "%Y" $dir/running.d/${subfilename})
new "Change mount data"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x1</name1><value1>foo</value1></mylist1></mount1></root></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
sleep 1
new "netconf commit 3"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check running subfile changed"
s1=$(stat -c "%Y" $dir/running.d/${subfilename})
if [ $s0 -eq $s1 ]; then
err "Timestamp changed" "$s0 = $s1"
fi
new "Add data to top-level (not mount)"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>y</name></mylist></top></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "netconf commit 4"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
sleep 1
new "Check running subfile not changed"
s2=$(stat -c "%Y" $dir/running.d/${subfilename})
if [ $s1 -ne $s2 ]; then
err "Timestamp not changed" "$s1 != $s2"
fi
new "Reset secondary adds"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><top xmlns=\"urn:example:clixon\"><mylist><name>x</name><root><mount1 xmlns=\"urn:example:mount1\"><mylist1><name1>x1</name1></mylist1></mount1><extra xmlns=\"urn:example:mount1\"><extraval>foo</extraval></extra></root></mylist></top></config><default-operation>replace</default-operation></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "netconf commit 5"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
new "Check running before restart"
check_db running ${subfilename}
echo "-s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1"
start_backend -s running -f $cfg -- -m clixon-mount1 -M urn:example:mount1
fi
new "Check running after restart"
check_db running ${subfilename}
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
sudo rm -rf $dir
unset dbname
unset filename
new "endtest"
endtest

2
test/test_yang_schema_mount.sh
View file

@ -254,7 +254,7 @@ if [ $BE -ne 0 ]; then
stop_backend -f $cfg
fi
rm -rf $dir
sudo rm -rf $dir
new "endtest"
endtest

19
yang/clixon/clixon-config@2024-04-01.yang
View file

@ -52,9 +52,10 @@ module clixon-config {
revision 2024-04-01 {
description
"Added options:
CLICON_NETCONF_DUPLICATE_ALLOW - Disable duplicate check in NETCONF messages.
CLICON_CLI_OUTPUT_FORMAT - Default CLI output format
CLICON_AUTOLOCK - Implicit locks
CLICON_XMLDB_MULTI: Split datastore into multiple sub files
CLICON_NETCONF_DUPLICATE_ALLOW: Disable duplicate check in NETCONF messages.
CLICON_CLI_OUTPUT_FORMAT: Default CLI output format
CLICON_AUTOLOCK: Implicit locks
Released in Clixon 7.1";
}
revision 2024-01-01 {
@ -993,6 +994,7 @@ module clixon-config {
description
"Directory where \"running\", \"candidate\" and \"startup\" are placed.";
}
leaf CLICON_DATASTORE_CACHE {
type datastore_cache;
default cache;
@ -1035,6 +1037,17 @@ module clixon-config {
Will fail startup if old yang not found or if old config does not match.
If not set, no yang check of old config is made until it is upgraded to new yang.";
}
leaf CLICON_XMLDB_MULTI {
type boolean;
default false;
description
"Split configure datastore into multiple sub files
Keep track of which part of the XML tree are dirty,
Do not sync sub files if cache is not dirty.
Uses .d/ directory structure with digest.xml/.json
Splits are marked in YANG made on mountpoints only (may be generalized)
See CLICON_YANG_SCHEMA_MOUNT";
}
leaf CLICON_XML_CHANGELOG {
type boolean;
default false;

16
yang/clixon/clixon-lib@2024-04-01.yang
View file

@ -66,11 +66,13 @@ module clixon-lib {
- source-host (see RFC6022)
- objectcreate
- objectexisted
- link # For split multiple XML files
";
revision 2024-04-01 {
description
"Added: Default format
"Added: xmldb-split extension
Added: Default format
Released in Clixon 7.1";
}
revision 2024-01-01 {
@ -212,7 +214,17 @@ module clixon-lib {
"The object should be ignored when comparing device configs for equality.
The object should never be added, modified, or deleted on target.
Essentially a read-only object
One example is auto-created objects by the , such as uid.";
One example is auto-created objects by the controller, such as uid.";
}
extension xmldb-split {
description
"When split configuration stores are used, ie CLICON_XMLDB_MULTI is set,
This extension marks where in the configuration tree, one file terminates
and a new sub-file is written.
A designer adds the 'xmldb-split' extension to a YANG node which should be split.
For example, a split could be made at mountpoints.
See also the 'link 'attribute.
";
}
md:annotation creator {
type string;