Optimization of PUT

This commit is contained in:
Olof hagsand 2024-02-05 17:14:36 +01:00
parent 4e3bd6fbdd
commit 23dcb58758
15 changed files with 221 additions and 131 deletions

View file

@ -248,7 +248,10 @@ startup_common(clixon_handle h,
else if (xml_sort_recurse(xt) < 0) {
clixon_err(OE_XML, EFAULT, "Yang sort error");
}
if (xmldb_dump(h, stdout, xt) < 0)
/* clear XML tree of defaults */
if (xml_tree_prune_flagged(xt, XML_FLAG_DEFAULT, 1) < 0)
goto done;
if (xmldb_dump(h, stdout, xt, WITHDEFAULTS_REPORT_ALL) < 0)
goto done;
exit(0); /* This is fairly abrupt , but need to avoid side-effects of rewinding
stack. Alternative is to make a separate function stack for this. */
@ -488,6 +491,12 @@ validate_common(clixon_handle h,
clixon_err(OE_FATAL, 0, "No DB_SPEC");
goto done;
}
if (xmldb_cache_get(h, db) != NULL){
if (xmldb_populate(h, db) < 0)
goto done;
if (xmldb_write_cache2file(h, db) < 0)
goto done;
}
/* This is the state we are going to */
if ((ret = xmldb_get0(h, db, YB_MODULE, NULL, "/", 0, 0, &td->td_target, NULL, xret)) < 0)
goto done;

View file

@ -390,6 +390,7 @@ filter_xpath_again(clixon_handle h,
* @param[in] nsc Namespace context of xpath
* @param[in] username User name for NACM access
* @param[in] depth Nr of levels to print, -1 is all, 0 is none
* @param[in] wdef With-defaults parameter
* @param[out] cbret Return xml tree, eg <rpc-reply>..., <rpc-error..
* @retval 0 OK
* @retval -1 Error

View file

@ -325,7 +325,6 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
* @param[in] yspec Yang spec
* @param[in] nsc Namespace context
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in] wdef With-defaults parameter, see RFC 6243
* @param[in,out] xret State XML tree is merged with existing tree.
* @retval 1 OK
* @retval 0 Statedata callback failed (xret set with netconf-error)

View file

@ -53,9 +53,12 @@
*/
typedef struct {
uint32_t de_id; /* If set, locked by this client/session id */
struct timeval de_tv; /* Timevalue */
struct timeval de_tv; /* Timevalue, set by lock/unlock */
cxobj *de_xml; /* cache */
int de_modified; /* Dirty since loaded/copied/committed/etc XXX:nocache? */
int de_modified; /* Dirty since loaded/copied/committed/etc Used by lock
* This set by NETCONF edit-config, copy, delete,
* reset by commit, discard
*/
int de_empty; /* Empty on read from file, xmldb_readfile and xmldb_put sets it */
} db_elmnt;

View file

@ -72,9 +72,10 @@ cxobj *xmldb_cache_get(clixon_handle h, const char *db);
int xmldb_modified_get(clixon_handle h, const char *db);
int xmldb_modified_set(clixon_handle h, const char *db, int value);
int xmldb_empty_get(clixon_handle h, const char *db);
int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt);
int xmldb_print(clixon_handle h, FILE *f);
int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix);
int xmldb_populate(clixon_handle h, const char *db);
int xmldb_write_cache2file(clixon_handle h, const char *db);
int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, withdefaults_type wdef);
#endif /* _CLIXON_DATASTORE_H */

View file

@ -201,7 +201,7 @@ enum format_enum{
#define XML_FLAG_ADD 0x04 /* Node is added (commits) or parent added rec*/
#define XML_FLAG_DEL 0x08 /* Node is deleted (commits) or parent deleted rec */
#define XML_FLAG_CHANGE 0x10 /* Node is changed (commits) or child changed rec */
#define XML_FLAG_NONE 0x20 /* Node is added as NONE */
#define XML_FLAG_NONE 0x20 /* Node is added as NETCONF edit-config operation=NONE */
#define XML_FLAG_DEFAULT 0x40 /* Added when a value is set as default @see xml_default */
#define XML_FLAG_TOP 0x80 /* Top datastore symbol */
#define XML_FLAG_BODYKEY 0x100 /* Text parsing key to be translated from body to key */
@ -248,7 +248,7 @@ cxobj *xml_child_i_set(cxobj *xt, int i, cxobj *xc);
int xml_child_order(cxobj *xn, cxobj *xc);
cxobj *xml_child_each(cxobj *xparent, cxobj *xprev, enum cxobj_type type);
cxobj *xml_child_each_attr(cxobj *xparent, cxobj *xprev);
int xml_child_insert_pos(cxobj *x, cxobj *xc, int i);
int xml_child_insert_pos(cxobj *x, cxobj *xc, int pos);
int xml_childvec_set(cxobj *x, int len);
cxobj **xml_childvec_get(cxobj *x);
int clixon_child_xvec_append(cxobj *x, clixon_xvec *xv);

View file

@ -50,6 +50,7 @@ typedef enum yang_class yang_class;
* Prototypes
*/
int xml_default_recurse(cxobj *xn, int state);
int xml_default_recurse_flag(cxobj *xn, int state, int flag);
int xml_global_defaults(clixon_handle h, cxobj *xn, cvec *nsc, const char *xpath, yang_stmt *yspec, int state);
int xml_defaults_nopresence(cxobj *xn, int purge);
int xml_add_default_tag(cxobj *x, uint16_t flags);

View file

@ -79,6 +79,8 @@
#include "clixon_netconf_lib.h"
#include "clixon_xml_bind.h"
#include "clixon_xml_default.h"
#include "clixon_xml_io.h"
#include "clixon_json.h"
#include "clixon_datastore.h"
#include "clixon_datastore_write.h"
#include "clixon_datastore_read.h"
@ -389,7 +391,6 @@ xmldb_exists(clixon_handle h,
goto done;
if (lstat(filename, &sb) < 0)
retval = 0;
else{
if (sb.st_size == 0)
retval = 0;
@ -719,3 +720,98 @@ xmldb_populate(clixon_handle h,
done:
return retval;
}
/* Dump a datastore to file, add modstate
*
* @param[in] h Clixon handle
* @param[in] db Name of database to search in (filename including dir path
* @param[in] xt Top of XML tree
* @param[in] wdef With-defaults parameter
* @retval 0 OK
* @retval -1 Error
*/
int
xmldb_dump(clixon_handle h,
FILE *f,
cxobj *xt,
withdefaults_type wdef)
{
int retval = -1;
cxobj *xm;
cxobj *xmodst = NULL;
char *format;
int pretty;
/* Add modstate first */
if ((xm = clicon_modst_cache_get(h, 1)) != NULL){
if ((xmodst = xml_dup(xm)) == NULL)
goto done;
if (xml_child_insert_pos(xt, xmodst, 0) < 0)
goto done;
xml_parent_set(xmodst, xt);
}
if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
clixon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
}
pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY");
if (strcmp(format,"json")==0){
if (clixon_json2file(f, xt, pretty, fprintf, 0, 0) < 0)
goto done;
}
else if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef) < 0)
goto done;
/* Remove modules state after writing to file */
if (xmodst && xml_purge(xmodst) < 0)
goto done;
retval = 0;
done:
return retval;
}
/*! Given a datastore, write the cache to file
*
* Also add mod-state if applicable
* @param[in] h Clixon handle
* @param[in] db Name of database to search in (filename including dir path
* @retval 0 OK
* @retval -1 Error
*/
int
xmldb_write_cache2file(clixon_handle h,
const char *db)
{
int retval = -1;
cxobj *xt;
FILE *f = NULL;
char *dbfile = NULL;
if (xmldb_db2file(h, db, &dbfile) < 0)
goto done;
if (dbfile==NULL){
clixon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
if ((xt = xmldb_cache_get(h, db)) == NULL){
clixon_err(OE_XML, 0, "XML cache not found");
goto done;
}
if ((f = fopen(dbfile, "w")) == NULL){
clixon_err(OE_CFG, errno, "Creating file %s", dbfile);
goto done;
}
if (xmldb_dump(h, f, xt, WITHDEFAULTS_EXPLICIT) < 0)
goto done;
if (f) {
fclose(f);
f = NULL;
}
retval = 0;
done:
if (dbfile)
free(dbfile);
if (f)
fclose(f);
return retval;
}

View file

@ -45,7 +45,6 @@
#include <limits.h>
#include <stdint.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <assert.h>

View file

@ -46,7 +46,6 @@
#include <limits.h>
#include <stdint.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <syslog.h>
@ -454,6 +453,7 @@ choice_delete_other(cxobj *x0,
* In an "ordered-by user" list, the attributes "insert" and "key" in
* the YANG XML namespace can be used to control where
* in the list the entry is inserted.
* Marks added objects with XML_FLAG_ADD
*/
static int
text_modify(clixon_handle h,
@ -617,7 +617,6 @@ text_modify(clixon_handle h,
if ((x0 = xml_new(x1name, NULL, CX_ELMNT)) == NULL)
goto done;
xml_spec_set(x0, y0);
/* Get namespace from x1
* Check if namespace exists in x0 parent
* if not add new binding and replace in x0.
@ -698,6 +697,7 @@ text_modify(clixon_handle h,
if (changed){
if (xml_insert(x0p, x0, insert, valstr, NULL) < 0)
goto done;
xml_flag_set(x0, XML_FLAG_ADD);
}
break;
case OP_DELETE:
@ -718,8 +718,10 @@ text_modify(clixon_handle h,
/* Purge if x1 value is NULL(match-all) or both values are equal */
if ((x1bstr == NULL) ||
((x0bstr=xml_body(x0)) != NULL && strcmp(x0bstr, x1bstr)==0)){
if (xml_purge(x0) < 0)
goto done;
xml_flag_set(x0p, XML_FLAG_DEL);
}
else {
if (op == OP_DELETE){
@ -947,7 +949,7 @@ text_modify(clixon_handle h,
#endif
if (xml_insert(x0p, x0, insert, keystr, nscx1) < 0)
goto done;
xml_flag_set(x0, XML_FLAG_ADD);
}
break;
case OP_DELETE:
@ -966,6 +968,7 @@ text_modify(clixon_handle h,
}
if (xml_purge(x0) < 0)
goto done;
xml_flag_set(x0p, XML_FLAG_DEL);
}
break;
default:
@ -1163,6 +1166,30 @@ text_modify_top(clixon_handle h,
goto done;
} /* text_modify_top */
/*! Callback function type for xml_apply
*
* @param[in] x XML node
* @param[in] arg General-purpose argument
* @retval 2 Locally abort this subtree, continue with others
* @retval 1 Abort, dont continue with others, return 1 to end user
* @retval 0 OK, continue
* @retval -1 Error, aborted at first error encounter, return -1 to end user
* @note WHEN node are not checked, but should be updated when doing validate. The reason is
* that clixon needs a global traversal to re-evaluate WHEN nodes depending on changed targets
*/
static int
xml_mark_added_ancestors(cxobj *x,
void *arg)
{
int flags = (intptr_t)arg;
if (xml_flag(x, flags)){
xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
return 2;
}
return 0;
}
/*! Modify database given an xml tree and an operation
*
* @param[in] h CLICON handle
@ -1197,20 +1224,15 @@ xmldb_put(clixon_handle h,
cbuf *cbret)
{
int retval = -1;
char *dbfile = NULL;
FILE *f = NULL;
yang_stmt *yspec;
cxobj *x0 = NULL;
db_elmnt *de = NULL;
db_elmnt de0 = {0,};
int ret;
cxobj *xnacm = NULL;
cxobj *xmodst = NULL;
cxobj *x;
int permit = 0; /* nacm permit all */
char *format;
cvec *nsc = NULL; /* nacm namespace context */
int firsttime = 0;
int pretty;
cxobj *xerr = NULL;
clixon_debug(CLIXON_DBG_DATASTORE|CLIXON_DBG_DETAIL, "db %s", db);
@ -1245,7 +1267,6 @@ xmldb_put(clixon_handle h,
xml_name(x0), DATASTORE_TOP_SYMBOL);
goto done;
}
/* XXX Ad-hoc:
* In yang mounts yang specs may not be available
* in initial parsing, but may be at a later stage.
@ -1256,13 +1277,8 @@ xmldb_put(clixon_handle h,
goto done;
}
/* Here x0 looks like: <config>...</config> */
#if 0 /* debug */
if (xml_apply0(x1, -1, xml_sort_verify, NULL) < 0)
clixon_log(h, LOG_NOTICE, "%s: verify failed #1", __FUNCTION__);
#endif
xnacm = clicon_nacm_cache(h);
permit = (xnacm==NULL);
/* Here assume if xnacm is set and !permit do NACM */
clicon_data_del(h, "objectexisted");
/*
@ -1283,11 +1299,10 @@ xmldb_put(clixon_handle h,
/* Remove NONE nodes if all subs recursively are also NONE */
if (xml_tree_prune_flagged_sub(x0, XML_FLAG_NONE, 0, NULL) <0)
goto done;
if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
(void*)(XML_FLAG_NONE|XML_FLAG_MARK)) < 0)
if (xml_apply(x0, CX_ELMNT, xml_mark_added_ancestors, (void*)(XML_FLAG_ADD|XML_FLAG_DEL)) < 0)
goto done;
/* Remove empty non-presence containers recursively.
* XXX should really be done for only new data in text_modify
* XXX should use xml_Mark_added_ancestors algorithm that xml_default_recurse uses for only
*/
if (xml_defaults_nopresence(x0, 3) < 0)
goto done;
@ -1296,105 +1311,31 @@ xmldb_put(clixon_handle h,
if (xml_global_defaults(h, x0, nsc, "/", yspec, 0) < 0)
goto done;
/* Add default recursive values */
if (xml_default_recurse(x0, 0) < 0)
if (xml_default_recurse_flag(x0, 0, XML_FLAG_ADD|XML_FLAG_DEL) < 0)
goto done;
/* Clear flags from previous steps */
if (xml_apply(x0, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
(void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE)) < 0)
goto done;
/* Write back to datastore cache if first time */
{
db_elmnt de0 = {0,};
if (de != NULL)
de0 = *de;
if (de0.de_xml == NULL)
de0.de_xml = x0;
de0.de_empty = (xml_child_nr(de0.de_xml) == 0);
clicon_db_elmnt_set(h, db, &de0);
}
if (xmldb_db2file(h, db, &dbfile) < 0)
goto done;
if (dbfile==NULL){
clixon_err(OE_XML, 0, "dbfile NULL");
goto done;
}
/* Add module revision info before writing to file)
* Only if CLICON_XMLDB_MODSTATE is set
*/
if ((x = clicon_modst_cache_get(h, 1)) != NULL){
if ((xmodst = xml_dup(x)) == NULL)
goto done;
if (xml_addsub(x0, xmodst) < 0)
goto done;
}
if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
clixon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
}
if ((f = fopen(dbfile, "w")) == NULL){
clixon_err(OE_CFG, errno, "Creating file %s", dbfile);
goto done;
}
pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY");
if (strcmp(format, "json")==0){
if (clixon_json2file(f, x0, pretty, fprintf, 0, 0) < 0)
goto done;
}
else if (clixon_xml2file1(f, x0, 0, pretty, NULL, fprintf, 0, 0, WITHDEFAULTS_EXPLICIT) < 0)
goto done;
/* Remove modules state after writing to file
*/
if (xmodst && xml_purge(xmodst) < 0)
/* Write cache to file */
if (xmldb_write_cache2file(h, db) < 0)
goto done;
retval = 1;
done:
clixon_debug(CLIXON_DBG_DATASTORE | CLIXON_DBG_DETAIL, "retval:%d", retval);
if (f != NULL)
fclose(f);
if (xerr)
xml_free(xerr);
if (nsc)
xml_nsctx_free(nsc);
if (dbfile)
free(dbfile);
return retval;
fail:
retval = 0;
goto done;
}
/* Dump a datastore to file including modstate
*/
int
xmldb_dump(clixon_handle h,
FILE *f,
cxobj *xt)
{
int retval = -1;
cxobj *x;
cxobj *xmodst = NULL;
char *format;
int pretty;
/* clear XML tree of defaults */
if (xml_tree_prune_flagged(xt, XML_FLAG_DEFAULT, 1) < 0)
goto done;
/* Add modstate first */
if ((x = clicon_modst_cache_get(h, 1)) != NULL){
if ((xmodst = xml_dup(x)) == NULL)
goto done;
if (xml_child_insert_pos(xt, xmodst, 0) < 0)
goto done;
}
if ((format = clicon_option_str(h, "CLICON_XMLDB_FORMAT")) == NULL){
clixon_err(OE_CFG, ENOENT, "No CLICON_XMLDB_FORMAT");
goto done;
}
pretty = clicon_option_bool(h, "CLICON_XMLDB_PRETTY");
if (strcmp(format,"json")==0){
if (clixon_json2file(f, xt, pretty, fprintf, 0, 0) < 0)
goto done;
}
else if (clixon_xml2file(f, xt, 0, pretty, NULL, fprintf, 0, 0) < 0)
goto done;
retval = 0;
done:
return retval;
}

View file

@ -439,6 +439,7 @@ clixon_err_fn(clixon_handle h,
}
}
va_start(ap, format);
/* This checks for customizable errors */
if (clixon_plugin_errmsg_all(h, fn, line, LOG_TYPE_ERR,
&category, &suberr, xerr, format, ap, &cb) < 0)
goto done;

View file

@ -1044,15 +1044,21 @@ xml_child_append(cxobj *xp,
return 0;
}
/*! Insert child xc at position i under parent xp
/*! Insert child XML at specific position under XML parent
*
* @param[in] xp Parent XML node
* @param[in] xc Child XML node
* @param[in] pos Position
* @retval 0 OK
* @retval -1 Error
* @see xml_child_append
* @note does not do anything with child, you may need to set its parent, etc
*/
int
xml_child_insert_pos(cxobj *xp,
cxobj *xc,
int i)
int pos)
{
size_t size;
@ -1070,9 +1076,9 @@ xml_child_insert_pos(cxobj *xp,
return -1;
}
}
size = (xml_child_nr(xp) - i - 1)*sizeof(cxobj *);
memmove(&xp->x_childvec[i+1], &xp->x_childvec[i], size);
xp->x_childvec[i] = xc;
size = (xml_child_nr(xp) - pos - 1)*sizeof(cxobj *);
memmove(&xp->x_childvec[pos+1], &xp->x_childvec[pos], size);
xp->x_childvec[pos] = xc;
return 0;
}

View file

@ -312,6 +312,7 @@ xml_default(yang_stmt *yt,
case Y_CASE:
yc = NULL;
while ((yc = yn_each(yt, yc)) != NULL) {
// XXX consider only data nodes for optimization?
/* If config parameter and local is config false */
if (!state && !yang_config(yc))
continue;
@ -349,6 +350,8 @@ xml_default(yang_stmt *yt,
/* If this is non-presence, (and it does not exist in xt) call
* recursively and create nodes if any default value exist first.
* Then continue and populate?
* Also this code expands some "when" statements that have nothing to do with
* defaults.
*/
if (xml_find_type(xt, NULL, yang_argument_get(yc), CX_ELMNT) == NULL){
/* No such container exist, recursively try if needed */
@ -396,12 +399,41 @@ xml_default(yang_stmt *yt,
int
xml_default_recurse(cxobj *xn,
int state)
{
return xml_default_recurse_flag(xn, state, 0x0);
}
/*! Selectively recursively fill in default values in an XML tree using flags
*
* Skip nodes that are not either CHANGE or "flag" (typically ADD|DEL)
* When ADD is encountered process all children.
* This will process all nodes that lead to ADD nodes and skip others.
* @param[in] xt XML tree
* @param[in] state If set expand defaults also for state data, otherwise only config
* @param[in] flag If set only traverse nodes marked with flag (or CHANGE)
* @retval 0 OK
* @retval -1 Error
* @see xml_default_recurse
*/
int
xml_default_recurse_flag(cxobj *xn,
int state,
int flag)
{
int retval = -1;
yang_stmt *yn;
cxobj *x;
yang_stmt *y;
if (flag){
if (xml_flag(xn, XML_FLAG_CHANGE) != 0)
; /* continue */
else if (xml_flag(xn, flag) != 0){
flag = 0x0; /* Pass all */
}
else
goto skip;
}
if ((yn = (yang_stmt*)xml_spec(xn)) != NULL){
if (xml_default(yn, xn, state) < 0)
goto done;
@ -412,9 +444,10 @@ xml_default_recurse(cxobj *xn,
if (!state && !yang_config(y))
continue;
}
if (xml_default_recurse(x, state) < 0)
if (xml_default_recurse_flag(x, state, flag) < 0)
goto done;
}
skip:
retval = 0;
done:
return retval;
@ -457,6 +490,7 @@ xml_global_defaults_create(cxobj *xt,
* @param[in] xpath Filter global defaults with this and merge with xt
* @param[in] yspec Top-level YANG specification tree, all modules
* @param[in] state Set if global state, otherwise config
* @param[in] flags Only traverse nodes where flag is set
* @retval 0 OK
* @retval -1 Error
* Uses cache?

View file

@ -172,6 +172,9 @@ expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS>
new "Set s3 to 99"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><np3 xmlns=\"urn:example:clixon\"><s3>99</s3></np3></config></edit-config></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "commit"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "get config np3 with npleaf and npext"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/ex:np3\" xmlns:ex=\"urn:example:clixon\" /></get-config></rpc>" "" "<rpc-reply $DEFAULTNS><data><np3 xmlns=\"urn:example:clixon\"><s3>99</s3></np3></data></rpc-reply>"

View file

@ -50,11 +50,8 @@ EOF
# data model.
cat <<EOF > $fyang
module example {
namespace "http://example.com/ns/interfaces";
prefix exam;
typedef status-type {
description "Interface status";
type enumeration {
@ -66,7 +63,6 @@ module example {
}
default ok;
}
container interfaces {
description "Example interfaces group";