Refactoring for better performance of large auto-cli specs

* Fixed: [very slow execution of load_set_file #288](https://github.com/clicon/clixon/issues/288)
* New `clixon-lib@2021-11-11.yang` revision
  * Modified option: RPC stats extended with YANG stats
* Modified `clixon-config@2021-11-11.yang` revision
  * Added option:
    * CLICON_PLUGIN_CALLBACK_CHECK
    * Enable to make plugin context check before and after all callbacks.
* Added statistics for YANG: number of objects and memory used
* Use the treeref no-copy option of CLIgen to reduce memory
* Refactored cli-generation/autocli-start code
* Refactored cligen glue functions to use cligen_eval directly (remove clicon_eval,clixon_cligen_eval)
This commit is contained in:
Olof hagsand 2021-11-25 12:04:05 +01:00
parent b91ce762d5
commit 5388aace12
29 changed files with 760 additions and 451 deletions

View file

@ -198,9 +198,9 @@ backend_client_rm(clicon_handle h,
* @retval -1 Error
*/
static int
clixon_stats_get_db(clicon_handle h,
char *dbname,
cbuf *cb)
clixon_stats_datastore_get(clicon_handle h,
char *dbname,
cbuf *cb)
{
int retval = -1;
cxobj *xt = NULL;
@ -234,6 +234,37 @@ clixon_stats_get_db(clicon_handle h,
return retval;
}
/*! Get clixon per datastore stats
* @param[in] h Clicon handle
* @param[in] dbname Datastore name
* @param[in,out] cb Cligen buf
* @retval 0 OK
* @retval -1 Error
*/
static int
clixon_stats_module_get(clicon_handle h,
yang_stmt *ys,
cbuf *cb)
{
int retval = -1;
uint64_t nr = 0;
size_t sz = 0;
cxobj *xn = NULL;
if (ys == NULL)
return 0;
if (yang_stats(ys, &nr, &sz) < 0)
goto done;
cprintf(cb, "<module xmlns=\"%s\"><name>%s</name><nr>%" PRIu64 "</nr>"
"<size>%zu</size></module>",
CLIXON_LIB_NS, yang_argument_get(ys), nr, sz);
retval = 0;
done:
if (xn)
xml_free(xn);
return retval;
}
/*! Loads all or part of a specified configuration to target configuration
*
* @param[in] h Clicon handle
@ -996,19 +1027,41 @@ from_client_stats(clicon_handle h,
void *arg,
void *regarg)
{
int retval = -1;
uint64_t nr;
int retval = -1;
uint64_t nr;
yang_stmt *ym;
cprintf(cbret, "<rpc-reply xmlns=\"%s\">", NETCONF_BASE_NAMESPACE);
xml_stats_global(&nr);
cprintf(cbret, "<global xmlns=\"%s\">", CLIXON_LIB_NS);
nr=0;
xml_stats_global(&nr);
cprintf(cbret, "<global xmlns=\"%s\"><xmlnr>%" PRIu64 "</xmlnr></global>", CLIXON_LIB_NS, nr);
if (clixon_stats_get_db(h, "running", cbret) < 0)
cprintf(cbret, "<xmlnr>%" PRIu64 "</xmlnr>", nr);
nr=0;
yang_stats_global(&nr);
cprintf(cbret, "<yangnr>%" PRIu64 "</yangnr>", nr);
cprintf(cbret, "</global>");
if (clixon_stats_datastore_get(h, "running", cbret) < 0)
goto done;
if (clixon_stats_get_db(h, "candidate", cbret) < 0)
if (clixon_stats_datastore_get(h, "candidate", cbret) < 0)
goto done;
if (clixon_stats_get_db(h, "startup", cbret) < 0)
if (clixon_stats_datastore_get(h, "startup", cbret) < 0)
goto done;
ym = NULL;
while ((ym = yn_each(clicon_config_yang(h), ym)) != NULL) {
if (clixon_stats_module_get(h, ym, cbret) < 0)
goto done;
}
ym = NULL;
while ((ym = yn_each(clicon_dbspec_yang(h), ym)) != NULL) {
if (clixon_stats_module_get(h, ym, cbret) < 0)
goto done;
}
ym = NULL;
while ((ym = yn_each(clicon_nacm_ext_yang(h), ym)) != NULL) {
if (clixon_stats_module_get(h, ym, cbret) < 0)
goto done;
}
cprintf(cbret, "</rpc-reply>");
retval = 0;
done:

View file

@ -931,20 +931,21 @@ from_client_restart_one(clicon_handle h,
yang_stmt *yspec;
int i;
cxobj *xn;
plugin_context_t *pc = NULL;
void *wh = NULL;
yspec = clicon_dbspec_yang(h);
if (xmldb_db_reset(h, db) < 0)
goto done;
/* Application may define extra xml in its reset function*/
if ((resetfn = clixon_plugin_api_get(cp)->ca_reset) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if ((retval = resetfn(h, db)) < 0) {
clicon_debug(1, "plugin_start() failed");
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
/* 1. Start transaction */
@ -998,7 +999,6 @@ from_client_restart_one(clicon_handle h,
xml_flag_set(xn, XML_FLAG_CHANGE);
xml_apply_ancestor(xn, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
}
/* Call plugin transaction start callbacks */
if (plugin_transaction_begin_one(cp, h, td) < 0)
goto fail;
@ -1026,8 +1026,6 @@ from_client_restart_one(clicon_handle h,
goto fail;
retval = 1;
done:
if (pc)
free(pc);
if (td){
xmldb_get0_free(h, &td->td_target);
transaction_free(td);

View file

@ -78,26 +78,27 @@ clixon_plugin_reset_one(clixon_plugin_t *cp,
clicon_handle h,
char *db)
{
int retval = -1;
plgreset_t *fn; /* callback */
plugin_context_t *pc = NULL;
int retval = -1;
plgreset_t *fn; /* callback */
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_reset) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, db) < 0) {
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (clicon_errno < 0)
clicon_log(LOG_WARNING, "%s: Internal error: Reset callback in plugin: %s returned -1 but did not make a clicon_err call",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -138,26 +139,26 @@ clixon_plugin_pre_daemon_one(clixon_plugin_t *cp,
{
int retval = -1;
plgdaemon_t *fn; /* Daemonize plugin callback function */
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_pre_daemon) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h) < 0) {
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (clicon_errno < 0)
clicon_log(LOG_WARNING, "%s: Internal error: Pre-daemon callback in plugin:\
%s returned -1 but did not make a clicon_err call",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -199,24 +200,25 @@ clixon_plugin_daemon_one(clixon_plugin_t *cp,
{
int retval = -1;
plgdaemon_t *fn; /* Daemonize plugin callback function */
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_daemon) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h) < 0) {
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (clicon_errno < 0)
clicon_log(LOG_WARNING, "%s: Internal error: Daemon callback in plugin: %s returned -1 but did not make a clicon_err call",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -283,28 +285,29 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
int retval = -1;
plgstatedata_t *fn; /* Plugin statedata fn */
cxobj *x = NULL;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_statedata) != NULL){
if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
goto done;
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, nsc, xpath, x) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (clicon_errno < 0)
clicon_log(LOG_WARNING, "%s: Internal error: State callback in plugin: %s returned -1 but did not make a clicon_err call",
__FUNCTION__, clixon_plugin_name_get(cp));
goto fail; /* Dont quit here on user callbacks */
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
if (xp && x)
*xp = x;
retval = 1;
done:
if (pc)
free(pc);
return retval;
fail:
retval = 0;
@ -437,20 +440,19 @@ clixon_plugin_lockdb_one(clixon_plugin_t *cp,
{
int retval = -1;
plglockdb_t *fn; /* Plugin statedata fn */
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_lockdb) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, db, lock, id) < 0)
goto done;
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -613,24 +615,25 @@ plugin_transaction_begin_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_begin) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -673,24 +676,26 @@ plugin_transaction_validate_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_validate) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -731,24 +736,25 @@ plugin_transaction_complete_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_complete) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -821,24 +827,25 @@ plugin_transaction_commit_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_commit) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -872,7 +879,6 @@ plugin_transaction_commit_all(clicon_handle h,
return retval;
}
/*! Call single plugin transaction_commit_done() in a commit transaction
* @param[in] cp Plugin handle
* @param[in] h Clixon handle
@ -887,24 +893,25 @@ plugin_transaction_commit_done_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_commit_done) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -945,24 +952,25 @@ plugin_transaction_end_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_end) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}
@ -996,24 +1004,25 @@ plugin_transaction_abort_one(clixon_plugin_t *cp,
{
int retval = -1;
trans_cb_t *fn;
plugin_context_t *pc = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_trans_abort) != NULL){
if ((pc = plugin_context_get()) == NULL)
wh = NULL;
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, (transaction_data)td) < 0){
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (!clicon_errno) /* sanity: log if clicon_err() is not called ! */
clicon_log(LOG_NOTICE, "%s: Plugin '%s' callback does not make clicon_err call on error",
__FUNCTION__, clixon_plugin_name_get(cp));
goto done;
}
if (plugin_context_check(pc, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
if (plugin_context_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
retval = 0;
done:
if (pc)
free(pc);
return retval;
}