experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optmize YANG memory: Yang-type cache only for original trees

Browse source
This commit is contained in:
Olof hagsand 2024-08-13 13:30:28 +02:00
parent 9da4939ee0
commit bd5214dde1
4 changed files with 109 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -16,9 +16,10 @@ Expected: October 2024
### Features
* Added YANG struct memory optimization
* Optimize YANG memory
* Added union and extended struct for uncommon fields
* Removed per-object YANG linenr info
* Yang-type cache only for original trees (not derived via grouping/augment)
* New: [CLI simple alias](https://github.com/clicon/cligen/issues/112)
* See: https://clixon-docs.readthedocs.io/en/latest/cli.html#cli-aliases
* List pagination: Added where, sort-by and direction parameter for configured data

5
lib/clixon/clixon_yang.h
View file

@ -320,11 +320,10 @@ int yang_config_ancestor(yang_stmt *ys);
int yang_features(clixon_handle h, yang_stmt *yt);
cvec *yang_arg2cvec(yang_stmt *ys, char *delimi);
int yang_key_match(yang_stmt *yn, char *name, int *lastkey);
int yang_type_cache_regexp_set(yang_stmt *ytype, int rxmode, cvec *regexps);
int yang_type_cache_get2(yang_stmt *ytype, yang_stmt **resolved, int *options,
cvec **cvv, cvec *patterns, int *rxmode, cvec *regexps, uint8_t *fraction);
cvec **cvv, cvec *patterns, cvec *regexps, uint8_t *fraction);
int yang_type_cache_set2(yang_stmt *ys, yang_stmt *resolved, int options, cvec *cvv,
cvec *patterns, uint8_t fraction);
cvec *patterns, uint8_t fraction, int rxmode, cvec *regexps);
yang_stmt *yang_anydata_add(yang_stmt *yp, char *name);
int yang_extension_value(yang_stmt *ys, char *name, char *ns, int *exist, char **value);
int yang_sort_subelements(yang_stmt *ys);

125
lib/src/clixon_yang.c
View file

@ -180,7 +180,6 @@ static map_ptr2ptr *_yang_mymodule_map = NULL;
/* Forward static */
static int yang_type_cache_free(yang_type_cache *ycache);
static int yang_type_cache_cp(yang_stmt *ynew, yang_stmt *yold);
/* Access functions
*/
@ -1071,30 +1070,40 @@ yn_realloc(yang_stmt *yn)
* Comments includes for:
* - nodes that could not be skipped and the failed test
* - nodes that have children in turn
* - nodes that can be refined
* @note RFC 7950 Sec 7.13.2. The "refine" Statement can change the following nodes
* - default values
* - description
* - reference
* - config
* - mandatory
* - presence
* - min/max-elements
* - if-feature
*/
static int
uses_orig_ptr(enum rfc_6020 keyword)
{
return
// keyword == Y_CONFIG // NO (test_openconfig.sh)
// keyword == Y_DEFAULT // NO (test_augment.sh)
keyword == Y_DESCRIPTION
// keyword == Y_CONFIG // NO (test_openconfig.sh) + refine
// keyword == Y_DEFAULT // NO (test_augment.sh) + refine
keyword == Y_DESCRIPTION // XXX refine
|| keyword == Y_ENUM // children
|| keyword == Y_ERROR_APP_TAG
|| keyword == Y_ERROR_MESSAGE
|| keyword == Y_FRACTION_DIGITS
// || keyword == Y_KEY // NO
|| keyword == Y_LENGTH // children
|| keyword == Y_MANDATORY
|| keyword == Y_MAX_ELEMENTS
|| keyword == Y_MIN_ELEMENTS
|| keyword == Y_MANDATORY // XXX refine
|| keyword == Y_MAX_ELEMENTS // XXX refine
|| keyword == Y_MIN_ELEMENTS // XXX refine
|| keyword == Y_MODIFIER
|| keyword == Y_ORDERED_BY
|| keyword == Y_PATH
|| keyword == Y_PATTERN // children
|| keyword == Y_POSITION
|| keyword == Y_PREFIX
|| keyword == Y_PRESENCE
|| keyword == Y_PRESENCE // XXX refine
|| keyword == Y_RANGE // children
|| keyword == Y_REQUIRE_INSTANCE
|| keyword == Y_STATUS
@ -1163,11 +1172,8 @@ ys_cp_one(yang_stmt *ynew,
}
switch (yold->ys_keyword) { /* type-specifi union fields */
case Y_TYPE:
if (yang_typecache_get(yold)){
if (yang_typecache_get(yold)) /* Dont copy type cache, use only original */
yang_typecache_set(ynew, NULL);
if (yang_type_cache_cp(ynew, yold) < 0)
goto done;
}
break;
default:
break;
@ -2503,10 +2509,10 @@ ys_populate_leaf(clixon_handle h,
int cvret;
int ret;
char *reason = NULL;
yang_stmt *yrestype; /* resolved type */
yang_stmt *yrestype = NULL; /* resolved type */
char *restype; /* resolved type */
char *origtype=NULL; /* original type */
uint8_t fraction_digits;
uint8_t fraction_digits = 0;
int options = 0x0;
yang_stmt *ytypedef; /* where type is define */
@ -3275,6 +3281,12 @@ ys_populate(yang_stmt *ys,
/*! Run after grouping expand and augment
*
* Run in yang_apply but also other places
* @param[in] yn yang node
* @param[in] arg Argument
* @retval n OK, abort traversal and return to caller with "n"
* @retval 0 OK, continue with next
* @retval -1 Error, abort
* @see ys_populate run before grouping expand and augment
*/
int
@ -3287,6 +3299,11 @@ ys_populate2(yang_stmt *ys,
int ret;
switch(ys->ys_keyword){
case Y_AUGMENT:
case Y_GROUPING:
retval = 2; /* Skip sub-tree */
goto done;
break;
case Y_LEAF:
case Y_LEAF_LIST:
if (ys_populate_leaf(h, ys) < 0)
@ -3828,7 +3845,6 @@ yang_arg2cvec(yang_stmt *ys,
return cvv;
}
/*! Check if yang node yn has key-stmt as child which matches name
*
* The function looks at the LIST argument string (not actual children)
@ -3883,7 +3899,6 @@ yang_key_match(yang_stmt *yn,
* @param[in] rxmode Which regexp engine to use, see enum regexp_mode
* @retval 0 OK
* @retval -1 Error
* @see yang_type_cache_regexp_set where cache is extended w compiled regexps
*/
int
yang_type_cache_set2(yang_stmt *ys,
@ -3891,7 +3906,9 @@ yang_type_cache_set2(yang_stmt *ys,
int options,
cvec *cvv,
cvec *patterns,
uint8_t fraction)
uint8_t fraction,
int rxmode,
cvec *regexps)
{
int retval = -1;
yang_type_cache *ycache;
@ -3919,45 +3936,13 @@ yang_type_cache_set2(yang_stmt *ys,
goto done;
}
ycache->yc_fraction = fraction;
retval = 0;
done:
return retval;
}
/*! Extend yang type cache with compiled regexps
*
* Compiled Regexps are computed in validate code - after initial cache set
* @param[in] ytype YANG type statement
* @param[in] rxmode Which regexp engine to use, see enum regexp_mode
* @param[in] regexps
* @retval 0 OK
* @retval -1 Error
*/
int
yang_type_cache_regexp_set(yang_stmt *ytype,
int rxmode,
cvec *regexps)
{
int retval = -1;
yang_type_cache *ycache;
if (regexps == NULL || yang_keyword_get(ytype) != Y_TYPE) {
clixon_err(OE_YANG, EINVAL, "regexps is NULL, or are already set, or ytype is not YTYPE");
goto done;
}
if ((ycache = ytype->ys_typecache) == NULL){
clixon_err(OE_YANG, 0, "Typecache is NULL");
goto done;
}
if (ycache->yc_regexps != NULL){
clixon_err(OE_YANG, 0, "regexp is already set");
goto done;
}
if (regexps != NULL) {
ycache->yc_rxmode = rxmode;
if ((ycache->yc_regexps = cvec_dup(regexps)) == NULL){
clixon_err(OE_UNIX, errno, "cvec_dup");
goto done;
}
}
retval = 0;
done:
return retval;
@ -3977,7 +3962,6 @@ yang_type_cache_get2(yang_stmt *ytype,
int *options,
cvec **cvv,
cvec *patterns,
int *rxmode,
cvec *regexps,
uint8_t *fraction)
{
@ -4006,8 +3990,6 @@ yang_type_cache_get2(yang_stmt *ytype,
while ((cv = cvec_each(ycache->yc_regexps, cv)) != NULL)
cvec_append_var(regexps, cv);
}
if (rxmode)
*rxmode = ycache->yc_rxmode;
if (fraction)
*fraction = ycache->yc_fraction;
retval = 1; /* cache exists and is returned OK */
@ -4015,40 +3997,6 @@ yang_type_cache_get2(yang_stmt *ytype,
return retval;
}
/*! Copy yang type cache
*/
static int
yang_type_cache_cp(yang_stmt *ynew,
yang_stmt *yold)
{
int retval = -1;
int options;
cvec *cvv;
cvec *patterns = NULL;
uint8_t fraction;
yang_stmt *resolved;
int ret;
if ((patterns = cvec_new(0)) == NULL){
clixon_err(OE_UNIX, errno, "cvec_new");
goto done;
}
/* Note, regexps are not copied since they are voids, if they were, they
* could not be freed in a simple way since copies are made at augment/group
*/
if ((ret = yang_type_cache_get2(yold,
&resolved, &options, &cvv, patterns, NULL, NULL, &fraction)) < 0)
goto done;
if (ret == 1 &&
yang_type_cache_set2(ynew, resolved, options, cvv, patterns, fraction) < 0)
goto done;
retval = 0;
done:
if (patterns)
cvec_free(patterns);
return retval;
}
/*! Free yang type cache
*/
static int
@ -4252,7 +4200,6 @@ yang_sort_subelements(yang_stmt *ys)
#endif
}
#ifdef XML_EXPLICIT_INDEX
/*! Mark element as search_index in list
*

114
lib/src/clixon_yang_type.c
View file

@ -34,7 +34,7 @@
* Yang type related functions
* Part of this is type resolving which is pretty complex
* +--> yang_type_cache_set
* +--> yang_type_cache_set2
* (called at parse) |
* ys_resolve_type --+ ys_populate_range, yang_enum_int_value(NULL)
* \ | cml
@ -44,11 +44,10 @@
* ^ ^ ^ ^
* | | | |
* | yang2cli_var | yang2cli_var_union_one
* ys_cv_validate---+ ys_cv_validate_union_one
* | \ /
* | \ / yang_type_cache_regexp_set
* ys_populate_leaf, +--> compile_pattern2regexp (compile regexps)
* xml_cv_cache (NULL) +--> cv_validate1 --> cv_validate_pattern (exec regexps)
* ys_cv_validate
* |
* ys_populate_leaf,
* xml_cv_cache (NULL)
* yang_type2cv (simplified)
*
* NOTE
@ -231,18 +230,19 @@ compile_pattern2regexp(clixon_handle h,
* @note unions not cached
*/
int
ys_resolve_type(yang_stmt *ys,
ys_resolve_type(yang_stmt *ytype,
void *arg)
{
// clixon_handle h = (clixon_handle)arg;
clixon_handle h = (clixon_handle)arg;
int retval = -1;
int options = 0x0;
cvec *cvv = NULL;
cvec *patterns = NULL;
uint8_t fraction = 0;
yang_stmt *resolved = NULL;
cvec *regexps = NULL;
if (yang_keyword_get(ys) != Y_TYPE){
if (yang_keyword_get(ytype) != Y_TYPE){
clixon_err(OE_YANG, EINVAL, "Expected Y_TYPE");
goto done;
}
@ -250,25 +250,35 @@ ys_resolve_type(yang_stmt *ys,
clixon_err(OE_UNIX, errno, "cvec_new");
goto done;
}
/* Recursively resolve ys -> resolve with restrictions(options, etc)
* Note that the resolved type could be ys itself.
/* Recursively resolve ytype -> resolve with restrictions(options, etc)
* Note that the resolved type could be ytype itself.
*/
if (yang_type_resolve(yang_parent_get(ys), yang_parent_get(ys),
ys, &resolved,
&options, &cvv, patterns, NULL, &fraction) < 0)
if (yang_type_resolve(yang_parent_get(ytype), yang_parent_get(ytype),
ytype, &resolved,
&options, &cvv, patterns, NULL, &fraction) < 0){
goto done;
}
if (resolved == NULL){
clixon_err(OE_YANG, 0, "result-type should not be NULL");
goto done;
}
/* Cache the type resolving locally. Only place where this is done.
* Why not do it in yang_type_resolve? (compile regexps needs clixon_handle)
*/
if (yang_type_cache_set2(ys, resolved, options, cvv,
patterns, fraction) < 0)
* Compile / initialize pattern regexp cache */
if (cvec_len(patterns) > 0) {
if ((regexps = cvec_new(0)) == NULL){
clixon_err(OE_UNIX, errno, "cvec_new");
goto done;
}
if (compile_pattern2regexp(h, patterns, regexps) < 1)
goto done;
}
if (yang_type_cache_set2(ytype, resolved, options, cvv,
patterns, fraction, clicon_yang_regexp(h), regexps) < 0)
goto done;
retval = 0;
done:
if (regexps)
cvec_free(regexps);
if (patterns)
cvec_free(patterns);
return retval;
@ -887,17 +897,6 @@ ys_cv_validate_union_one(clixon_handle h,
}
if (retval == 0)
goto done;
/* The regexp cache may be invalidated, in that case re-compile
* eg due to copying
*/
if (cvec_len(patterns)!=0 && cvec_len(regexps)==0){
if (compile_pattern2regexp(h, patterns, regexps) < 1)
goto done;
if (yang_type_cache_regexp_set(yt,
clicon_yang_regexp(h),
regexps) < 0)
goto done;
}
if ((retval = cv_validate1(h, cvt, cvtype, options, cvv,
regexps, yrestype, restype, reason)) < 0)
goto done;
@ -1001,12 +1000,12 @@ ys_cv_validate(clixon_handle h,
cvec *regexps = NULL;
enum cv_type cvtype;
char *origtype = NULL; /* orig type */
yang_stmt *yrestype; /* resolved type */
yang_stmt *yrestype = NULL; /* resolved type */
char *restype;
uint8_t fraction = 0;
int retval2;
char *val;
cg_var *cvt=NULL;
cg_var *cvt = NULL;
if (reason)
*reason=NULL;
@ -1015,23 +1014,23 @@ ys_cv_validate(clixon_handle h,
goto done;
}
ycv = yang_cv_get(ys);
if ((regexps = cvec_new(0)) == NULL){
if ((patterns = cvec_new(0)) == NULL){
clixon_err(OE_UNIX, errno, "cvec_new");
goto done;
}
if ((patterns = cvec_new(0)) == NULL){
if ((regexps = cvec_new(0)) == NULL){
clixon_err(OE_UNIX, errno, "cvec_new");
goto done;
}
if (yang_type_get(ys, &origtype, &yrestype,
&options, &cvv,
patterns, regexps,
patterns,
regexps,
&fraction) < 0)
goto done;
restype = yrestype?yang_argument_get(yrestype):NULL;
if (clicon_type2cv(origtype, restype, ys, &cvtype) < 0)
goto done;
if (cv_type_get(ycv) != cvtype){
/* special case: dbkey has rest syntax-> cv but yang cant have that */
if (cvtype == CGV_STRING && cv_type_get(ycv) == CGV_REST)
@ -1058,19 +1057,6 @@ ys_cv_validate(clixon_handle h,
retval = retval2; /* invalid (0) with latest reason or valid 1 */
}
else{
/* The regexp cache may be invalidated, in that case re-compile
* eg due to copying
*/
if (cvec_len(patterns)!=0 && cvec_len(regexps)==0){
yang_stmt *yt;
if (compile_pattern2regexp(h, patterns, regexps) < 1)
goto done;
yt = yang_find(ys, Y_TYPE, NULL);
if (yang_type_cache_regexp_set(yt,
clicon_yang_regexp(h),
regexps) < 0)
goto done;
}
/* Leafref needs to resolve referred node for type information
* From rfc7950 Sec 9.9:
* The leafref built-in type is restricted to the value space of some
@ -1329,11 +1315,11 @@ yang_type_resolve(yang_stmt *yorig,
cvec *regexps,
uint8_t *fraction)
{
int retval = -1;
yang_stmt *rytypedef = NULL; /* Resolved typedef of ytype */
yang_stmt *rytype; /* Resolved type of ytype */
char *type = NULL;
char *prefix = NULL;
int retval = -1;
yang_stmt *yn;
yang_stmt *yrmod; /* module where resolved type is looked for */
int ret;
@ -1341,24 +1327,13 @@ yang_type_resolve(yang_stmt *yorig,
if (options)
*options = 0x0;
*yrestype = NULL; /* Initialization of resolved type that may not be necessary */
if (nodeid_split(yang_argument_get(ytype), &prefix, &type) < 0)
goto done;
/* Cache does not work for eg string length 32? */
#if 1
if ((ret = yang_type_cache_get2(ytype, yrestype,
options, cvv, patterns, NULL, regexps, fraction)) < 0)
options, cvv, patterns, regexps, fraction)) < 0)
goto done;
if (ret == 1)
goto ok;
#else
if (yang_typecache_get(ytype) != NULL){
if (yang_type_cache_get2(ytype, yrestype,
options, cvv, patterns, NULL, regexps, fraction) < 0)
if (nodeid_split(yang_argument_get(ytype), &prefix, &type) < 0)
goto done;
goto ok;
}
#endif
/* Check if type is basic type. If so, return that */
if ((prefix == NULL && yang_builtin(type))){
*yrestype = ytype;
@ -1366,7 +1341,6 @@ yang_type_resolve(yang_stmt *yorig,
goto done;
goto ok;
}
/* Not basic type. Now check if prefix which means we look in other module */
if (prefix){ /* Go to top and find import that matches */
if ((yrmod = yang_find_module_by_prefix(ytype, prefix)) == NULL){
@ -1400,7 +1374,7 @@ yang_type_resolve(yang_stmt *yorig,
clixon_err(OE_DB, 0, "mandatory type object is not found");
goto done;
}
/* recursively resolve this new type */
/* Recursively resolve this new type */
if (yang_type_resolve(yorig, ys, rytype, yrestype,
options, cvv,
patterns, regexps,
@ -1483,6 +1457,7 @@ yang_type_get(yang_stmt *ys,
{
int retval = -1;
yang_stmt *ytype; /* type */
yang_stmt *yorig;
char *type = NULL;
if (yrestype == NULL){
@ -1491,6 +1466,10 @@ yang_type_get(yang_stmt *ys,
}
if (options)
*options = 0x0;
/* Use original tree to resolve types */
if ((yorig = yang_orig_get(ys)) != NULL) {
ys = yorig;
}
/* Find mandatory type */
if ((ytype = yang_find(ys, Y_TYPE, NULL)) == NULL){
clixon_err(OE_DB, ENOENT, "mandatory type object is not found");
@ -1504,8 +1483,11 @@ yang_type_get(yang_stmt *ys,
clixon_err(OE_XML, errno, "stdup");
goto done;
}
if (yang_type_resolve(ys, ys, ytype, yrestype,
options, cvv, patterns, regexps, fraction) < 0)
if (yang_type_resolve(ys, ys, ytype,
yrestype,
options,
cvv, patterns, regexps,
fraction) < 0)
goto done;
if (*yrestype == NULL){
clixon_err(OE_YANG, 0, "result-type should not be NULL");