diff --git a/CHANGELOG.md b/CHANGELOG.md index a140bbe2..ec425c25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,10 @@ Expected: October 2024 ### Features -* Added YANG struct memory optimization +* Optimize YANG memory * Added union and extended struct for uncommon fields * Removed per-object YANG linenr info + * Yang-type cache only for original trees (not derived via grouping/augment) * New: [CLI simple alias](https://github.com/clicon/cligen/issues/112) * See: https://clixon-docs.readthedocs.io/en/latest/cli.html#cli-aliases * List pagination: Added where, sort-by and direction parameter for configured data diff --git a/lib/clixon/clixon_yang.h b/lib/clixon/clixon_yang.h index 699b0aa5..4d915864 100644 --- a/lib/clixon/clixon_yang.h +++ b/lib/clixon/clixon_yang.h @@ -320,11 +320,10 @@ int yang_config_ancestor(yang_stmt *ys); int yang_features(clixon_handle h, yang_stmt *yt); cvec *yang_arg2cvec(yang_stmt *ys, char *delimi); int yang_key_match(yang_stmt *yn, char *name, int *lastkey); -int yang_type_cache_regexp_set(yang_stmt *ytype, int rxmode, cvec *regexps); int yang_type_cache_get2(yang_stmt *ytype, yang_stmt **resolved, int *options, - cvec **cvv, cvec *patterns, int *rxmode, cvec *regexps, uint8_t *fraction); + cvec **cvv, cvec *patterns, cvec *regexps, uint8_t *fraction); int yang_type_cache_set2(yang_stmt *ys, yang_stmt *resolved, int options, cvec *cvv, - cvec *patterns, uint8_t fraction); + cvec *patterns, uint8_t fraction, int rxmode, cvec *regexps); yang_stmt *yang_anydata_add(yang_stmt *yp, char *name); int yang_extension_value(yang_stmt *ys, char *name, char *ns, int *exist, char **value); int yang_sort_subelements(yang_stmt *ys); diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index f544add3..41899946 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -180,7 +180,6 @@ static map_ptr2ptr *_yang_mymodule_map = NULL; /* Forward static */ static int yang_type_cache_free(yang_type_cache *ycache); -static int yang_type_cache_cp(yang_stmt *ynew, yang_stmt *yold); /* Access functions */ @@ -1071,30 +1070,40 @@ yn_realloc(yang_stmt *yn) * Comments includes for: * - nodes that could not be skipped and the failed test * - nodes that have children in turn + * - nodes that can be refined + * @note RFC 7950 Sec 7.13.2. The "refine" Statement can change the following nodes + * - default values + * - description + * - reference + * - config + * - mandatory + * - presence + * - min/max-elements + * - if-feature */ static int uses_orig_ptr(enum rfc_6020 keyword) { return - // keyword == Y_CONFIG // NO (test_openconfig.sh) - // keyword == Y_DEFAULT // NO (test_augment.sh) - keyword == Y_DESCRIPTION + // keyword == Y_CONFIG // NO (test_openconfig.sh) + refine + // keyword == Y_DEFAULT // NO (test_augment.sh) + refine + keyword == Y_DESCRIPTION // XXX refine || keyword == Y_ENUM // children || keyword == Y_ERROR_APP_TAG || keyword == Y_ERROR_MESSAGE || keyword == Y_FRACTION_DIGITS // || keyword == Y_KEY // NO || keyword == Y_LENGTH // children - || keyword == Y_MANDATORY - || keyword == Y_MAX_ELEMENTS - || keyword == Y_MIN_ELEMENTS + || keyword == Y_MANDATORY // XXX refine + || keyword == Y_MAX_ELEMENTS // XXX refine + || keyword == Y_MIN_ELEMENTS // XXX refine || keyword == Y_MODIFIER || keyword == Y_ORDERED_BY || keyword == Y_PATH || keyword == Y_PATTERN // children || keyword == Y_POSITION || keyword == Y_PREFIX - || keyword == Y_PRESENCE + || keyword == Y_PRESENCE // XXX refine || keyword == Y_RANGE // children || keyword == Y_REQUIRE_INSTANCE || keyword == Y_STATUS @@ -1163,11 +1172,8 @@ ys_cp_one(yang_stmt *ynew, } switch (yold->ys_keyword) { /* type-specifi union fields */ case Y_TYPE: - if (yang_typecache_get(yold)){ + if (yang_typecache_get(yold)) /* Dont copy type cache, use only original */ yang_typecache_set(ynew, NULL); - if (yang_type_cache_cp(ynew, yold) < 0) - goto done; - } break; default: break; @@ -2503,10 +2509,10 @@ ys_populate_leaf(clixon_handle h, int cvret; int ret; char *reason = NULL; - yang_stmt *yrestype; /* resolved type */ + yang_stmt *yrestype = NULL; /* resolved type */ char *restype; /* resolved type */ char *origtype=NULL; /* original type */ - uint8_t fraction_digits; + uint8_t fraction_digits = 0; int options = 0x0; yang_stmt *ytypedef; /* where type is define */ @@ -3275,6 +3281,12 @@ ys_populate(yang_stmt *ys, /*! Run after grouping expand and augment * + * Run in yang_apply but also other places + * @param[in] yn yang node + * @param[in] arg Argument + * @retval n OK, abort traversal and return to caller with "n" + * @retval 0 OK, continue with next + * @retval -1 Error, abort * @see ys_populate run before grouping expand and augment */ int @@ -3287,6 +3299,11 @@ ys_populate2(yang_stmt *ys, int ret; switch(ys->ys_keyword){ + case Y_AUGMENT: + case Y_GROUPING: + retval = 2; /* Skip sub-tree */ + goto done; + break; case Y_LEAF: case Y_LEAF_LIST: if (ys_populate_leaf(h, ys) < 0) @@ -3828,7 +3845,6 @@ yang_arg2cvec(yang_stmt *ys, return cvv; } - /*! Check if yang node yn has key-stmt as child which matches name * * The function looks at the LIST argument string (not actual children) @@ -3883,7 +3899,6 @@ yang_key_match(yang_stmt *yn, * @param[in] rxmode Which regexp engine to use, see enum regexp_mode * @retval 0 OK * @retval -1 Error - * @see yang_type_cache_regexp_set where cache is extended w compiled regexps */ int yang_type_cache_set2(yang_stmt *ys, @@ -3891,7 +3906,9 @@ yang_type_cache_set2(yang_stmt *ys, int options, cvec *cvv, cvec *patterns, - uint8_t fraction) + uint8_t fraction, + int rxmode, + cvec *regexps) { int retval = -1; yang_type_cache *ycache; @@ -3919,44 +3936,12 @@ yang_type_cache_set2(yang_stmt *ys, goto done; } ycache->yc_fraction = fraction; - retval = 0; - done: - return retval; -} - -/*! Extend yang type cache with compiled regexps - * - * Compiled Regexps are computed in validate code - after initial cache set - * @param[in] ytype YANG type statement - * @param[in] rxmode Which regexp engine to use, see enum regexp_mode - * @param[in] regexps - * @retval 0 OK - * @retval -1 Error - */ -int -yang_type_cache_regexp_set(yang_stmt *ytype, - int rxmode, - cvec *regexps) -{ - int retval = -1; - yang_type_cache *ycache; - - if (regexps == NULL || yang_keyword_get(ytype) != Y_TYPE) { - clixon_err(OE_YANG, EINVAL, "regexps is NULL, or are already set, or ytype is not YTYPE"); - goto done; - } - if ((ycache = ytype->ys_typecache) == NULL){ - clixon_err(OE_YANG, 0, "Typecache is NULL"); - goto done; - } - if (ycache->yc_regexps != NULL){ - clixon_err(OE_YANG, 0, "regexp is already set"); - goto done; - } - ycache->yc_rxmode = rxmode; - if ((ycache->yc_regexps = cvec_dup(regexps)) == NULL){ - clixon_err(OE_UNIX, errno, "cvec_dup"); - goto done; + if (regexps != NULL) { + ycache->yc_rxmode = rxmode; + if ((ycache->yc_regexps = cvec_dup(regexps)) == NULL){ + clixon_err(OE_UNIX, errno, "cvec_dup"); + goto done; + } } retval = 0; done: @@ -3973,13 +3958,12 @@ yang_type_cache_regexp_set(yang_stmt *ytype, */ int yang_type_cache_get2(yang_stmt *ytype, - yang_stmt **resolved, - int *options, - cvec **cvv, - cvec *patterns, - int *rxmode, - cvec *regexps, - uint8_t *fraction) + yang_stmt **resolved, + int *options, + cvec **cvv, + cvec *patterns, + cvec *regexps, + uint8_t *fraction) { int retval = -1; cg_var *cv = NULL; @@ -4006,8 +3990,6 @@ yang_type_cache_get2(yang_stmt *ytype, while ((cv = cvec_each(ycache->yc_regexps, cv)) != NULL) cvec_append_var(regexps, cv); } - if (rxmode) - *rxmode = ycache->yc_rxmode; if (fraction) *fraction = ycache->yc_fraction; retval = 1; /* cache exists and is returned OK */ @@ -4015,40 +3997,6 @@ yang_type_cache_get2(yang_stmt *ytype, return retval; } -/*! Copy yang type cache - */ -static int -yang_type_cache_cp(yang_stmt *ynew, - yang_stmt *yold) -{ - int retval = -1; - int options; - cvec *cvv; - cvec *patterns = NULL; - uint8_t fraction; - yang_stmt *resolved; - int ret; - - if ((patterns = cvec_new(0)) == NULL){ - clixon_err(OE_UNIX, errno, "cvec_new"); - goto done; - } - /* Note, regexps are not copied since they are voids, if they were, they - * could not be freed in a simple way since copies are made at augment/group - */ - if ((ret = yang_type_cache_get2(yold, - &resolved, &options, &cvv, patterns, NULL, NULL, &fraction)) < 0) - goto done; - if (ret == 1 && - yang_type_cache_set2(ynew, resolved, options, cvv, patterns, fraction) < 0) - goto done; - retval = 0; - done: - if (patterns) - cvec_free(patterns); - return retval; -} - /*! Free yang type cache */ static int @@ -4252,7 +4200,6 @@ yang_sort_subelements(yang_stmt *ys) #endif } - #ifdef XML_EXPLICIT_INDEX /*! Mark element as search_index in list * diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index 705a5776..af1f446c 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -34,7 +34,7 @@ * Yang type related functions * Part of this is type resolving which is pretty complex - * +--> yang_type_cache_set + * +--> yang_type_cache_set2 * (called at parse) | * ys_resolve_type --+ ys_populate_range, yang_enum_int_value(NULL) * \ | cml @@ -44,11 +44,10 @@ * ^ ^ ^ ^ * | | | | * | yang2cli_var | yang2cli_var_union_one - * ys_cv_validate---+ ys_cv_validate_union_one - * | \ / - * | \ / yang_type_cache_regexp_set - * ys_populate_leaf, +--> compile_pattern2regexp (compile regexps) - * xml_cv_cache (NULL) +--> cv_validate1 --> cv_validate_pattern (exec regexps) + * ys_cv_validate + * | + * ys_populate_leaf, + * xml_cv_cache (NULL) * yang_type2cv (simplified) * * NOTE @@ -231,18 +230,19 @@ compile_pattern2regexp(clixon_handle h, * @note unions not cached */ int -ys_resolve_type(yang_stmt *ys, - void *arg) +ys_resolve_type(yang_stmt *ytype, + void *arg) { - // clixon_handle h = (clixon_handle)arg; - int retval = -1; - int options = 0x0; - cvec *cvv = NULL; - cvec *patterns = NULL; - uint8_t fraction = 0; - yang_stmt *resolved = NULL; + clixon_handle h = (clixon_handle)arg; + int retval = -1; + int options = 0x0; + cvec *cvv = NULL; + cvec *patterns = NULL; + uint8_t fraction = 0; + yang_stmt *resolved = NULL; + cvec *regexps = NULL; - if (yang_keyword_get(ys) != Y_TYPE){ + if (yang_keyword_get(ytype) != Y_TYPE){ clixon_err(OE_YANG, EINVAL, "Expected Y_TYPE"); goto done; } @@ -250,25 +250,35 @@ ys_resolve_type(yang_stmt *ys, clixon_err(OE_UNIX, errno, "cvec_new"); goto done; } - /* Recursively resolve ys -> resolve with restrictions(options, etc) - * Note that the resolved type could be ys itself. + /* Recursively resolve ytype -> resolve with restrictions(options, etc) + * Note that the resolved type could be ytype itself. */ - if (yang_type_resolve(yang_parent_get(ys), yang_parent_get(ys), - ys, &resolved, - &options, &cvv, patterns, NULL, &fraction) < 0) + if (yang_type_resolve(yang_parent_get(ytype), yang_parent_get(ytype), + ytype, &resolved, + &options, &cvv, patterns, NULL, &fraction) < 0){ goto done; + } if (resolved == NULL){ clixon_err(OE_YANG, 0, "result-type should not be NULL"); goto done; } /* Cache the type resolving locally. Only place where this is done. - * Why not do it in yang_type_resolve? (compile regexps needs clixon_handle) - */ - if (yang_type_cache_set2(ys, resolved, options, cvv, - patterns, fraction) < 0) + * Compile / initialize pattern regexp cache */ + if (cvec_len(patterns) > 0) { + if ((regexps = cvec_new(0)) == NULL){ + clixon_err(OE_UNIX, errno, "cvec_new"); + goto done; + } + if (compile_pattern2regexp(h, patterns, regexps) < 1) + goto done; + } + if (yang_type_cache_set2(ytype, resolved, options, cvv, + patterns, fraction, clicon_yang_regexp(h), regexps) < 0) goto done; retval = 0; done: + if (regexps) + cvec_free(regexps); if (patterns) cvec_free(patterns); return retval; @@ -887,17 +897,6 @@ ys_cv_validate_union_one(clixon_handle h, } if (retval == 0) goto done; - /* The regexp cache may be invalidated, in that case re-compile - * eg due to copying - */ - if (cvec_len(patterns)!=0 && cvec_len(regexps)==0){ - if (compile_pattern2regexp(h, patterns, regexps) < 1) - goto done; - if (yang_type_cache_regexp_set(yt, - clicon_yang_regexp(h), - regexps) < 0) - goto done; - } if ((retval = cv_validate1(h, cvt, cvtype, options, cvv, regexps, yrestype, restype, reason)) < 0) goto done; @@ -1001,12 +1000,12 @@ ys_cv_validate(clixon_handle h, cvec *regexps = NULL; enum cv_type cvtype; char *origtype = NULL; /* orig type */ - yang_stmt *yrestype; /* resolved type */ + yang_stmt *yrestype = NULL; /* resolved type */ char *restype; uint8_t fraction = 0; int retval2; char *val; - cg_var *cvt=NULL; + cg_var *cvt = NULL; if (reason) *reason=NULL; @@ -1015,23 +1014,23 @@ ys_cv_validate(clixon_handle h, goto done; } ycv = yang_cv_get(ys); - if ((regexps = cvec_new(0)) == NULL){ + if ((patterns = cvec_new(0)) == NULL){ clixon_err(OE_UNIX, errno, "cvec_new"); goto done; } - if ((patterns = cvec_new(0)) == NULL){ + if ((regexps = cvec_new(0)) == NULL){ clixon_err(OE_UNIX, errno, "cvec_new"); goto done; } if (yang_type_get(ys, &origtype, &yrestype, &options, &cvv, - patterns, regexps, + patterns, + regexps, &fraction) < 0) goto done; restype = yrestype?yang_argument_get(yrestype):NULL; if (clicon_type2cv(origtype, restype, ys, &cvtype) < 0) goto done; - if (cv_type_get(ycv) != cvtype){ /* special case: dbkey has rest syntax-> cv but yang cant have that */ if (cvtype == CGV_STRING && cv_type_get(ycv) == CGV_REST) @@ -1058,19 +1057,6 @@ ys_cv_validate(clixon_handle h, retval = retval2; /* invalid (0) with latest reason or valid 1 */ } else{ - /* The regexp cache may be invalidated, in that case re-compile - * eg due to copying - */ - if (cvec_len(patterns)!=0 && cvec_len(regexps)==0){ - yang_stmt *yt; - if (compile_pattern2regexp(h, patterns, regexps) < 1) - goto done; - yt = yang_find(ys, Y_TYPE, NULL); - if (yang_type_cache_regexp_set(yt, - clicon_yang_regexp(h), - regexps) < 0) - goto done; - } /* Leafref needs to resolve referred node for type information * From rfc7950 Sec 9.9: * The leafref built-in type is restricted to the value space of some @@ -1329,11 +1315,11 @@ yang_type_resolve(yang_stmt *yorig, cvec *regexps, uint8_t *fraction) { + int retval = -1; yang_stmt *rytypedef = NULL; /* Resolved typedef of ytype */ yang_stmt *rytype; /* Resolved type of ytype */ char *type = NULL; char *prefix = NULL; - int retval = -1; yang_stmt *yn; yang_stmt *yrmod; /* module where resolved type is looked for */ int ret; @@ -1341,24 +1327,13 @@ yang_type_resolve(yang_stmt *yorig, if (options) *options = 0x0; *yrestype = NULL; /* Initialization of resolved type that may not be necessary */ - - if (nodeid_split(yang_argument_get(ytype), &prefix, &type) < 0) - goto done; - /* Cache does not work for eg string length 32? */ -#if 1 if ((ret = yang_type_cache_get2(ytype, yrestype, - options, cvv, patterns, NULL, regexps, fraction)) < 0) + options, cvv, patterns, regexps, fraction)) < 0) goto done; if (ret == 1) goto ok; -#else - if (yang_typecache_get(ytype) != NULL){ - if (yang_type_cache_get2(ytype, yrestype, - options, cvv, patterns, NULL, regexps, fraction) < 0) - goto done; - goto ok; - } -#endif + if (nodeid_split(yang_argument_get(ytype), &prefix, &type) < 0) + goto done; /* Check if type is basic type. If so, return that */ if ((prefix == NULL && yang_builtin(type))){ *yrestype = ytype; @@ -1366,7 +1341,6 @@ yang_type_resolve(yang_stmt *yorig, goto done; goto ok; } - /* Not basic type. Now check if prefix which means we look in other module */ if (prefix){ /* Go to top and find import that matches */ if ((yrmod = yang_find_module_by_prefix(ytype, prefix)) == NULL){ @@ -1400,7 +1374,7 @@ yang_type_resolve(yang_stmt *yorig, clixon_err(OE_DB, 0, "mandatory type object is not found"); goto done; } - /* recursively resolve this new type */ + /* Recursively resolve this new type */ if (yang_type_resolve(yorig, ys, rytype, yrestype, options, cvv, patterns, regexps, @@ -1481,9 +1455,10 @@ yang_type_get(yang_stmt *ys, uint8_t *fraction ) { - int retval = -1; - yang_stmt *ytype; /* type */ - char *type = NULL; + int retval = -1; + yang_stmt *ytype; /* type */ + yang_stmt *yorig; + char *type = NULL; if (yrestype == NULL){ clixon_err(OE_YANG, EINVAL, "Expected yrestype != NULL"); @@ -1491,6 +1466,10 @@ yang_type_get(yang_stmt *ys, } if (options) *options = 0x0; + /* Use original tree to resolve types */ + if ((yorig = yang_orig_get(ys)) != NULL) { + ys = yorig; + } /* Find mandatory type */ if ((ytype = yang_find(ys, Y_TYPE, NULL)) == NULL){ clixon_err(OE_DB, ENOENT, "mandatory type object is not found"); @@ -1504,8 +1483,11 @@ yang_type_get(yang_stmt *ys, clixon_err(OE_XML, errno, "stdup"); goto done; } - if (yang_type_resolve(ys, ys, ytype, yrestype, - options, cvv, patterns, regexps, fraction) < 0) + if (yang_type_resolve(ys, ys, ytype, + yrestype, + options, + cvv, patterns, regexps, + fraction) < 0) goto done; if (*yrestype == NULL){ clixon_err(OE_YANG, 0, "result-type should not be NULL");