Autocli optimization feature for generating smaller CLISPECs for large YANGs using treerefs

New `grouping-treeref` option added to clixon-autocli.yang
Default is disabled, set to true to generate smaller memory footprint of clixon_cl
Add prefix "mtpoint:" to cli api-paths to identify change of yang schemas

CHANGELOG.md

@@ -60,6 +60,7 @@ Users may have to change how they access the system
   * rpc `<stats>` is not backward compatible
 * New `clixon-autocli@2023-05-01.yang` revision
   * New `alias` and `skip` extensions
+  * New `grouping-treeref` option
   
 ### C/CLI-API changes on existing features
 Developers may need to change their code
@@ -72,6 +73,9 @@ Developers may need to change their code
 
 ### Minor features
 
+* Autocli optimization feature for generating smaller CLISPECs for large YANGs using treerefs
+   * New `grouping-treeref` option added to clixon-autocli.yang
+   * Default is disabled, set to true to generate smaller memory footprint of clixon_cli
 * Changed YANG uses/grouping to keep uses statement and flag it with YANG_FLAG_USES_EXP
 * Removed  extras/ and build-root/ build code since they are not properly maintained
 * Refactored cli-syntax code to use cligen pt_head instead (long overdue)

apps/cli/cli_auto.c

@@ -135,8 +135,8 @@ cli_auto_edit(clicon_handle h,
     }
     api_path_fmt = cv_string_get(cvec_i(argv, argc++));
     str = cv_string_get(cvec_i(argv, argc++));
-    if (str && str[0] == '/'){ /* ad-hoc to see if 2nd arg is mountpoint */
+    if (str && strncmp(str, "mtpoint:", strlen("mtpoint:")) == 0){
-        mtpoint = str;
+        mtpoint = str + strlen("mtpoint:");
         clicon_debug(1, "%s mtpoint:%s", __FUNCTION__, mtpoint);
         treename = cv_string_get(cvec_i(argv, argc++));
     }

apps/cli/cli_autocli.c

@@ -384,6 +384,49 @@ autocli_completion(clicon_handle h,
     return retval;
 }
 
+/*! Return autocli grouping treeref option
+ *
+ * When false replaces uses with grouping, when true use tree reference
+ * @param[in]  h          Clixon handle
+ * @param[out] treeref    grouping using treerefs enabled
+ * @retval    -1          Error
+ * @retval     0          OK
+ */
+int
+autocli_grouping_treeref(clicon_handle h,
+                         int          *treeref)
+{
+    int     retval = -1;
+    char   *str;
+    uint8_t val;
+    char   *reason = NULL;
+    int     ret;
+    cxobj  *xautocli;
+    
+    if (treeref == NULL){
+        clicon_err(OE_YANG, EINVAL, "Argument is NULL");
+        goto done;
+    }
+    if ((xautocli = clicon_conf_autocli(h)) == NULL){
+        clicon_err(OE_YANG, 0, "No clixon-autocli");
+        goto done;
+    }
+    if ((str = xml_find_body(xautocli, "grouping-treeref")) == NULL){
+        clicon_err(OE_XML, EINVAL, "No grouping-treeref rule");
+        goto done;
+    }
+    if ((ret = parse_bool(str, &val, &reason)) < 0){
+        clicon_err(OE_CFG, errno, "parse_bool");
+        goto done;
+    }
+    *treeref = val;
+    retval = 0; 
+ done:
+    if (reason)
+        free(reason);
+    return retval;
+}
+
 /*! Return default autocli list keyword setting
  *
  * Currently only returns list-keyword-default, could be extended to rules

apps/cli/cli_autocli.h

@@ -52,6 +52,7 @@ enum autocli_op{
  */
 int autocli_module(clicon_handle h, char *modname, int *enable);
 int autocli_completion(clicon_handle h, int *completion);
+int autocli_grouping_treeref(clicon_handle h, int *grouping_treeref);
 int autocli_list_keyword(clicon_handle h, autocli_listkw_t *listkw);
 int autocli_compress(clicon_handle h, yang_stmt *ys, int *compress);
 int autocli_treeref_state(clicon_handle h, int *treeref_state);

apps/cli/cli_common.c

@@ -348,7 +348,10 @@ mtpoint_paths(yang_stmt  *yspec0,
  *
  * @param[in]  h     Clicon handle
  * @param[in]  cvv   Vector of cli string and instantiated variables 
- * @param[in]  argv  Vector: <apipathfmt> [<mointpt>], eg "/aaa/%s"
+ * @param[in]  argv  Arguments given at the callback: 
+ *   <api_path_fmt>  Generated API PATH (this is added implicitly, not actually given in the cvv)
+ *   <api_path_fmt>* Added by merge in treeref_merge_co
+ *   [<mt-point>]    Optional YANG path-arg/xpath from mount-point
  * @param[in]  op    Operation to perform on database
  * @param[in]  nsctx Namespace context for last value added
  * cvv first contains the complete cli string, and then a set of optional
@@ -373,10 +376,10 @@ cli_dbxml(clicon_handle       h,
           cvec               *nsctx)
 {
     int        retval = -1;
+    cbuf      *api_path_fmt_cb = NULL;    /* xml key format */
     char      *api_path_fmt;    /* xml key format */
     char      *api_path_fmt01 = NULL;
     char      *api_path = NULL; 
-    cg_var    *arg;
     cbuf      *cb = NULL;
     cxobj     *xbot = NULL;     /* xpath, NULL if datastore */
     yang_stmt *y = NULL;        /* yang spec of xpath */
@@ -384,24 +387,46 @@ cli_dbxml(clicon_handle       h,
     cxobj     *xerr = NULL;
     int        ret;
     cg_var    *cv;
+    char      *str;
     int        cvvi = 0;
     char      *mtpoint = NULL;
     yang_stmt *yspec0 = NULL;
+    int        argc = 0;
+    int        i;
 
-    if (cvec_len(argv) != 1 && cvec_len(argv) != 2){
-        clicon_err(OE_PLUGIN, EINVAL, "Requires one element to be xml key format string");
-        goto done;
-    }
     /* Top-level yspec */
     if ((yspec0 = clicon_dbspec_yang(h)) == NULL){
         clicon_err(OE_FATAL, 0, "No DB_SPEC");
         goto done;
     }
-    arg = cvec_i(argv, 0);
+    if ((api_path_fmt_cb = cbuf_new()) == NULL){
-    api_path_fmt = cv_string_get(arg);
+        clicon_err(OE_UNIX, errno, "cbuf_new");
-    if (cvec_len(argv) > 1){ 
+        goto done;
-        arg = cvec_i(argv, 1);
+    }
-        mtpoint = cv_string_get(arg);
+    /* Iterate through all api_path_fmt:s, assume they start with / */
+    for (argc=0; argc<cvec_len(argv); argc++){
+        cv = cvec_i(argv, argc);
+        str = cv_string_get(cv);
+        if (str[0] != '/')
+            break;
+    }
+    if (argc == 0){
+        clicon_err(OE_PLUGIN, EINVAL, "No <api_path_fmt> in argv");
+        goto done;
+    }
+    /* Append a api_path_fmt from sub-parts */
+    for (i=argc-1; i>=0; i--){
+        cprintf(api_path_fmt_cb, "%s", cv_string_get(cvec_i(argv, i)));
+    }
+    api_path_fmt = cbuf_get(api_path_fmt_cb);
+    if (cvec_len(argv) > argc){ 
+        cv = cvec_i(argv, argc++);
+        str = cv_string_get(cv);
+        if (strncmp(str, "mtpoint:", strlen("mtpoint:")) != 0){
+            clicon_err(OE_PLUGIN, 0, "mtpoint does not begin with 'mtpoint:'");
+            goto done;
+        }
+        mtpoint = str + strlen("mtpoint:");
     }
     /* Remove all keywords */
     if (cvec_exclude_keys(cvv) < 0)
@@ -483,6 +508,8 @@ cli_dbxml(clicon_handle       h,
         goto done;
     retval = 0;
  done:
+    if (api_path_fmt_cb)
+        cbuf_free(api_path_fmt_cb);
     if (api_path_fmt01)
         free(api_path_fmt01);
     if (xerr)

apps/cli/cli_generate.c

@@ -103,12 +103,6 @@ You can see which CLISPEC it generates via clixon_cli -D 2:
 #include "cli_autocli.h"
 #include "cli_generate.h"
 
-/*
- * Constants
- */
-/* variable expand function */
-#define GENERATE_EXPAND_XMLDB "expand_dbvar"
-
 /*! Create cligen variable expand entry with api-pathfmt format string as argument
  *
  * Add api-fmt as first argument to callback, eg:
@@ -160,8 +154,9 @@ cli_expand_var_generate(clicon_handle h,
     cprintf(cb, " %s(\"candidate\",\"%s\"",
             GENERATE_EXPAND_XMLDB,
             api_path_fmt);
-    if (cv)     /* Add optional mountpoint */
+    if (cv){     /* Add optional mountpoint */
-        cprintf(cb, ",\"%s\"", cv_string_get(cv));
+        cprintf(cb, ",\"%s%s\"", MTPOINT_PREFIX, cv_string_get(cv));
+    }
     cprintf(cb, ")>");
     retval = 0;
  done:
@@ -192,8 +187,9 @@ cli_callback_generate(clicon_handle h,
     if (yang2api_path_fmt(ys, 0, &api_path_fmt) < 0)
         goto done;
     cprintf(cb, ",%s(\"%s\"", GENERATE_CALLBACK, api_path_fmt);
-    if (cv)     /* Add optional mountpoint */
+    if (cv){     /* Add optional mountpoint */
-        cprintf(cb, ",\"%s\"", cv_string_get(cv));
+        cprintf(cb, ",\"%s%s\"", MTPOINT_PREFIX, cv_string_get(cv));
+    }
     cprintf(cb, ")");
     retval = 0;
  done:
@@ -1105,11 +1101,97 @@ yang2cli_choice(clicon_handle h,
     return retval;
 }
 
+/*! Generate clispec for all modules in a grouping
+ *
+ * Called in cli main function for top-level yangs. But may also be called dynamically for
+ * mountpoints.
+ * @param[in]  h         Clixon handle
+ * @param[in]  ys        Top-level Yang statement
+ * @param[in]  treename  Name of tree
+ * @retval     0         OK
+ * @retval    -1         Error
+ * @note Tie-break of same top-level symbol: prefix is NYI
+ * @see yang2cli_yspec  for original
+ */
+static int yang2cli_grouping(clicon_handle h, yang_stmt *ys, char *treename);
+
+/*! Generate CLI code for Yang uses statement
+ *
+ * @param[in]  h     Clixon handle
+ * @param[in]  ys    Yang statement
+ * @param[in]  level Indentation level
+ * @param[out] cb    Buffer where cligen code is written
+ * @retval     0     OK
+ * @retval    -1     Error
+ */
+static int
+yang2cli_uses(clicon_handle h,
+              yang_stmt    *ys,
+              int           level,
+              cbuf         *cb)
+{
+    int        retval = -1;
+    char      *id = NULL;
+    char      *prefix = NULL;
+    char      *ns;
+    yang_stmt *ygrouping;
+    cbuf      *cbtree = NULL;
+    char      *api_path_fmt = NULL;
+    yang_stmt *yp;
+    int        ret;
+
+    if (nodeid_split(yang_argument_get(ys), &prefix, &id) < 0)
+        goto done;
+    if (ys_grouping_resolve(ys, prefix, id, &ygrouping) < 0)
+        goto done;
+    if (ygrouping == NULL){
+        fprintf(stderr, "%s grouping %s not found\n", __FUNCTION__, id);
+        goto ok;
+    }
+    if ((cbtree = cbuf_new()) == NULL){
+        clicon_err(OE_XML, errno, "cbuf_new");
+        goto done;
+    }
+    /* prefix is not globally unique, need namespace */
+    if ((ns = yang_find_mynamespace(ygrouping)) == NULL)
+        goto done;
+    cprintf(cbtree, "grouping-%s-%s", ns, id);
+    if (cligen_ph_find(cli_cligen(h), cbuf_get(cbtree)) == NULL){
+        /* No such tree, generate it */
+        if ((ret = yang2cli_grouping(h, ygrouping, cbuf_get(cbtree))) < 0)
+            goto done;
+        if (ret == 0) /* tree empty */
+            goto ok;
+    }
+    cprintf(cb, "%*s@%s", level*3, "", cbuf_get(cbtree));
+    /* get api-path to parent, do not include "uses" argument since it is replaced */
+    yp = yang_parent_get(ys);
+    if (yang2api_path_fmt(yp, 0, &api_path_fmt) < 0)
+        goto done;
+    /* This adds a "prepend" callback with decendant argument */
+    cprintf(cb, ",%s(\"%s\")", GROUPING_CALLBACK, api_path_fmt);
+    cprintf(cb, ";\n");
+ ok:
+    retval = 0;
+ done:
+    if (api_path_fmt)
+        free(api_path_fmt);
+    if (prefix)
+        free(prefix);
+    if (id)
+        free(id);
+    if (cbtree)
+        cbuf_free(cbtree);
+    return retval;
+}
+
 /*! Generate CLI code for Yang statement
  * @param[in]  h     Clixon handle
  * @param[in]  ys    Yang statement
  * @param[in]  level Indentation level
  * @param[out] cb    Buffer where cligen code is written
+ * @retval     0     OK
+ * @retval    -1     Error
  */
 static int
 yang2cli_stmt(clicon_handle h, 
@@ -1117,9 +1199,10 @@ yang2cli_stmt(clicon_handle h,
               int           level, 
               cbuf         *cb)
 {
-    yang_stmt *yc;
     int        retval = -1;
+    yang_stmt *yc;
     int        treeref_state = 0;
+    int        grouping_treeref;
 
     if (ys == NULL){
         clicon_err(OE_YANG, EINVAL, "No yang spec");
@@ -1135,7 +1218,19 @@ yang2cli_stmt(clicon_handle h,
     /* Only produce autocli for YANG non-config only if autocli-treeref-state is true */
     if (autocli_treeref_state(h, &treeref_state) < 0)
         goto done;
+    if (autocli_grouping_treeref(h, &grouping_treeref) < 0)
+        goto done;
     if (treeref_state || yang_config(ys)){
+        int cornercase = 0;
+        if (grouping_treeref){
+#ifdef AUTOCLI_GROUPING_TOPLEVEL_SKIP
+            cornercase = yang_keyword_get(yang_parent_get(ys)) == Y_MODULE || yang_keyword_get(yang_parent_get(ys)) == Y_SUBMODULE;
+#endif
+            if (yang_keyword_get(ys) != Y_USES && yang_flag_get(ys, YANG_FLAG_GROUPING) 
+                && !cornercase
+                )
+                goto ok;
+        }
         switch (yang_keyword_get(ys)){
         case Y_CONTAINER:
             if (yang2cli_container(h, ys, level, cb) < 0)
@@ -1157,6 +1252,10 @@ yang2cli_stmt(clicon_handle h,
                               cb) < 0)
                 goto done;
             break;
+        case Y_USES:
+            if (grouping_treeref && !cornercase && yang2cli_uses(h, ys, level, cb) < 0)
+                goto done;
+            break;
         case Y_CASE:
         case Y_SUBMODULE:
         case Y_MODULE:
@@ -1340,6 +1439,143 @@ yang2cli_post(clicon_handle h,
     return retval;
 }
 
+/*! Generate clispec for all modules in a grouping
+ *
+ * Called in cli main function for top-level yangs. But may also be called dynamically for
+ * mountpoints.
+ * @param[in]  h         Clixon handle
+ * @param[in]  ys        Top-level Yang statement
+ * @param[in]  treename  Name of tree
+ * @retval     1         OK
+ * @retval     0         OK but empty clispec, no tree produced
+ * @retval    -1         Error
+ * @note Tie-break of same top-level symbol: prefix is NYI
+ * @see yang2cli_yspec and yang2cli_stmt for original
+ * XXX merge with yang2cli_yspec
+ */
+static int
+yang2cli_grouping(clicon_handle      h, 
+                  yang_stmt         *ys,
+                  char              *treename)
+{
+    int             retval = -1;
+    parse_tree     *pt0 = NULL;
+    parse_tree     *pt = NULL;
+    yang_stmt      *yc;
+    pt_head        *ph;
+    cbuf           *cb = NULL;
+    int             treeref_state = 0;
+    char           *prefix;
+    cg_obj         *co;
+    int             config;
+    int             i;
+    
+    if ((pt0 = pt_new()) == NULL){
+        clicon_err(OE_UNIX, errno, "pt_new");
+        goto done;
+    }
+    if ((cb = cbuf_new()) == NULL){
+        clicon_err(OE_XML, errno, "cbuf_new");
+        goto done;
+    }
+    /* Traverse YANG, loop through all modules and generate CLI, inline of yang2cli_stmt */
+    if (yang_find(ys, Y_STATUS, "obsolete") != NULL){
+        clicon_debug(4, "%s obsolete: %s %s, skipped", __FUNCTION__, yang_argument_get(ys), yang_argument_get(ys_module(ys)));
+        goto empty;
+    }
+    if (yang_find(ys, Y_STATUS, "deprecated") != NULL){
+        clicon_debug(4, "%s deprecated: %s %s", __FUNCTION__, yang_argument_get(ys), yang_argument_get(ys_module(ys)));
+    }
+    /* Only produce autocli for YANG non-config only if autocli-treeref-state is true */
+    if (autocli_treeref_state(h, &treeref_state) < 0)
+        goto done;
+    if (treeref_state || yang_config(ys)){
+        yc = NULL;
+        while ((yc = yn_each(ys, yc)) != NULL)
+            if (yang2cli_stmt(h, yc, 1, cb) < 0)
+                goto done;
+    }
+    if (cbuf_len(cb) == 0)
+        goto empty;
+    /* Note Tie-break of same top-level symbol: prefix is NYI
+     * Needs to move cligen_parse_str() call here instead of later
+     */
+    if ((prefix = yang_find_myprefix(ys)) == NULL){
+        clicon_err(OE_YANG, 0, "Module %s lacks prefix", yang_argument_get(ys)); /* shouldnt happen */
+        goto done;
+    }
+    if ((pt = pt_new()) == NULL){
+        clicon_err(OE_UNIX, errno, "pt_new");
+        goto done;
+    }
+    /* Parse the buffer using cligen parser. load cli syntax */
+    if (clispec_parse_str(cli_cligen(h), cbuf_get(cb), (char*)__FUNCTION__, NULL, pt, NULL) < 0){
+        fprintf(stderr, "%s\n", cbuf_get(cb));
+        goto done;
+    }
+    clicon_debug(CLIXON_DBG_DEFAULT, "%s Generated auto-cli for grouping:%s",
+                 __FUNCTION__, yang_argument_get(ys));
+    /* Add prefix: assume new are appended */
+    for (i=0; i<pt_len_get(pt); i++){
+        if ((co = pt_vec_i_get(pt, i)) != NULL){
+            clicon_debug(CLIXON_DBG_DEFAULT, "%s command: %s",
+                         __FUNCTION__, co->co_command);
+            co_prefix_set(co, prefix);
+        }
+    }
+    /* Post-processing, iterate over the generated cligen parse-tree with corresponding yang
+     * Note cannot do it inline in yang2cli above since:
+     * 1. labels cannot be set on "empty"
+     * 2. a; <a>, fn() cannot be set properly
+     */
+    config = 1;
+    if (yang2cli_post(h, NULL, pt, 0, ys, NULL, &config) < 0){
+        goto done;
+    }
+    if (clicon_data_int_get(h, "autocli-print-debug") == 1)
+        clicon_log(LOG_NOTICE, "%s: Top-level cli-spec %s:\n%s",
+                   __FUNCTION__, treename, cbuf_get(cb));
+    else
+        clicon_debug(CLIXON_DBG_DETAIL, "%s: Top-level cli-spec %s:\n%s",
+                     __FUNCTION__, treename, cbuf_get(cb));
+    if (cligen_parsetree_merge(pt0, NULL, pt) < 0){
+        clicon_err(OE_YANG, errno, "cligen_parsetree_merge");
+        goto done;
+    }
+    pt_free(pt, 1);
+    pt = NULL;
+
+    /* Resolve the expand callback functions in the generated syntax.
+     * This "should" only be GENERATE_EXPAND_XMLDB
+     * handle=NULL for global namespace, this means expand callbacks must be in
+     * CLICON namespace, not in a cli frontend plugin.
+     */
+    if (cligen_expandv_str2fn(pt0, (expandv_str2fn_t*)clixon_str2fn, NULL) < 0)
+        goto done;
+    /* Append cligen tree and name it */
+    if ((ph = cligen_ph_add(cli_cligen(h), treename)) == NULL){
+        clicon_err(OE_UNIX, 0, "cligen_ph_add");
+        goto done;
+    }
+    if (cligen_ph_parsetree_set(ph, pt0) < 0){
+        clicon_err(OE_UNIX, 0, "cligen_ph_parsetree_set");
+        goto done;
+    }
+    pt0 = NULL;
+    retval = 1;
+ done:
+    if (pt)
+        pt_free(pt, 1);
+    if (pt0)
+        pt_free(pt0, 1);
+    if (cb)
+        cbuf_free(cb);
+    return retval;
+ empty:
+    retval = 0;
+    goto done;
+}
+
 /*! Generate clispec for all modules in yspec (except excluded)
  * 
  * Called in cli main function for top-level yangs. But may also be called dynamically for
@@ -1402,7 +1638,6 @@ yang2cli_yspec(clicon_handle      h,
             clicon_err(OE_UNIX, errno, "pt_new");
             goto done;
         }
-
         /* Parse the buffer using cligen parser. load cli syntax */
         if (clispec_parse_str(cli_cligen(h), cbuf_get(cb), "yang2cli", NULL, pt, NULL) < 0){
             fprintf(stderr, "%s\n", cbuf_get(cb));

apps/cli/cli_generate.h

@@ -48,6 +48,11 @@
  * where the virtual callback (overwrite_me) is overwritten by cli_set.
  */
 #define GENERATE_CALLBACK "overwrite_me"
+#define GROUPING_CALLBACK "prepend_me"
+#define MTPOINT_PREFIX    "mtpoint:"
+
+/* variable expand function */
+#define GENERATE_EXPAND_XMLDB "expand_dbvar"
 
 /* Name of autocli CLIgen treename
  */

apps/cli/cli_show.c

@@ -176,10 +176,14 @@ xpath_append(cbuf      *cb0,
  * @param[in]   h        clicon handle
  * @param[in]   name     Name of this function (eg "expand_dbvar")
  * @param[in]   cvv      The command so far. Eg: cvec [0]:"a 5 b"; [1]: x=5;
- * @param[in]   argv     Arguments given at the callback: <db> <apipathfmt> [<mointpt>]
+ * @param[in]   argv     Arguments given at the callback:
+ *   <db>            Name of datastore, such as "running"
+ *   <api_path_fmt>  Generated API PATH (this is added implicitly, not actually given in the cvv)
+ *   [<mt-point>]    Optional YANG path-arg/xpath from mount-point
  * @param[out]  commands vector of function pointers to callback functions
  * @param[out]  helptxt  vector of pointers to helptexts
  * @see cli_expand_var_generate  This is where arg is generated
+ * @see cli_expand_var_generate where api_path_fmt + mt-point are generated
  */
 int
 expand_dbvar(void   *h, 
@@ -190,7 +194,7 @@ expand_dbvar(void   *h,
              cvec   *helptexts)
 {
     int              retval = -1;
-    char            *api_path_fmt;
+    char            *api_path_fmt = NULL;
     char            *api_path = NULL;
     char            *api_path_fmt01 = NULL;
     char            *dbstr;
@@ -220,6 +224,9 @@ expand_dbvar(void   *h,
     yang_stmt       *yspec;
     yang_stmt       *yu = NULL;
     cvec            *nsc0 = NULL;
+    char            *str;
+    int              grouping_treeref;
+    cbuf            *cbprepend;
 
     if (argv == NULL || (cvec_len(argv) != 2 && cvec_len(argv) != 3)){
         clicon_err(OE_PLUGIN, EINVAL, "requires arguments: <db> <apipathfmt> [<mountpt>]");
@@ -244,10 +251,33 @@ expand_dbvar(void   *h,
         clicon_err(OE_PLUGIN, 0, "Error when accessing argument <api_path>");
         goto done;
     }
-    api_path_fmt = cv_string_get(cv);
+    if (autocli_grouping_treeref(h, &grouping_treeref) < 0)
+        goto done;
+    if (grouping_treeref &&
+        (cbprepend = cligen_expand_prepend_get(cli_cligen(h))) != NULL){
+        cbuf *cb;
+        if ((cb = cbuf_new()) == NULL){
+            clicon_err(OE_UNIX, errno, "cbuf_new");
+            goto done;
+        }
+        cprintf(cb, "%s%s", cbuf_get(cbprepend), cv_string_get(cv));
+        if ((api_path_fmt = strdup(cbuf_get(cb))) == NULL){
+            clicon_err(OE_UNIX, errno, "strdup");
+            goto done;
+        }
+    }
+    else if ((api_path_fmt = strdup(cv_string_get(cv))) == NULL){
+        clicon_err(OE_UNIX, errno, "strdup");
+        goto done;
+    }
     if (cvec_len(argv) > 2){ 
         cv = cvec_i(argv, 2);
-        mtpoint = cv_string_get(cv);
+        str = cv_string_get(cv);
+        if (strncmp(str, "mtpoint:", strlen("mtpoint:")) != 0){
+            clicon_err(OE_PLUGIN, 0, "mtpoint does not begin with 'mtpoint:'");
+            goto done;
+        }
+        mtpoint = str + strlen("mtpoint:");
     }
     /* api_path_fmt = /interface/%s/address/%s
      * api_path: -->  /interface/eth0/address/.*
@@ -384,6 +414,8 @@ expand_dbvar(void   *h,
   done:
     if (nsc0)
         cvec_free(nsc0);
+    if (api_path_fmt)
+        free(api_path_fmt);
     if (api_path_fmt01)
         free(api_path_fmt01);
     if (cbxpath)
@@ -847,8 +879,8 @@ int cli_show_version(clicon_handle h,
  * @param[in]  h     Clixon handle
  * @param[in]  cvv   Vector of variables from CLIgen command-line
  * @param[in]  argv  String vector of show options, format:
- *   <api_path_fmt>  Generated API PATH (this is added implicitly, not actually given in the cvv)
+ *   <api_path_fmt>  Generated API PATH (this is added implicitly, not actually given in argv)
- *  [<api-path-fmt>] Extra api-path from mount-point
+ *  [<mt-point>]     Optional YANG path-arg/xpath from mount-point
  *   <dbname>        Name of datastore, such as "running"
  * -- from here optional:
  *   <format>        "text"|"xml"|"json"|"cli"|"netconf" (see format_enum), default: xml
@@ -872,6 +904,7 @@ int cli_show_version(clicon_handle h,
  * @see cli_show_auto_mode  autocli with edit menu support
  *
  * XXX merge cli_show_auto and cli_show_auto_mode
+ * @see cli_callback_generate where api_path_fmt + mt-point are generated
  */
 int 
 cli_show_auto(clicon_handle h,
@@ -903,8 +936,8 @@ cli_show_auto(clicon_handle h,
     }
     api_path_fmt = cv_string_get(cvec_i(argv, argc++));
     str = cv_string_get(cvec_i(argv, argc++));
-    if (str && str[0] == '/'){ /* ad-hoc to see if 2nd arg is mountpoint */
+    if (str && strncmp(str, "mtpoint:", strlen("mtpoint:")) == 0){
-        mtpoint = str;
+        mtpoint = str + strlen("mtpoint:");
         dbname = cv_string_get(cvec_i(argv, argc++));
     }
     else

include/clixon_custom.h

@@ -156,3 +156,17 @@
  * Consider moving to configure.ac(compile-time) or to clixon-config.yang(run-time)
  */
 #define PRETTYPRINT_INDENT 3
+
+/*! Autocli uses/grouping references for top-level
+ * Exception of expand-grouping in clixon-autocli.yang
+ * If enabled do not expand-grouping if a yang uses is directly under module or submodule
+ * Disabled does not work today and is temporary and for documentation
+ */
+#define AUTOCLI_GROUPING_TOPLEVEL_SKIP
+
+/*! Autocli uses/grouping references for augment/uses
+ * Exception of expand-grouping in clixon-autocli.yang
+ * If enabled do not expand-grouping if a yang uses is directly under augment
+ * Disabled does not work today and is temporary and for documentation
+ */
+#define AUTOCLI_GROUPING_AUGMENT_SKIP

lib/clixon/clixon_yang_parse_lib.h

@@ -51,6 +51,7 @@
 /*
  * Prototypes
  */
+int        ys_grouping_resolve(yang_stmt *yuses, char *prefix, char *name, yang_stmt **ygrouping0);
 yang_stmt *yang_parse_file(FILE *fp, const char *name, yang_stmt *ysp);
 int        yang_file_find_match(clicon_handle h, const char *module, const char *revision, cbuf *fbuf);
 yang_stmt *yang_parse_filename(clicon_handle h, const char *filename, yang_stmt  *ysp);

lib/src/clixon_path.c

@@ -290,8 +290,8 @@ xml_yang_root(cxobj  *x,
  * @param[in]  ys      Yang statement
  * @param[in]  inclkey If set include key leaf (eg last leaf d in ex)
  * @param[out] cb      api_path_fmt,
- * @retval     0    OK
+ * @retval     0       OK
- * @retval    -1    Error
+ * @retval    -1       Error
  * @see RFC8040 3.5.3 where "api-path" is defined as "URI-encoded path expression"
  */ 
 static int
@@ -305,18 +305,26 @@ yang2api_path_fmt_1(yang_stmt *ys,
     int        i;
     cvec      *cvk = NULL; /* vector of index keys */
     int        retval = -1;
+    enum rfc_6020 ypkeyw = -1;
     
     if ((yp = yang_parent_get(ys)) == NULL){
         clicon_err(OE_YANG, EINVAL, "yang expected parent %s", yang_argument_get(ys));
         goto done;
     }
-    if (yp != NULL && /* XXX rm */
+    if (yp)
-        yang_keyword_get(yp) != Y_MODULE && 
+        ypkeyw = yang_keyword_get(yp);
-        yang_keyword_get(yp) != Y_SUBMODULE){
+    switch (ypkeyw){
-
+    case Y_MODULE:
+    case Y_SUBMODULE:
+        cprintf(cb, "/%s:", yang_argument_get(yp));
+        break;
+    case Y_GROUPING: /* mainly for expand-grouping in clixon-autocli.yang */
+        cprintf(cb, "/");
+        break;
+    default:
         if (yang2api_path_fmt_1((yang_stmt *)yp, 1, cb) < 0) /* recursive call */
             goto done;
-        if (yang_keyword_get(yp) != Y_CHOICE && yang_keyword_get(yp) != Y_CASE){
+        if (ypkeyw != Y_CHOICE && ypkeyw != Y_CASE){
 #if 0
             /* In some cases, such as cli_show_auto, a trailing '/' should
              * NOT be present if ys is a key in a list.
@@ -324,7 +332,7 @@ yang2api_path_fmt_1(yang_stmt *ys,
              * so a patch is added in cli_show_auto instead.
              */
                 if (yang_keyword_get(ys) == Y_LEAF && yp && 
-                    yang_keyword_get(yp) == Y_LIST &&
+                    ypkeyw == Y_LIST &&
                     yang_key_match(yp, ys->ys_argument, NULL) == 1)
                 ;
             else
@@ -338,17 +346,15 @@ yang2api_path_fmt_1(yang_stmt *ys,
             goto done;
         if (ypmod != ymod)
             cprintf(cb, "%s:", yang_argument_get(ymod));
+        break;
     }
-    else /* top symbol - mark with name prefix */
-        cprintf(cb, "/%s:", yang_argument_get(yp));
-
     if (inclkey){
         if (yang_keyword_get(ys) != Y_CHOICE && yang_keyword_get(ys) != Y_CASE)
             cprintf(cb, "%s", yang_argument_get(ys));
     }
     else{
         if (yang_keyword_get(ys) == Y_LEAF && yp && 
-            yang_keyword_get(yp) == Y_LIST){
+            ypkeyw == Y_LIST){
             if (yang_key_match(yp, yang_argument_get(ys), NULL) == 0)
                 cprintf(cb, "%s", yang_argument_get(ys)); /* Not if leaf and key */
         }
@@ -381,6 +387,7 @@ yang2api_path_fmt_1(yang_stmt *ys,
 }
 
 /*! Construct an api_path_format from yang statement using wildcards for keys
+ *
  * Recursively construct it to the top.
  * Example: 
  *   yang:  container a -> list b -> key c -> leaf d
@@ -389,7 +396,7 @@ yang2api_path_fmt_1(yang_stmt *ys,
  * @param[in]  inclkey      If set include key leaf (eg last leaf d in ex)
  * @param[out] api_path_fmt XML api path. Needs to be freed after use.
  * @retval     0            OK
- * @retval     -1           Error
+ * @retval    -1            Error
  * "api-path" is "URI-encoded path expression" definition in RFC8040 3.5.3
  */ 
 int

lib/src/clixon_yang_parse_lib.c

@@ -156,7 +156,7 @@ ys_grouping_module_resolve(yang_stmt *ymod,
  * @retval     0          OK, ygrouping may be NULL
  * @retval    -1          Error, with clicon_err called
  */
-static int
+int
 ys_grouping_resolve(yang_stmt  *yuses, 
                     char       *prefix, 
                     char       *name,
@@ -235,7 +235,6 @@ yang_augment_node(clicon_handle h,
         clicon_err(OE_YANG, 0, "My yang module not found");
         goto done;
     }
-    /* */
     schema_nodeid = yang_argument_get(ys);
     clicon_debug(CLIXON_DBG_DETAIL, "%s %s", __FUNCTION__, schema_nodeid);
     /* Find the target */
@@ -279,7 +278,11 @@ yang_augment_node(clicon_handle h,
     while ((yc0 = yn_each(ys, yc0)) != NULL) {
         childkey = yang_keyword_get(yc0);
         /* Only shemanodes and extensions */
-        if (!yang_schemanode(yc0) && childkey != Y_UNKNOWN) 
+        if (!yang_schemanode(yc0) && childkey != Y_UNKNOWN
+#ifndef AUTOCLI_GROUPING_AUGMENT_SKIP
+            && childkey != Y_USES
+#endif
+            )
             continue;
         switch (targetkey){
         case Y_CONTAINER:
@@ -344,9 +347,12 @@ yang_augment_node(clicon_handle h,
         default:
             break;
         }
-
         if ((yc = ys_dup(yc0)) == NULL)
             goto done;
+#ifdef AUTOCLI_GROUPING_AUGMENT_SKIP
+        /* cornercase: always expand uses under augment */
+        yang_flag_reset(yc, YANG_FLAG_GROUPING);
+#endif
         yc->ys_mymodule = ymod;
 
         if (yn_insert(ytarget, yc) < 0)
@@ -679,6 +685,7 @@ yang_expand_uses_node(yang_stmt *yn,
             yang_flag_set(yg, YANG_FLAG_NOKEY);
         yn->ys_stmt[i+k+1] = yg;
         yg->ys_parent = yn;
+        yang_flag_set(yg, YANG_FLAG_GROUPING);
         k++;
     }
     /* Remove the grouping copy */

test/test_autocli_grouping.sh

@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+# Tests for saving memory by not expanding grouping/uses in the autocli
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+cfg=$dir/conf_yang.xml
+fyang=$dir/example.yang
+fyang2=$dir/example-external.yang
+fyang3=$dir/example-external3.yang
+
+# Whether grouping treeref is enabled
+grouping_treeref=true
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
+  <CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_YANG_LIBRARY>false</CLICON_YANG_LIBRARY>
+  <autocli>
+    <module-default>false</module-default>
+     <list-keyword-default>kw-nokey</list-keyword-default>
+     <grouping-treeref>${grouping_treeref}</grouping-treeref>
+     <rule>
+        <name>include ${APPNAME}</name>
+        <operation>enable</operation>
+        <module-name>${APPNAME}*</module-name>
+     </rule>
+  </autocli>
+</clixon-config>
+EOF
+
+cat <<EOF > $dir/example.cli
+CLICON_MODE="example";
+CLICON_PROMPT="%U@%H %W> ";
+
+# Autocli syntax tree operations
+edit @datamodel, cli_auto_edit("datamodel");
+up, cli_auto_up("datamodel");
+top, cli_auto_top("datamodel");
+set @datamodel, cli_auto_set();
+merge @datamodel, cli_auto_merge();
+create @datamodel, cli_auto_create();
+commit("Commit the changes"), cli_commit();
+delete("Delete a configuration item") {
+      @datamodel, cli_auto_del(); 
+      all("Delete whole candidate configuration"), delete_all("candidate");
+}
+show("Show a particular state of the system"){
+    configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", false, false);
+}
+EOF
+
+# Yang specs must be here first for backend. But then the specs are changed but just for CLI
+# Annotate original Yang spec example  directly
+# First annotate /table/parameter 
+# Had a problem with unknown in grouping -> test uses uses/grouping
+cat <<EOF > $fyang
+module example {
+  namespace "urn:example:clixon";
+  prefix ex;
+  import clixon-autocli{
+      prefix autocli;
+  }
+  import example-external{
+      prefix ext;
+  }
+  grouping pg1 {
+     leaf value1{
+        description "a value";
+        type string;
+     }
+     list index1{
+       key i;
+       leaf i{
+         type string;
+       }
+       leaf iv{
+         description "a value";
+         type string;
+       }
+     }
+  }
+  container table{
+    list parameter{
+      key name;
+      leaf name{
+        type string;
+      }
+      leaf value0{
+        description "a value";
+        type string;
+      }
+      uses pg1;
+      uses ext:pg2;
+    }
+  }
+  uses pg1;
+}
+EOF
+
+# Original no annotations for backend
+cat <<EOF > $fyang2
+module example-external {
+   namespace "urn:example:external";
+   prefix ext;
+   import example-external3{
+      prefix ext3;
+   }
+   grouping pg2 {
+      leaf value2{
+        description "a value";
+        type string;
+      }
+      container c2{
+         uses ext3:pg3;
+      }
+   }
+}
+EOF
+
+# Original no annotations for backend
+cat <<EOF > $fyang3
+module example-external3 {
+   namespace "urn:example:external";
+   prefix ext3;
+   grouping pg3 {
+      leaf value3{
+        description "a value";
+        type string;
+      }
+   }
+}
+EOF
+
+new "test params: -f $cfg"
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -z -f $cfg
+    if [ $? -ne 0 ]; then
+        err
+    fi
+    new "start backend -s init -f $cfg"
+    start_backend -s init -f $cfg
+fi
+
+new "wait backend"
+wait_backend
+
+# Test of generated clispecs. With autocli grouping_treeref, there should be treerefs for
+# groupings. Without they should not be present
+# The testcase assumes enabled
+if ${grouping_treeref}; then
+    new "verify grouping is enabled"
+    expectpart "$($clixon_cli -f $cfg -G -1 2>&1)" 0 "@grouping-urn:example:clixon-pg1" "@grouping-urn:example:external-pg2" "@grouping-urn:example:external-pg3"
+else
+    new "verify grouping is disabled"
+    expectpart "$($clixon_cli -f $cfg -G -1 2>&1)" 0 --not-- "@grouping-urn:example:clixon-pg1" "@grouping-urn:example:external-pg2" "@grouping-urn:example:external-pg3"
+fi
+
+new "set top-level grouping"
+expectpart "$($clixon_cli -f $cfg -1 set value1 39)" 0 ""
+
+new "set inline grouping value"
+expectpart "$($clixon_cli -f $cfg -1 set table parameter x value0 40)" 0 ""
+
+new "set grouping in same module"
+expectpart "$($clixon_cli -f $cfg -1 set table parameter x value1 41)" 0 ""
+
+new "set list grouping in same module"
+expectpart "$($clixon_cli -f $cfg -1 set table parameter x index1 a iv foo)" 0 ""
+
+new "set grouping in other module"
+expectpart "$($clixon_cli -f $cfg -1 set table parameter x value2 42)" 0 ""
+
+new "set grouping in other+other module"
+expectpart "$($clixon_cli -f $cfg -1 set table parameter x c2 value3 43)" 0 ""
+
+new "commit"
+expectpart "$($clixon_cli -f $cfg -1 commit)" 0 ""
+
+new "show config"
+expectpart "$($clixon_cli -f $cfg -1 show config)" 0 "<table xmlns=\"urn:example:clixon\"><parameter><name>x</name><value0>40</value0><value1>41</value1><index1><i>a</i><iv>foo</iv></index1><value2>42</value2><c2><value3>43</value3></c2></parameter></table>" "<value1 xmlns=\"urn:example:clixon\">39</value1>"
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u root -f clixon_backend)
+    if [ -z "$pid" ]; then
+        err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+rm -rf $dir
+
+new "endtest"
+endtest

yang/clixon/clixon-autocli@2023-05-01.yang

@@ -44,6 +44,7 @@ module clixon-autocli{
     revision 2023-05-01 {
         description
             "Added extensions skip and alias
+             Added grouping-treeref
              Released in Clixon 6.3";
     }
     revision 2022-02-11 {
@@ -199,6 +200,21 @@ module clixon-autocli{
             type boolean;
             default true;
         }
+        leaf grouping-treeref {
+            description
+                "Controls the behaviour when generating CLISPEC of YANG 'uses' statements into the
+                 corresponding 'grouping' definition: macro expansion.
+                 For optimization of memory footprint.
+                 If 'false', replace the uses definition with the grouping definition.
+                 If 'true' use indirect tree reference '@treeref' to reference the grouping definition. This
+                 saves memory for large YANGs.
+
+                 See also AUTOCLI_GROUPING_TOPLEVEL_SKIP and AUTOCLI_GROUPING_AUGMENT_SKIP for 
+                 temporary disabled cornercases.
+                 This option was introduced in Clixon 6.3";
+            type boolean;
+            default false;
+        }
         /* rules */
         list rule {
             description