* Renamed clixon-clispec.yang to clixon-autocli.yang

* First version of clixon-autocli.yang semantics
   * Default rules for module exclusion, list-keywords, completion, treeref-state
   * Specialized rules for compress and exclusion of modules
   * See [autocli documentation](https://clixon-docs.readthedocs.io/en/latest/cli.html#autocli)
* Obsoleted and moved autocli config options from clixon-config.yang to clixon-autocli.yang as follows:
   * `CLICON_CLI_GENMODEL`, use: `autocli/module-default=false` instead
      * Removed `clicon_cli_genmodel()`
   * `CLICON_CLI_GENMODEL_TYPE`, use `autocli/list-keyword-default` and compress rules instead)
      * Removed `clicon_cli_genmodel_type()`
   * `CLICON_CLI_GENMODEL_COMPLETION`, use `autocli/completion-default` instead
      * Removed `clicon_cli_genmodel_completion()`
   * `CLICON_CLI_AUTOCLI_EXCLUDE`, use `autocli/rule/operation=exclude` instead
   * `CLICON_CLI_MODEL_TREENAME`, use constant `AUTOCLI_TREENAME` instead
     * Removed `clicon_cli_model_treename()`
* New YANG functions: yang_single_child_type, yang_find_namespace_by_prefix, yang_str2key
* Changed return values of yang_find_prefix_by_namespace
* Merged `cli_cli2xml()` into `cli2xml()`

.gitignore

@@ -74,4 +74,6 @@ test/vagrant/site.mk
 test/cicd/site.mk
 doc/html
 
+test/fuzz/*/conf.xml
+
 .idea/

CHANGELOG.md

@@ -38,6 +38,10 @@ Planned: January, 2022
 ### New features
 
 * Changed auto-cli design
+  * See [autocli documentation](https://clixon-docs.readthedocs.io/en/latest/cli.html#autocli)
+  * Added new YANG clixon-autocli.yang, placing autocli options there
+    * Default rules for module exclusion, list-keywords, completion, treeref-state
+    * Specialized rules for module exclusion and compression
   * Replaced separate autocli trees with a single `@basemodel` tree by using filter labels
     * Filter labels are added to the fill tree and then filtered out using `@remove:<label>`
     * Labels include: termfirstkeys, termlist, termleaf, leafvar, nonconfig,
@@ -48,10 +52,8 @@ Planned: January, 2022
       * `@datamodel` translated to `@basemodel, @remove:termfirstkeys, @remove:termlist, @remove:termleaf, @remove:nonconfig`
       * `@datamodelshow` translated to `@basemodel, @remove:leafvar, @remove:nonconfig`
       * `@datamodelstate` translated to `@basemodel, @remove:leafvar`
-    * Note: autocli mode support is not backward compatible
+    * Note: while @datamodel etc are backward compatible, the autocli redesign is NOT backward compatible
-      * see main example
+      * see API changes
-  * Added new YANG clixon-clispec.yang
-    * This yang replaces many autocli options
 
 ### API changes on existing protocol/config features
 
@@ -69,6 +71,9 @@ Users may have to change how they access the system
     * Instead if you use fgci/nginx:
       * Use `restconf/fcgi-socket`
       * Ensure `<CLICON_FEATURE>clixon-restconf:fcgi</CLICON_FEATURE>` is set
+  * Marked as obsolete and moved autocli config options from clixon-config.yang to clixon-autocli.yang
+    * Use: `<config><autocli>...` for configuring the autocli
+    * For details, see [autocli upgrade documentation](https://clixon-docs.readthedocs.io/en/latest/cli.html#upgrade-from-clixon-5-4)
 
 ### Minor features
 

apps/cli/Makefile.in

@@ -100,6 +100,7 @@ LIBSRC	       += cli_handle.c
 LIBSRC	       += cli_plugin.c
 LIBSRC	       += cli_auto.c
 LIBSRC	       += cli_generate.c
+LIBSRC	       += cli_autocli.c
 LIBOBJ		= $(LIBSRC:.c=.o)
 
 # Name of lib
@@ -151,7 +152,7 @@ install-lib: $(MYLIBSTATIC)
 	install -d -m 0755 $(DESTDIR)$(libdir)/clixon/plugins/cli
 endif
 
-install-include:	clixon_cli.h clixon_cli_api.h cli_generate.h
+install-include:	clixon_cli.h clixon_cli_api.h cli_generate.h cli_autocli.h
 	install -d -m 0755 $(DESTDIR)$(includedir)/clixon
 	install -m 0644 $^ $(DESTDIR)$(includedir)/clixon
 

apps/cli/cli_auto.c

@@ -72,6 +72,7 @@
 #include <clixon/clixon.h>
 
 #include "clixon_cli_api.h"
+#include "cli_autocli.h"
 #include "cli_common.h"
 
 /*
@@ -315,122 +316,6 @@ cli_xml2txt(cxobj            *xn,
     return retval;
 }
 
-/*! Translate from XML to CLI commands
- * Howto: join strings and pass them down. 
- * Identify unique/index keywords for correct set syntax.
- * @param[in] xn        XML Parse-tree (to translate)
- * @param[in] prepend  Print this text in front of all commands.
- * @param[in] gt       option to steer cli syntax
- * @param[in] fn       Callback to make print function
- * @see xml2cli
- */
-int
-cli_xml2cli(cxobj             *xn,
-	    char              *prepend,
-	    enum genmodel_type gt,
-	    clicon_output_cb  *fn)
-{
-    int              retval = -1;
-    cxobj           *xe = NULL;
-    cbuf            *cbpre = NULL;
-    yang_stmt       *ys;
-    int              match;
-    char            *body;
-    char         	*opext = NULL;
-
-    if (xml_type(xn)==CX_ATTR)
-	goto ok;
-    if ((ys = xml_spec(xn)) == NULL)
-	goto ok;
-    /* Look for autocli-op defined in clixon-lib.yang */
-    if (yang_extension_value(xml_spec(xn), "autocli-op", CLIXON_LIB_NS, NULL, &opext) < 0) {
-	goto ok;
-    }
-    if ((opext != NULL) && ((strcmp(opext, "hide-database") == 0) || (strcmp(opext, "hide-database-auto-completion") == 0))){
-	goto ok;
-    }
-    /* If leaf/leaf-list or presence container, then print line */
-    if (yang_keyword_get(ys) == Y_LEAF ||
-	yang_keyword_get(ys) == Y_LEAF_LIST){
-	if (prepend)
-	    (*fn)(stdout, "%s", prepend);
-	if (gt == GT_ALL || gt == GT_VARS || gt == GT_HIDE || gt == GT_OC_COMPRESS)
-	    (*fn)(stdout, "%s ", xml_name(xn));
-	if ((body = xml_body(xn)) != NULL){
-	    if (index(body, ' '))
-		(*fn)(stdout, "\"%s\"", body);
-	    else
-		(*fn)(stdout, "%s", body);
-	}
-	(*fn)(stdout, "\n");
-	goto ok;
-    }
-    /* Create prepend variable string */
-    if ((cbpre = cbuf_new()) == NULL){
-	clicon_err(OE_PLUGIN, errno, "cbuf_new");	
-	goto done;
-    }
-    if (prepend)
-	cprintf(cbpre, "%s", prepend);
-
-    /* If non-presence container && HIDE mode && only child is 
-     * a list, then skip container keyword
-     * See also yang2cli_container */
-    if (yang_container_cli_hide(ys, gt) == 0)
-	cprintf(cbpre, "%s ", xml_name(xn));
-
-    /* If list then first loop through keys */
-    if (yang_keyword_get(ys) == Y_LIST){
-	xe = NULL;
-	while ((xe = xml_child_each(xn, xe, -1)) != NULL){
-	    if ((match = yang_key_match(ys, xml_name(xe), NULL)) < 0)
-		goto done;
-	    if (!match)
-		continue;
-	    if (gt == GT_ALL)
-		cprintf(cbpre, "%s ", xml_name(xe));
-	    cprintf(cbpre, "%s ", xml_body(xe));
-	}
-    }
-    else if ((yang_keyword_get(ys) == Y_CONTAINER) &&
-	     yang_find(ys, Y_PRESENCE, NULL) != NULL){
-	/* If presence container, then print as leaf (but continue to children) */
-	if (prepend)
-	    (*fn)(stdout, "%s", prepend);
-	if (gt == GT_ALL || gt == GT_VARS || gt == GT_HIDE)
-	    (*fn)(stdout, "%s ", xml_name(xn));
-	if ((body = xml_body(xn)) != NULL){
-	    if (index(body, ' '))
-		(*fn)(stdout, "\"%s\"", body);
-	    else
-		(*fn)(stdout, "%s", body);
-	}
-	(*fn)(stdout, "\n");
-    }
-
-    /* For lists, print cbpre before its elements */
-    if (yang_keyword_get(ys) == Y_LIST)
-	(*fn)(stdout, "%s\n", cbuf_get(cbpre));	
-    /* Then loop through all other (non-keys) */
-    xe = NULL;
-    while ((xe = xml_child_each(xn, xe, -1)) != NULL){
-	if (yang_keyword_get(ys) == Y_LIST){
-	    if ((match = yang_key_match(ys, xml_name(xe), NULL)) < 0)
-		goto done;
-	    if (match)
-		continue; /* Not key itself */
-	}
-	if (cli_xml2cli(xe, cbuf_get(cbpre), gt, fn) < 0)
-	    goto done;
-    }
- ok:
-    retval = 0;
- done:
-    if (cbpre)
-	cbuf_free(cbpre);
-    return retval;
-}
-
 /*! Enter a CLI edit mode
  * @param[in]  h    CLICON handle
  * @param[in]  cvv  Vector of variables from CLIgen command-line
@@ -645,7 +530,6 @@ cli_auto_show(clicon_handle h,
     char       *api_path = NULL;
     char       *formatstr;
     enum format_enum format;
-    enum genmodel_type gt;
     pt_head    *ph;
     char       *xpath = NULL;
     cxobj      *xp;
@@ -763,13 +647,11 @@ cli_auto_show(clicon_handle h,
 		    cli_xml2txt(xc, cligen_output, 0); /* tree-formed text */
 	    break;
 	case FORMAT_CLI:
-	    if ((gt = clicon_cli_genmodel_type(h)) == GT_ERR)
-		goto done;
 	    if (isroot)
-		cli_xml2cli(xp, prefix, gt, cligen_output); /* cli syntax */
+		xml2cli(h, stdout, xp, prefix, cligen_output);
 	    else
 		while ((xc = xml_child_each(xp, xc, CX_ELMNT)) != NULL)
-		    cli_xml2cli(xc, prefix, gt, cligen_output); /* cli syntax */
+		    xml2cli(h, stdout, xc, prefix, cligen_output);
 	    break;
 	case FORMAT_NETCONF:
 	    fprintf(stdout, "<rpc xmlns=\"%s\" %s><edit-config><target><candidate/></target><config>",

apps/cli/cli_autocli.c

@@ -0,0 +1,462 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+  *
+  * C-code corresponding to clixon-autocli.yang
+ */
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <syslog.h>
+#include <signal.h>
+#include <fnmatch.h>
+#include <sys/param.h>
+
+/* cligen */
+#include <cligen/cligen.h>
+
+/* libclixon */
+#include <clixon/clixon.h>
+
+#include "clixon_cli_api.h"
+#include "cli_autocli.h"
+
+/* Mapping from YANG autocli-op to C enum */
+static const map_str2int autocli_op_map[] = {
+    {"enable",   AUTOCLI_OP_ENABLE},
+    {"compress", AUTOCLI_OP_COMPRESS},
+    {NULL,       -1}
+};
+
+/* Mapping from YANG list-keyword-type to C enum */
+static const map_str2int list_kw_map[] = {
+    {"kw-none",  AUTOCLI_LISTKW_NONE},
+    {"kw-nokey", AUTOCLI_LISTKW_NOKEY},
+    {"kw-all",   AUTOCLI_LISTKW_ALL},
+    {NULL,       -1}
+};
+
+
+static int
+autocli_str2op(char *str)
+{
+    return clicon_str2int(autocli_op_map, str);
+}
+
+#ifdef NOTUSED
+static const char *
+autocli_op2str(int op)
+{
+    return clicon_int2str(autocli_op_map, op);
+}
+#endif
+
+static int
+autocli_listkw_str2int(char *str)
+{
+    return clicon_str2int(list_kw_map, str);
+}
+
+#ifdef NOTUSED
+static const char *
+autocli_listkw_int2str(int listkw)
+{
+    return clicon_int2str(list_kw_map, listkw);
+}
+#endif
+
+/*! Filter module name according to cli_autocli.yang setting
+ *
+ * Traverse and find module/include/exclude rules
+ * @param[in]  h        Clixon handle
+ * @param[in]  modname  Name of YANG module, or NULL for ANY module (eg default)
+ * @param[out] enablep  Include this module in autocli
+ * @retval    -1        Error
+ * @retval     0        OK, and enablep set
+ */
+int
+autocli_module(clicon_handle    h,
+	       char            *modname,
+	       int             *enablep)
+{
+    int    retval = -1;
+    cxobj *xrule;
+    cxobj *xmod;
+    char  *str;
+    char  *element;
+    int    enable = 0;
+    cxobj *xautocli;
+    char  *body;
+
+    if (enablep == NULL){
+	clicon_err(OE_YANG, EINVAL, "Argument is NULL");
+	goto done;
+    }
+    enable = 0;
+    if ((xautocli = clicon_conf_autocli(h)) == NULL)
+	goto ok;
+    /* Default rule */
+    if ((str = xml_find_body(xautocli, "module-default")) == NULL){
+	clicon_err(OE_XML, EINVAL, "No module-default rule");
+	goto done;
+    }
+    enable = strcmp(str, "true") == 0;
+    if (!enable){
+	xrule = NULL;
+	while ((xrule = xml_child_each(xautocli, xrule, CX_ELMNT)) != NULL) {
+	    if (strcmp(xml_name(xrule), "rule") != 0)
+		continue;
+	    if ((str = xml_find_body(xrule, "operation")) == NULL)
+		continue;
+	    /* Peek in element of rule to skip non-compress operations */
+	    if (autocli_str2op(str) != AUTOCLI_OP_ENABLE)
+		continue;
+	    /* At this point this rule is a compress rule
+	     * enable rules logic is:
+	     * - If match, break, done
+	     */
+	    xmod = NULL;
+	    while ((xmod = xml_child_each(xrule, xmod, CX_ELMNT)) != NULL) {
+		if ((element = xml_name(xmod)) == NULL)
+		    continue;
+		if (strcmp(element, "module-name") == 0){
+		    if (modname == NULL)
+			break; /* match */
+		    if ((body = xml_body(xmod)) == NULL)
+			continue; /* invalid rule? */
+		    if (fnmatch(body, modname, 0) == 0)
+			break; /* match */
+		}
+	    }
+	    if (xmod != NULL){ /* break: found match */
+		enable = 1;
+		break;
+	    }
+	}
+    }
+ ok:
+    *enablep = enable;
+    retval = 0; 
+ done:
+    return retval;
+}
+
+/*! Specialization of autocli_compress for extension element
+ */
+static int
+autocli_compress_extension(yang_stmt *ys,
+			   yang_stmt *ymod,
+			   char      *body,
+			   int       *match)
+{
+    int   retval = -1;
+    char *prefix = NULL;
+    char *id = NULL;
+    char *ns = NULL;
+    int   exist = 0;
+    int   ret;
+		    
+    if (nodeid_split(body, &prefix, &id) < 0)
+	goto done;
+    if (prefix != NULL){
+	if ((ret = yang_find_namespace_by_prefix(ys, prefix, &ns)) < 0)
+	    goto done;
+	if (ret == 1){
+	    /* First try local node, then try module */
+	    if (yang_extension_value(ys, id, ns, &exist, NULL) < 0)
+		goto done;
+	    if (exist == 0)
+		if (yang_extension_value(ymod, id, ns, &exist, NULL) < 0)
+		    goto done;
+	    if (exist == 0)
+		*match = 0;
+	}
+    }
+    retval = 0;
+ done:
+    if (prefix)
+	free(prefix);
+    if (id)
+	free(id);
+    return retval;
+}
+
+/*! Check if yang symbol shpuld be compressed, ie skipped for its child(ren)
+ *
+ * For each Traverse "compress" rule, check:
+ * 1. Yang keyword match
+ * 2. Schema-nodeid match (yang argument)
+ * 3. module-name match
+ * 4. Single list child?
+ * The rules are OR:d, which means:
+ * - At least one compress rules that match
+ *
+ * @param[in]  h        Clixon handle
+ * @param[out] compress 
+ * @retval    -1        Error
+ * @retval     0        OK, and compress set
+
+ * Canonical examples:
+The config and state containers are "compressed" out of the schema.
+    + op=COMPRESS
+    - node-id is config/state
+    + yang keyword is CONTAINER
+    + module-name=openconfig*
+
+The surrounding container entities are removed from list nodes.
+    + op=COMPRESS
+    + yang keyword is CONTAINER
+    - Only one child + Child keyword is LIST
+ */
+int
+autocli_compress(clicon_handle h,
+		 yang_stmt    *ys,
+		 int          *compress)
+{
+    int        retval = -1;
+    cxobj     *xautocli = NULL;
+    cxobj     *xrule;
+    cxobj     *xmod;
+    yang_stmt *ymod;
+    char      *modname;
+    char      *str;
+    char      *element;
+    char      *nodeid;
+    enum rfc_6020 keyw;
+    char      *keywstr;
+    int        match = 0;
+    char      *body;
+    
+    if (compress == NULL){
+	clicon_err(OE_YANG, EINVAL, "Argument is NULL");
+	goto done;
+    }
+    if ((xautocli = clicon_conf_autocli(h)) == NULL){
+	clicon_err(OE_YANG, 0, "No clixon-autocli");
+	goto done;
+    }
+    ymod = ys_module(ys);
+    modname = yang_argument_get(ymod);
+    keyw = yang_keyword_get(ys);
+    keywstr = yang_key2str(keyw);
+    nodeid = yang_argument_get(ys);
+    xrule = NULL;
+    while ((xrule = xml_child_each(xautocli, xrule, CX_ELMNT)) != NULL) {
+	if (strcmp(xml_name(xrule), "rule") != 0)
+	    continue;
+	if ((str = xml_find_body(xrule, "operation")) == NULL)
+	    continue;
+	/* Peek in element of rule to skip non-compress operations */
+	if (autocli_str2op(str) != AUTOCLI_OP_COMPRESS)
+	    continue;
+	/* At this point this rule is a compress rule
+	 * compress rule logic is "OR", the logic is:
+	 * - If match, break, done
+	 * - If not match, continue to next rule
+	 */
+	match = 1;
+	xmod = NULL;
+	while ((xmod = xml_child_each(xrule, xmod, CX_ELMNT)) != NULL) {
+	    if ((element = xml_name(xmod)) == NULL)
+		continue;
+	    if (strcmp(element, "name") == 0 ||
+		strcmp(element, "operation") == 0)
+		continue;
+	    if ((body = xml_body(xmod)) == NULL)
+		continue;
+	    if (strcmp(element, "yang-keyword") == 0){
+		if (strcmp(body, keywstr) != 0){
+		    match = 0;
+		    break;
+		}
+	    }
+	    else if (strcmp(element, "schema-nodeid") == 0){
+		if (strcmp(body, nodeid) != 0){
+		    match = 0;
+		    break;
+		}
+	    }
+	    else if (strcmp(element, "module-name") == 0){
+		if (fnmatch(body, modname, 0) != 0){
+		    match = 0;
+		    break;
+		}
+	    }
+	    else if (strcmp(element, "extension") == 0){
+		if (autocli_compress_extension(ys, ymod, body, &match) < 0)
+		    goto done;
+		if (match == 0)
+		    break;
+	    }
+	    else if (strcmp(element, "yang-keyword-child") == 0){
+		enum rfc_6020 subkeyw;
+		subkeyw = yang_str2key(body);
+		if (yang_single_child_type(ys, subkeyw) == 0){
+		    match = 0;
+		    break;
+		}
+	    }
+	}
+	if (match) /* At least one compress rule matches */
+	    break;
+    }
+    *compress = match;
+    retval = 0;
+ done:
+    return retval;
+}
+
+/*! Return default autocli completion setting
+ *
+ * Currently only returns list-keyword-default, could be extended to rules
+ * @param[in]  h          Clixon handle
+ * @param[out] completion Completion enabled
+ * @retval    -1          Error
+ * @retval     0          OK
+ */
+int
+autocli_completion(clicon_handle h,
+    		   int          *completion)
+{
+    int     retval = -1;
+    char   *str;
+    uint8_t val;
+    char   *reason = NULL;
+    int     ret;
+    cxobj  *xautocli;
+    
+    if (completion == NULL){
+	clicon_err(OE_YANG, EINVAL, "Argument is NULL");
+	goto done;
+    }
+    if ((xautocli = clicon_conf_autocli(h)) == NULL){
+	clicon_err(OE_YANG, 0, "No clixon-autocli");
+	goto done;
+    }
+    if ((str = xml_find_body(xautocli, "completion-default")) == NULL){
+	clicon_err(OE_XML, EINVAL, "No completion-default rule");
+	goto done;
+    }
+    if ((ret = parse_bool(str, &val, &reason)) < 0){
+	clicon_err(OE_CFG, errno, "parse_bool");
+        goto done;
+    }
+    *completion = val;
+    retval = 0; 
+ done:
+    if (reason)
+	free(reason);
+    return retval;
+}
+
+/*! Return default autocli list keyword setting
+ *
+ * Currently only returns list-keyword-default, could be extended to rules
+ * @param[in]  h          Clixon handle
+ * @param[out] listkw     List keyword setting
+ * @retval    -1          Error
+ * @retval     0          OK
+ */
+int
+autocli_list_keyword(clicon_handle     h,
+		     autocli_listkw_t *listkw)
+{
+    int    retval = -1;
+    char  *str;
+    cxobj *xautocli = NULL;
+    
+    if (listkw == NULL){
+	clicon_err(OE_YANG, EINVAL, "Argument is NULL");
+	goto done;
+    }
+    if ((xautocli = clicon_conf_autocli(h)) == NULL){
+	clicon_err(OE_YANG, 0, "No clixon-autocli");
+	goto done;
+    }
+    if ((str = xml_find_body(xautocli, "list-keyword-default")) == NULL){
+	clicon_err(OE_XML, EINVAL, "No list-keyword-default rule");
+	goto done;
+    }
+    *listkw = autocli_listkw_str2int(str);
+    retval = 0; 
+ done:
+    return retval;
+}
+
+/*! Return default autocli treeref state setting, ie generate CLI from YANG non-config
+ *
+ * @param[in]  h             Clixon handle
+ * @param[out] treeref_state If true, generate CLI from state
+ * @retval    -1             Error
+ * @retval     0             OK
+ */
+int
+autocli_treeref_state(clicon_handle h,
+		      int          *treeref_state)
+{
+    int     retval = -1;
+    char   *str;
+    uint8_t val;
+    char   *reason = NULL;
+    int     ret;
+    cxobj  *xautocli;
+    
+    if (treeref_state == NULL){
+	clicon_err(OE_YANG, EINVAL, "Argument is NULL");
+	goto done;
+    }
+    if ((xautocli = clicon_conf_autocli(h)) == NULL){
+	clicon_err(OE_YANG, 0, "No clixon-autocli");
+	goto done;
+    }
+    if ((str = xml_find_body(xautocli, "treeref-state-default")) == NULL){
+	clicon_err(OE_XML, EINVAL, "No treeref-state-default rule");
+	goto done;
+    }
+    if ((ret = parse_bool(str, &val, &reason)) < 0){
+	clicon_err(OE_CFG, errno, "parse_bool");
+        goto done;
+    }
+    *treeref_state = val;
+    retval = 0; 
+ done:
+    if (reason)
+	free(reason);
+    return retval;
+}
+

apps/cli/cli_autocli.h

@@ -0,0 +1,59 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2019 Olof Hagsand
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+  *
+  * C-code corresponding to clixon-autocli.yang
+ */
+
+#ifndef _CLI_AUTOCLI_H_
+#define _CLI_AUTOCLI_H_
+
+/*
+ * Types
+ */
+/*! Autocli operation, see clixon-autocli.yang autocli-op type
+ */
+enum autocli_op{
+    AUTOCLI_OP_ENABLE,
+    AUTOCLI_OP_COMPRESS,
+};
+
+/*
+ * Prototypes
+ */
+int autocli_module(clicon_handle h, char *modname, int *enable);
+int autocli_completion(clicon_handle h, int *completion);
+int autocli_list_keyword(clicon_handle h, autocli_listkw_t *listkw);
+int autocli_compress(clicon_handle h, yang_stmt *ys, int *compress);
+int autocli_treeref_state(clicon_handle h, int *treeref_state);
+
+#endif  /* _CLI_AUTOCLI_H_ */

apps/cli/cli_common.c

@@ -68,7 +68,6 @@
 #include <clixon/clixon.h>
 
 #include "clixon_cli_api.h"
-
 #include "cli_common.h"
 
 /*! Register log notification stream
@@ -916,7 +915,6 @@ save_config_file(clicon_handle h,
     cxobj             *x;
     cxobj             *xerr;
     FILE              *f = NULL;
-    enum genmodel_type gt;
     char              *formatstr;
     enum format_enum   format = FORMAT_XML;
     char              *prefix = "set "; /* XXX hardcoded */
@@ -980,11 +978,9 @@ save_config_file(clicon_handle h,
 	    goto done;
 	break;
     case FORMAT_CLI:
-	if ((gt = clicon_cli_genmodel_type(h)) == GT_ERR)
-	    goto done;
 	x = NULL;
 	while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
-	    if (xml2cli(f, x, prefix, gt) < 0)
+	    if (xml2cli(h, f, x, prefix, fprintf) < 0)
 		goto done;
 	}
 	break;

apps/cli/cli_generate.c

@@ -43,8 +43,11 @@
  * YANG generate CLI
 
  * A special tree called @datamodel is generated by the yang2cli function.
- * This tree contains generated CLIgen syntax for all loaded YANG modules, except the ones given by
+ * This tree contains generated CLIgen syntax for loaded YANG modules, according to the
- * CLICON_CLI_AUTOCLI_EXCLUDE
+ * include/exclude logic in clixon-autocli.yang defined by the following fields:
+ *   module-default
+ *   rule/module-name
+ *   rule/operation=exclude|include
  * The @datamodel tree can be used using the CLIgen "tree reference" functionality as described in
  * the cligen tutorial Secion 2.7.
  * The tree can be modified by removing labels.
@@ -89,16 +92,18 @@ You can see which CLISPEC it generates via clixon_cli -D 2:
 #include <fcntl.h>
 #include <syslog.h>
 #include <signal.h>
+#include <assert.h>
 #include <sys/param.h>
 
 /* cligen */
 #include <cligen/cligen.h>
 
-/* Clicon */
+/* libclixon */
 #include <clixon/clixon.h>
 
 #include "clixon_cli_api.h"
 #include "cli_plugin.h"
+#include "cli_autocli.h"
 #include "cli_generate.h"
 
 /*
@@ -370,8 +375,7 @@ yang2cli_var_pattern(clicon_handle h,
 }
 
 /* Forward */
-static int yang2cli_stmt(clicon_handle h, yang_stmt *ys, genmodel_type gt,
+static int yang2cli_stmt(clicon_handle h, yang_stmt *ys, int level, cbuf *cb);
-			 int level, cbuf *cb);
 
 static int yang2cli_var_union(clicon_handle h, yang_stmt *ys, char *origtype,
 			      yang_stmt *ytype, char *helptext, cbuf *cb);
@@ -592,13 +596,14 @@ yang2cli_var_leafref(clicon_handle h,
      */
     type = yrestype?yang_argument_get(yrestype):NULL;
     cvtypestr = cv_type2str(cvtype);
-    if (type)
+    if (autocli_completion(h, &completionp) < 0)
-	completionp = clicon_cli_genmodel_completion(h) &&
+	goto done;
-	    strcmp(type, "enumeration") != 0 &&
+    if (type && completionp){
+	completionp = strcmp(type, "enumeration") != 0 &&
 	    strcmp(type, "identityref") != 0 &&
 	    strcmp(type, "bits") != 0;
-    else
+    }
-	completionp = clicon_cli_genmodel_completion(h);
+
     if (completionp)
 	cprintf(cb, "(");
     if (yang2cli_var_sub(h, ys, yrestype, helptext, cvtype, 
@@ -653,6 +658,7 @@ yang2cli_var(clicon_handle h,
     char         *cvtypestr;
     int           options = 0;
     int           result;
+    int           completionp;
 
     if ((patterns = cvec_new(0)) == NULL){
 	clicon_err(OE_UNIX, errno, "cvec_new");
@@ -675,7 +681,9 @@ yang2cli_var(clicon_handle h,
 	cprintf(cb, "(");
 	if (yang2cli_var_union(h, ys, origtype, yrestype, helptext, cb) < 0)
 	    goto done;
-	if (clicon_cli_genmodel_completion(h)){
+	if (autocli_completion(h, &completionp) < 0)
+	    goto done;
+	if (completionp){
 	    if ((result = cli_expand_var_generate(h, ys, cvtypestr, 
 						  options, fraction_digits,cb)) < 0)
 		goto done;
@@ -735,7 +743,6 @@ yang2cli_var(clicon_handle h,
 /*! Generate CLI code for Yang leaf statement
  * @param[in]  h     Clixon handle
  * @param[in]  ys    Yang statement
- * @param[in]  gt    CLI Generate style 
  * @param[in]  level Indentation level
  * @param[in]  callback  If set, include a "; cli_set()" callback, otherwise not
  * @param[in]  key_leaf    Is leaf in a key in a list module
@@ -744,7 +751,6 @@ yang2cli_var(clicon_handle h,
 static int
 yang2cli_leaf(clicon_handle h, 
 	      yang_stmt    *ys, 
-	      genmodel_type gt,
 	      int           level,
 	      int           callback,
 	      int 	    key_leaf,
@@ -756,6 +762,7 @@ yang2cli_leaf(clicon_handle h,
     char      *s;
     char      *opext = NULL;
     int        extralevel = 0;
+    autocli_listkw_t listkw;
 
     /* description */
     if ((yd = yang_find(ys, Y_DESCRIPTION, NULL)) != NULL){
@@ -770,7 +777,10 @@ yang2cli_leaf(clicon_handle h,
     /* Look for autocli-op defined in clixon-lib.yang */
     if (yang_extension_value(ys, "autocli-op", CLIXON_LIB_NS, NULL, &opext) < 0)
 	goto done;
-    if (gt == GT_VARS || gt == GT_ALL || gt == GT_HIDE || gt == GT_OC_COMPRESS){
+    if (autocli_list_keyword(h, &listkw) < 0)
+	goto done;
+    if (listkw == AUTOCLI_LISTKW_ALL ||
+	(!key_leaf && listkw == AUTOCLI_LISTKW_NOKEY)){
 	cprintf(cb, "%s", yang_argument_get(ys));
 	yang2cli_helptext(cb, helptext);
 	cprintf(cb, " ");
@@ -806,14 +816,12 @@ yang2cli_leaf(clicon_handle h,
 /*! Generate CLI code for Yang container statement
  * @param[in]  h     Clixon handle
  * @param[in]  ys    Yang statement
- * @param[in]  gt    CLI Generate style 
  * @param[in]  level Indentation level
  * @param[out] cb  Buffer where cligen code is written
  */
 static int
 yang2cli_container(clicon_handle h, 
 		   yang_stmt    *ys, 
-		   genmodel_type gt,
 		   int           level,
 		   cbuf         *cb)
 {
@@ -822,28 +830,20 @@ yang2cli_container(clicon_handle h,
     int           retval = -1;
     char         *helptext = NULL;
     char         *s;
-    int           hide = 0;
+    int           compress = 0;
     int           hide_oc = 0;
-    int           isoc = 0;
     char         *opext = NULL;
     yang_stmt    *ymod = NULL;
 
     if (ys_real_module(ys, &ymod) < 0)
 	goto done;
-    /* Hide container "config" if openconfig and OC_COMPRESS */
-    if (strcmp(yang_argument_get(ys), "config") == 0){
-	if (yang_extension_value(ymod, "openconfig-version", "http://openconfig.net/yang/openconfig-ext", &isoc, NULL) < 0)
-	    goto done;
-	if (isoc &&
-	    gt == GT_OC_COMPRESS)
-	    hide_oc = 1;
-    }
-
     /* If non-presence container && HIDE mode && only child is 
      * a list, then skip container keyword
      * See also xml2cli
      */
-    if ((hide = yang_container_cli_hide(ys, gt)) == 0 && hide_oc == 0){
+    if (autocli_compress(h, ys, &compress) < 0)
+	goto done;
+    if (!compress && hide_oc == 0){
 	cprintf(cb, "%*s%s", level*3, "", yang_argument_get(ys));
 	if ((yd = yang_find(ys, Y_DESCRIPTION, NULL)) != NULL){
 	    if ((helptext = strdup(yang_argument_get(yd))) == NULL){
@@ -871,9 +871,9 @@ yang2cli_container(clicon_handle h,
 
     yc = NULL;
     while ((yc = yn_each(ys, yc)) != NULL) 
-	if (yang2cli_stmt(h, yc, gt, level+1, cb) < 0)
+	if (yang2cli_stmt(h, yc, level+1, cb) < 0)
 	    goto done;
-    if (hide == 0 && hide_oc == 0)
+    if (!compress && hide_oc == 0)
 	cprintf(cb, "%*s}\n", level*3, "");
     retval = 0;
  done:
@@ -885,14 +885,12 @@ yang2cli_container(clicon_handle h,
 /*! Generate CLI code for Yang list statement
  * @param[in]  h     Clixon handle
  * @param[in]  ys    Yang statement
- * @param[in]  gt    CLI Generate style 
  * @param[in]  level Indentation level
  * @param[out] cb    Buffer where cligen code is written
  */
 static int
 yang2cli_list(clicon_handle h,
 	      yang_stmt    *ys, 
-	      genmodel_type gt,
 	      int           level,
 	      cbuf         *cb)
 {
@@ -949,8 +947,10 @@ yang2cli_list(clicon_handle h,
 	    cprintf(cb, "{\n");
 	}
 	if (yang2cli_leaf(h, yleaf,
-			  (gt==GT_VARS||gt==GT_HIDE||gt==GT_OC_COMPRESS)?GT_NONE:gt, level+1,
+			  level+1,
-			  last_key, 1, cb) < 0)
+			  last_key,
+			  1,         /* key_leaf */
+			  cb) < 0)
 	    goto done;
     }
     cprintf(cb, "{\n");
@@ -967,7 +967,7 @@ yang2cli_list(clicon_handle h,
 	}
 	if (cvi != NULL)
 	    continue;
-	if (yang2cli_stmt(h, yc, gt, level+1, cb) < 0)
+	if (yang2cli_stmt(h, yc, level+1, cb) < 0)
 	    goto done;
     }
     cprintf(cb, "%*s}\n", level*3, "");
@@ -984,7 +984,6 @@ yang2cli_list(clicon_handle h,
  *
  * @param[in]  h     Clixon handle
  * @param[in]  ys    Yang statement
- * @param[in]  gt    CLI Generate style 
  * @param[in]  level Indentation level
  * @param[out] cb    Buffer where cligen code is written
 @example
@@ -999,7 +998,6 @@ yang2cli_list(clicon_handle h,
 static int
 yang2cli_choice(clicon_handle h, 
 		yang_stmt    *ys, 
-		genmodel_type gt,
 		int           level,
     		cbuf         *cb)
 {
@@ -1010,7 +1008,7 @@ yang2cli_choice(clicon_handle h,
     while ((yc = yn_each(ys, yc)) != NULL) {
 	switch (yang_keyword_get(yc)){
 	case Y_CASE:
-	    if (yang2cli_stmt(h, yc, gt, level+2, cb) < 0)
+	    if (yang2cli_stmt(h, yc, level+2, cb) < 0)
 		goto done;
 	    break;
 	case Y_CONTAINER:
@@ -1018,7 +1016,7 @@ yang2cli_choice(clicon_handle h,
 	case Y_LEAF_LIST:
 	case Y_LIST:
 	default:
-	    if (yang2cli_stmt(h, yc, gt, level+1, cb) < 0)
+	    if (yang2cli_stmt(h, yc, level+1, cb) < 0)
 		goto done;
 	    break;
 	}
@@ -1031,36 +1029,39 @@ yang2cli_choice(clicon_handle h,
 /*! Generate CLI code for Yang statement
  * @param[in]  h     Clixon handle
  * @param[in]  ys    Yang statement
- * @param[in]  gt    CLI Generate style 
  * @param[in]  level Indentation level
  * @param[out] cb    Buffer where cligen code is written
  */
 static int
 yang2cli_stmt(clicon_handle h, 
 	      yang_stmt    *ys, 
-	      genmodel_type gt,
 	      int           level, 
 	      cbuf         *cb)
 {
     yang_stmt *yc;
     int        retval = -1;
+    int        treeref_state = 0;
     
+    /* Only produce autocli for YANG non-config only if autocli-treeref-state is true */
+    if (autocli_treeref_state(h, &treeref_state) < 0)
+	goto done;
+    if (treeref_state || yang_config(ys)){
 	switch (yang_keyword_get(ys)){
 	case Y_CONTAINER:
-	if (yang2cli_container(h, ys, gt, level, cb) < 0)
+	    if (yang2cli_container(h, ys, level, cb) < 0)
 		goto done;
 	    break;
 	case Y_LIST:
-	if (yang2cli_list(h, ys, gt, level, cb) < 0)
+	    if (yang2cli_list(h, ys, level, cb) < 0)
 		goto done;
 	    break;
 	case Y_CHOICE:
-	if (yang2cli_choice(h, ys, gt, level, cb) < 0)
+	    if (yang2cli_choice(h, ys, level, cb) < 0)
 		goto done;
 	    break;
 	case Y_LEAF_LIST:
 	case Y_LEAF:
-	if (yang2cli_leaf(h, ys, gt, level, 1, 0, cb) < 0)
+	    if (yang2cli_leaf(h, ys, level, 1, 0, cb) < 0)
 		goto done;
 	    break;
 	case Y_CASE:
@@ -1068,12 +1069,13 @@ yang2cli_stmt(clicon_handle h,
 	case Y_MODULE:
 	    yc = NULL;
 	    while ((yc = yn_each(ys, yc)) != NULL)
-	    if (yang2cli_stmt(h, yc, gt, level+1, cb) < 0)
+		if (yang2cli_stmt(h, yc, level+1, cb) < 0)
 		    goto done;
 	    break;
 	default: /* skip */
 	    break;
 	}
+    }
     retval = 0;
  done:
     return retval;
@@ -1107,13 +1109,16 @@ cvec_add_name(cvec *cvv,
 
 /*! Recursive post processing of generated cligen parsetree: populate with co_cvec labels
  *
+
  * This function adds labels to the generated CLIgen tree using YANG as follows:
+ * These labels can be filtered when applying them with the @treeref, @add:<label> syntax.
  * (terminal entry means eg "a ;" where ; is an "empty" child of "a" representing a terminal)
  * 1. Add "termfirstkeys" label on terminal entries of LIST keys, except last
  * 2. Add "termlist" label on terminal entries of LIST
  * 3. Add "termleaf" label on terminal entries of non-empty LEAF/LEAF_LISTs
  * 4. Add "leafvar" label on nodes which are children of non-key LEAFs, eg "a <a>" -> "a <a>,leaf"
  * 5. Add "nonconfig" label on nodes which have YANG "config false" as children
+ * NYI 6. Add "config" label on nodes which have no config false children recursively
  *
  * Then, later, labels can be grouped into specific usages:
  * - config:    @remove:termfirstkeys,@remote:termlist,@remove:termleaf,@remove:nonconfig,
@@ -1124,39 +1129,46 @@ cvec_add_name(cvec *cvv,
  * @param[in]  cop Parent cliegn object (if any)
  * @param[in]  pt  CLIgen parse-tree (generated syntax)
  * @param[in]  i0  Offset into pt
- * @param[in]  y   YANG node of "pt"
+ * @param[in]  yp  YANG parent node of "pt"
  * @param[in]  ykey Special case, If y is list, yc can be a leaf key
  * @retval     0   OK
  * @retval    -1   Error
+ * @note Top-level lookup is O(n^2) since:
+ *    for y in modules
+ *       for co in top-level commands
+ *          if yc = find(y,co)=NULL continue; <----
+ * Also adds an empty node under LIST which is a kludge, it cannot be expressed in the CLIgen syntax
  */
 static int
 yang2cli_post(clicon_handle h,
 	      cg_obj       *cop,
 	      parse_tree   *pt,
 	      int           i0,
-	      yang_stmt    *y,
+	      yang_stmt    *yp,
 	      yang_stmt    *ykey)
 {
     int           retval = -1;
     cg_obj       *co;
     int           i;
+    int           j;
     yang_stmt    *yc;
     int           yciskey;
     int           ycislastkey;
-    enum rfc_6020 ykeyword;
+    enum rfc_6020 ypkeyword;
+    cg_obj       *coj;
 
-    ykeyword = yang_keyword_get(y);
+    ypkeyword = yang_keyword_get(yp);
     for (i = i0; i<pt_len_get(pt); i++){
 	if ((co = pt_vec_i_get(pt, i)) == NULL){
 	    clicon_err(OE_YANG, 0, "Empty object in parsetreelist"); /* shouldnt happen */
 	    goto done;
 	}
 	if (co->co_type == CO_EMPTY){
-	    if (ykeyword == Y_LIST){
+	    if (ypkeyword == Y_LIST){
 		if (ykey){
 		    /* key, list has a <cr> which is marked as "show" */
 		    ycislastkey = 0;
-		    yang_key_match(y, yang_argument_get(ykey), &ycislastkey);
+		    yang_key_match(yp, yang_argument_get(ykey), &ycislastkey);
 		    if (!ycislastkey ||	(cop && cop->co_type==CO_COMMAND))
 			if ((co->co_cvec = cvec_add_name(co->co_cvec, "termfirstkeys")) == NULL)
 			    goto done;
@@ -1167,10 +1179,10 @@ yang2cli_post(clicon_handle h,
 			goto done;
 		}
 	    }
-	    else if (ykeyword == Y_LEAF || ykeyword == Y_LEAF_LIST){
+	    else if (ypkeyword == Y_LEAF || ypkeyword == Y_LEAF_LIST){
 		char *origtype = NULL;
 		yang_stmt *yrestype = NULL;
-		if (yang_type_get(y, &origtype, &yrestype, NULL, NULL, NULL, NULL, NULL) < 0)
+		if (yang_type_get(yp, &origtype, &yrestype, NULL, NULL, NULL, NULL, NULL) < 0)
 		    goto done;
 		if (origtype && strcmp(origtype,"empty") != 0)
 		    if ((co->co_cvec = cvec_add_name(co->co_cvec, "termleaf")) == NULL)
@@ -1179,16 +1191,14 @@ yang2cli_post(clicon_handle h,
 		    free(origtype);
 	    }
 	    continue;
-	}
+	} /* empty */
-	if ((yc = yang_find_datanode(y, co->co_command)) == NULL)
+	/* note This is quadratic, ie highly inefficient */
+	if ((yc = yang_find_datanode(yp, co->co_command)) == NULL)
  	    continue;
-	yciskey = yang_keyword_get(y) == Y_LIST && yang_key_match(y, co->co_command, NULL);
+	yciskey = ypkeyword == Y_LIST && yang_key_match(yp, co->co_command, NULL);
-	/* If leaf add "leafvar" label for non-key leafs 
+	switch (yang_keyword_get(yc)){
-	 * Not a key leaf?
+	case Y_LEAF:
-	 * : y is LIST & 
+	case Y_LEAF_LIST:
-	 */
-	if ((yang_keyword_get(yc) == Y_LEAF) ||
-	    yang_keyword_get(yc) == Y_LEAF_LIST){
 	    /* add empty show 
 	       regular should have ; on last
 	       other ; should be marked as ;
@@ -1202,9 +1212,6 @@ yang2cli_post(clicon_handle h,
 		coe = co_insert(co_pt_get(co), coe);
 	    }
 	    /* XXX move to next recursion level ? */
-	    int     j;
-	    cg_obj *coj;
-	    
 	    if (!yciskey)
 		for (j = 0; j<pt_len_get(co_pt_get(co)); j++){
 		    if ((coj = pt_vec_i_get(co_pt_get(co), j)) == NULL)
@@ -1214,6 +1221,9 @@ yang2cli_post(clicon_handle h,
 		    if ((coj->co_cvec = cvec_add_name(coj->co_cvec, "leafvar")) == NULL)
 			goto done;
 		}
+	    break;
+	default:
+	    break;
 	}
 	/* If state: Add nonconfig label*/
 	if (!yang_config(yc)){
@@ -1222,78 +1232,97 @@ yang2cli_post(clicon_handle h,
 	}
 	/* If y is list and yc is key, then call with y */
 	if (yciskey){
-	    if (yang2cli_post(h, co, co_pt_get(co), 0, y, yc) < 0) // note y not yc
+	    if (yang2cli_post(h, co, co_pt_get(co), 0, yp, yc) < 0) // note y not yc
 		goto done;
 	}
 	else if (yang2cli_post(h, co, co_pt_get(co), 0, yc, NULL) < 0)
 	    goto done;
-    }
+    } /* for */
     retval = 0;
  done:
     return retval;
 }
 
-/*! Autocli generator
+/*! Generate clispec for all modules in yspec (except excluded)
- * Note mix of compile-time runtime 
+ * 
+ * @param[in]  h         Clixon handle
+ * @param[in]  yspec     Top-level Yang statement of type Y_SPEC
+ * @param[in]  treename  Name of tree
+ * @param[in]  xautocli  Autocli config tree (instance of clixon-autocli.yang)
+ * @param[in]  printgen  Log the generated CLIgen syntax
+ * @note Tie-break of same top-level symbol: prefix is NYI
  */
 int
 yang2cli_yspec(clicon_handle      h, 
-	       yang_stmt         *yn, 
+	       yang_stmt         *yspec,
-	       char              *name0,
+	       char              *treename,
 	       int                printgen)
 {
     int             retval = -1;
     parse_tree     *pt0 = NULL;
-    cbuf         *cb0 = NULL;
+    yang_stmt      *ymod;
-    genmodel_type gt;
-    char         *excludelist;
-    char        **exvec = NULL;
-    int           nexvec = 0;
-    int           e;
-    yang_stmt    *ym;
     pt_head        *ph;
-    size_t        len0;
+    int             enable;
+    cbuf           *cb = NULL;
+    char           *prefix;
+    cg_obj         *co;
+    int             previ=0;
+    int             i;
 
     if ((pt0 = pt_new()) == NULL){
 	clicon_err(OE_UNIX, errno, "pt_new");
 	goto done;
     }
-    /* List of modules that should not generate autocli */
+    if ((cb = cbuf_new()) == NULL){
-    if ((excludelist = clicon_option_str(h, "CLICON_CLI_AUTOCLI_EXCLUDE")) != NULL){
-	if ((exvec = clicon_strsep(excludelist, " \t", &nexvec)) == NULL)
-	    goto done;
-    }
-    gt = clicon_cli_genmodel_type(h);
-    if ((cb0 = cbuf_new()) == NULL){
 	clicon_err(OE_XML, errno, "cbuf_new");
 	goto done;
     }
     /* Traverse YANG, loop through all modules and generate CLI */
-    ym = NULL;
+    ymod = NULL;
-    while ((ym = yn_each(yn, ym)) != NULL){
+    while ((ymod = yn_each(yspec, ymod)) != NULL){
-	/* Check if module is in exclude list */
+	/* Filter module name according to cli_autocli.yang setting
-	for (e = 0; e < nexvec; e++){
+	 * Default is pass and ordering is significant
-	    if (strcmp(yang_argument_get(ym), exvec[e]) == 0)
+	 */
-		break;
+	if (autocli_module(h, yang_argument_get(ymod), &enable) < 0)
-	}
+	    goto done;
-	if (e < nexvec)
+	if (!enable)
-	    continue;
+	    continue;
-	len0 = cbuf_len(cb0);
+	cbuf_reset(cb);
-	if (yang2cli_stmt(h, ym, gt, 0, cb0) < 0)
+	if (yang2cli_stmt(h, ymod, 0, cb) < 0)
+	    goto done;
+	if (cbuf_len(cb) == 0)
+	    continue;
+	/* Note Tie-break of same top-level symbol: prefix is NYI
+	 * Needs to move cligen_parse_str() call here instead of later
+	 */
+	if ((prefix = yang_find_myprefix(ymod)) == NULL){
+	    clicon_err(OE_YANG, 0, "Module %s lacks prefix", yang_argument_get(ymod)); /* shouldnt happen */
 	    goto done;
-	if (len0 != cbuf_len(cb0))
-	    clicon_debug(1, "%s Generated auto-cli for %s", __FUNCTION__, yang_argument_get(ym));
 	}
+	/* Parse the buffer using cligen parser. load cli syntax */
+	if (cligen_parse_str(cli_cligen(h), cbuf_get(cb), "yang2cli", pt0, NULL) < 0)
+	    goto done;
+	/* Add prefix: assume new are appended */
+	for (i=previ; i<pt_len_get(pt0); i++){
+	    if ((co = pt_vec_i_get(pt0, i)) != NULL)
+		co_prefix_set(co, prefix);
+	}
+	/* Post-processing, iterate over the generated cligen parse-tree with corresponding yang
+	 * Note cannot do it inline in yang2cli above since:
+	 * 1. labels cannot be set on "empty"
+	 * 2. a; <a>, fn() cannot be set properly
+	 */
+	if (yang2cli_post(h, NULL, pt0, previ, ymod, NULL) < 0)
+	    goto done;
+	previ = pt_len_get(pt0);
+	clicon_debug(1, "%s Generated auto-cli for %s", __FUNCTION__, yang_argument_get(ymod));
 	if (printgen)
 	    clicon_log(LOG_NOTICE, "%s: Top-level cli-spec %s:\n%s",
-		   __FUNCTION__, name0, cbuf_get(cb0));
+		       __FUNCTION__, treename, cbuf_get(cb));
 	else
 	    clicon_debug(2, "%s: Top-level cli-spec %s:\n%s",
-		   __FUNCTION__, name0, cbuf_get(cb0));
+			 __FUNCTION__, treename, cbuf_get(cb));
-
+    } /* ymod */
-    /* load top-level yangspec cli syntax (that point to modules) */
-    if (cligen_parse_str(cli_cligen(h), cbuf_get(cb0), "yang2cli", pt0, NULL) < 0)
-	goto done;
     /* Resolve the expand callback functions in the generated syntax.
      * This "should" only be GENERATE_EXPAND_XMLDB
      * handle=NULL for global namespace, this means expand callbacks must be in
@@ -1301,44 +1330,21 @@ yang2cli_yspec(clicon_handle      h,
      */
     if (cligen_expandv_str2fn(pt0, (expandv_str2fn_t*)clixon_str2fn, NULL) < 0)     
 	goto done;
-    /* Post-processing, iterate over the generated cligen parse-tree with corresponding yang
-     * Note cannot do it inline in yang2cli above since:
-     * 1. labels cannot be set on "empty"
-     * 2. a; <a>, fn() cannot be set properly
-     */
-    ym = NULL;
-    while ((ym = yn_each(yn, ym)) != NULL){
-	/* Check if module is in exclude list */
-	for (e = 0; e < nexvec; e++){
-	    if (strcmp(yang_argument_get(ym), exvec[e]) == 0)
-		break;
-	}
-	if (e < nexvec)
-	    continue;
-	/* Top level find co from yang:
-	 *   XXX: top-level without namespace means check parsetree symbol against each module
-	 * This maye break if there are two top-level symbols with the same name
-	 */
-	if (yang2cli_post(h, NULL, pt0, 0, ym, NULL) < 0)
-	    goto done;
-    }
     /* Append cligen tree and name it */
-    if ((ph = cligen_ph_add(cli_cligen(h), name0)) == NULL)
+    if ((ph = cligen_ph_add(cli_cligen(h), treename)) == NULL)
 	goto done;
     if (cligen_ph_parsetree_set(ph, pt0) < 0)
 	goto done;
 #if 0
     if (printgen){
-	clicon_log(LOG_NOTICE, "%s: Top-level cli-spec %s", __FUNCTION__, name0);
+	clicon_log(LOG_NOTICE, "%s: Top-level cli-spec %s", __FUNCTION__, treename);
 	pt_print1(stderr, pt0, 0);
     }
 #endif
     retval = 0;
  done:
-    if (exvec)
+    if (cb)
-	free(exvec);
+	cbuf_free(cb);
-    if (cb0)
-	cbuf_free(cb0);
     return retval;
 }
 

apps/cli/cli_generate.h

@@ -49,10 +49,14 @@
  */
 #define GENERATE_CALLBACK "overwrite_me"
 
+/* Name of autocli CLIgen treename
+ */
+#define AUTOCLI_TREENAME "basemodel"
+
 /*
  * Prototypes
  */
-int yang2cli_yspec(clicon_handle h, yang_stmt *yspec, char *name0, int printgen);
+int yang2cli_yspec(clicon_handle h, yang_stmt *yspec, char *treename, int printgen);
 int yang2cli_init(clicon_handle h);
 
 #endif  /* _CLI_GENERATE_H_ */

apps/cli/cli_main.c

@@ -67,6 +67,7 @@
 #include "clixon_cli_api.h"
 
 #include "cli_plugin.h"
+#include "cli_autocli.h"
 #include "cli_generate.h"
 #include "cli_common.h"
 #include "cli_handle.h"
@@ -263,17 +264,28 @@ autocli_start(clicon_handle h,
     yang_stmt    *yspec;
     pt_head      *ph;
     parse_tree   *pt = NULL; 
+    int           enable = 0;
 	
-    /* Init yang2cli, mainly labels */
+    clicon_debug(1, "%s", __FUNCTION__);
+    /* There is no single "enable-autocli" flag,
+     * but set 
+     *   <module-default>false</module-default> 
+     * with no rules:
+     *   <rule><operation>enable</operation>
+     * is disable
+     */
+    if (autocli_module(h, NULL, &enable) < 0)
+	goto done;
+    if (!enable){
+	clicon_debug(1, "%s Autocli not enabled (clixon-autocli)", __FUNCTION__);
+	goto ok;
+    }
+    /* Init yang2cli */
     if (yang2cli_init(h) < 0)
 	goto done;
     yspec = clicon_dbspec_yang(h);
-    /* Load clispec for autocli */
+    /* The actual generating call frm yang to clispec for the complete yang spec */
-    if (yang_spec_parse_module(h, "clixon-clispec", NULL, yspec)< 0)
+    if (yang2cli_yspec(h, yspec, AUTOCLI_TREENAME, printgen) < 0)
-	goto done;
-    /* Get the autocli type, ie HOW the cli is generated (could be much more here) */
-    /* basemodel is labelled tree */
-    if (yang2cli_yspec(h, yspec, "basemodel", printgen) < 0)
 	goto done;
 
     /* Create backward compatible tree: @datamodel */
@@ -311,6 +323,7 @@ autocli_start(clicon_handle h,
 	goto done;
     if (cligen_ph_parsetree_set(ph, pt) < 0)
 	goto done;
+ ok:
     retval = 0;
  done:
     return retval;
@@ -337,7 +350,7 @@ usage(clicon_handle h,
             "\t-m <mode>\tSpecify plugin syntax mode\n"
 	    "\t-q \t\tQuiet mode, dont print greetings or prompt, terminate on ctrl-C\n"
 	    "\t-p <dir>\tYang directory path (see CLICON_YANG_DIR)\n"
-	    "\t-G \t\tPrint auo-cli CLI syntax generated from YANG (if CLICON_CLI_GENMODEL enabled)\n"
+	    "\t-G \t\tPrint auo-cli CLI syntax generated from YANG\n"
 	    "\t-L \t\tDebug print dynamic CLI syntax including completions and expansions\n"
 	    "\t-l <s|e|o|f<file>> \tLog on (s)yslog, std(e)rr, std(o)ut or (f)ile (stderr is default)\n"
 	    "\t-y <file>\tOverride yang spec file (dont include .yang suffix)\n"

apps/cli/cli_plugin.c

@@ -158,10 +158,10 @@ syntax_append(clicon_handle h,
 
     if ((csm = syntax_mode_find(stx, name, 1)) == NULL) 
 	return -1;
-
+    if (cligen_parsetree_merge(csm->csm_pt, NULL, pt) < 0){
-    if (cligen_parsetree_merge(csm->csm_pt, NULL, pt) < 0)
+	clicon_err(OE_PLUGIN, errno, "cligen_parsetree_merge");
 	return -1;
-    
+    }
     return 0;
 }
 
@@ -360,8 +360,10 @@ cli_load_syntax_file(clicon_handle h,
 	 * and add to all syntaxes after all files have been loaded. At this point
 	 * all modes may not be known (not yet loaded)
 	 */
-	if (cligen_parsetree_merge(ptall, NULL, pt) < 0)
+	if (cligen_parsetree_merge(ptall, NULL, pt) < 0){
-	    return -1;
+	    clicon_err(OE_PLUGIN, errno, "cligen_parsetree_merge");
+	    goto done;
+	}
     }
     else {
 	for (i = 0; i < nvec; i++) {
@@ -378,7 +380,6 @@ cli_load_syntax_file(clicon_handle h,
 
     cligen_parsetree_free(pt, 1);
     retval = 0;
-    
 done:
     if (cvv)
 	cvec_free(cvv);
@@ -460,8 +461,10 @@ cli_syntax_load(clicon_handle h)
      */
     m = stx->stx_modes;
     do {
-	if (cligen_parsetree_merge(m->csm_pt, NULL, ptall) < 0)
+	if (cligen_parsetree_merge(m->csm_pt, NULL, ptall) < 0){
-	    return -1;
+	    clicon_err(OE_PLUGIN, errno, "cligen_parsetree_merge");
+	    goto done;
+	}
 	if (gen_parse_tree(h, m) != 0)
 	    goto done;
 	m = NEXTQ(cli_syntaxmode_t *, m);
@@ -616,6 +619,10 @@ clicon_parse(clicon_handle  h,
 	    fprintf(f, "CLI syntax error: \"%s\" is ambiguous\n", cmd);
 	    break;
 	} /* switch result */
+	if (cvv){
+	    cvec_free(cvv);
+	    cvv = NULL;
+	}
     }
     retval = 0;
 done:

apps/cli/cli_show.c

@@ -70,6 +70,7 @@
 
 /* Exported functions in this file are in clixon_cli_api.h */
 #include "clixon_cli_api.h"
+#include "cli_autocli.h"
 #include "cli_common.h" /* internal functions */
 
 /*! Given an xpath encoded in a cbuf, append a second xpath into the first
@@ -435,7 +436,6 @@ cli_show_config1(clicon_handle h,
     cxobj           *xt = NULL;
     cxobj           *xc;
     cxobj           *xerr;
-    enum genmodel_type gt;
     yang_stmt       *yspec;
     char            *namespace = NULL;
     cvec            *nsc = NULL;
@@ -508,12 +508,10 @@ cli_show_config1(clicon_handle h,
 	    cli_xml2txt(xc, cligen_output, 0); /* tree-formed text */
 	break;
     case FORMAT_CLI:
-	/* get CLI generatade mode: VARS|ALL */
-	if ((gt = clicon_cli_genmodel_type(h)) == GT_ERR)
-	    goto done;
 	xc = NULL; /* Dont print xt itself */
 	while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL)
-	    cli_xml2cli(xc, prefix, gt, cligen_output); /* cli syntax */
+	    if (xml2cli(h, stdout, xc, prefix, cligen_output) < 0)
+		goto done;
 	break;
     case FORMAT_NETCONF:
 	cligen_output(stdout, "<rpc xmlns=\"%s\" %s><edit-config><target><candidate/></target><config>\n",
@@ -697,7 +695,6 @@ cli_show_auto1(clicon_handle h,
     cxobj           *xp;
     cxobj           *xp_helper;
     cxobj           *xerr;
-    enum genmodel_type gt;
     char            *api_path = NULL;
     char            *prefix = NULL;
     enum rfc_6020    ys_keyword;
@@ -762,9 +759,8 @@ cli_show_auto1(clicon_handle h,
 
 	switch (format){
 	case FORMAT_CLI:
-	    if ((gt = clicon_cli_genmodel_type(h)) == GT_ERR)
+	    if (xml2cli(h, stdout, xp, prefix, cligen_output) < 0) /* cli syntax */
 		goto done;
-	    cli_xml2cli(xp, prefix, gt, cligen_output); /* cli syntax */
 	    break;
 	case FORMAT_NETCONF:
 	    fprintf(stdout, "<rpc><edit-config><target><candidate/></target><config>\n");
@@ -981,7 +977,8 @@ cli_pagination(clicon_handle h,
 		xml2txt_cb(stdout, xc, cligen_output); /* tree-formed text */
 		break;
 	    case FORMAT_CLI:
-		xml2cli_cb(stdout, xc, NULL, GT_HIDE, cligen_output); /* cli syntax */
+		/* hardcoded to compress and list-keyword = nokey */
+		xml2cli(h, stdout, xc, NULL, cligen_output);
 		break;
 	    default:
 		break;
@@ -1016,3 +1013,128 @@ cli_pagination(clicon_handle h,
 	cbuf_free(cb);
     return retval;
 }
+
+/*! Translate from XML to CLI commands
+ *
+ * Howto: join strings and pass them down. 
+ * Identify unique/index keywords for correct set syntax.
+ * @param[in] h        Clicon handle
+ * @param[in] f        Output FILE (eg stdout)
+ * @param[in] xn       XML Parse-tree (to translate)
+ * @param[in] prepend  Print this text in front of all commands.
+ * @param[in] fn       Callback to make print function
+ * @see xml2cli  XXX should probably use the generic function
+ */
+int
+xml2cli(clicon_handle    h,
+	FILE            *f, 
+	cxobj           *xn,
+	char            *prepend,
+	clicon_output_cb *fn)
+{
+    int        retval = -1;
+    cxobj     *xe = NULL;
+    cbuf      *cbpre = NULL;
+    yang_stmt *ys;
+    int        match;
+    char      *body;
+    char      *opext = NULL;
+    int        compress = 0;
+    autocli_listkw_t listkw;
+
+    if (autocli_list_keyword(h, &listkw) < 0)
+	goto done;
+    if (xml_type(xn)==CX_ATTR)
+	goto ok;
+    if ((ys = xml_spec(xn)) == NULL)
+	goto ok;
+    /* Look for autocli-op defined in clixon-lib.yang */
+    if (yang_extension_value(xml_spec(xn), "autocli-op", CLIXON_LIB_NS, NULL, &opext) < 0) {
+	goto ok;
+    }
+    if ((opext != NULL) && ((strcmp(opext, "hide-database") == 0) || (strcmp(opext, "hide-database-auto-completion") == 0))){
+	goto ok;
+    }
+    /* If leaf/leaf-list or presence container, then print line */
+    if (yang_keyword_get(ys) == Y_LEAF ||
+	yang_keyword_get(ys) == Y_LEAF_LIST){
+	if (prepend)
+	    (*fn)(f, "%s", prepend);
+	if (listkw != AUTOCLI_LISTKW_NONE)
+	    (*fn)(f, "%s ", xml_name(xn));
+	if ((body = xml_body(xn)) != NULL){
+	    if (index(body, ' '))
+		(*fn)(f, "\"%s\"", body);
+	    else
+		(*fn)(f, "%s", body);
+	}
+	(*fn)(f, "\n");
+	goto ok;
+    }
+    /* Create prepend variable string */
+    if ((cbpre = cbuf_new()) == NULL){
+	clicon_err(OE_PLUGIN, errno, "cbuf_new");	
+	goto done;
+    }
+    if (prepend)
+	cprintf(cbpre, "%s", prepend);
+
+    /* If non-presence container && HIDE mode && only child is 
+     * a list, then skip container keyword
+     * See also yang2cli_container */
+    if (autocli_compress(h, ys, &compress) < 0)
+	goto done;
+    if (!compress)
+	cprintf(cbpre, "%s ", xml_name(xn));
+
+    /* If list then first loop through keys */
+    if (yang_keyword_get(ys) == Y_LIST){
+	xe = NULL;
+	while ((xe = xml_child_each(xn, xe, -1)) != NULL){
+	    if ((match = yang_key_match(ys, xml_name(xe), NULL)) < 0)
+		goto done;
+	    if (!match)
+		continue;
+	    if (listkw == AUTOCLI_LISTKW_ALL)
+		cprintf(cbpre, "%s ", xml_name(xe));
+	    cprintf(cbpre, "%s ", xml_body(xe));
+	}
+    }
+    else if ((yang_keyword_get(ys) == Y_CONTAINER) &&
+	     yang_find(ys, Y_PRESENCE, NULL) != NULL){
+	/* If presence container, then print as leaf (but continue to children) */
+	if (prepend)
+	    (*fn)(f, "%s", prepend);
+	if (listkw != AUTOCLI_LISTKW_NONE)
+	    (*fn)(f, "%s ", xml_name(xn));
+	if ((body = xml_body(xn)) != NULL){
+	    if (index(body, ' '))
+		(*fn)(f, "\"%s\"", body);
+	    else
+		(*fn)(f, "%s", body);
+	}
+	(*fn)(f, "\n");
+    }
+
+    /* For lists, print cbpre before its elements */
+    if (yang_keyword_get(ys) == Y_LIST)
+	(*fn)(f, "%s\n", cbuf_get(cbpre));	
+    /* Then loop through all other (non-keys) */
+    xe = NULL;
+    while ((xe = xml_child_each(xn, xe, -1)) != NULL){
+	if (yang_keyword_get(ys) == Y_LIST){
+	    if ((match = yang_key_match(ys, xml_name(xe), NULL)) < 0)
+		goto done;
+	    if (match)
+		continue; /* Not key itself */
+	}
+	if (xml2cli(h, f, xe, cbuf_get(cbpre), fn) < 0)
+	    goto done;
+    }
+ ok:
+    retval = 0;
+ done:
+    if (cbpre)
+	cbuf_free(cbpre);
+    return retval;
+}

apps/cli/clixon_cli_api.h

@@ -39,6 +39,25 @@
 #ifndef _CLIXON_CLI_API_H_
 #define _CLIXON_CLI_API_H_
 
+/*
+ * Types
+ */
+/*! Autocli list keyword type, see clixon-autocli.yang list-keyword-type
+ * Assume a YANG LIST: 
+ *    list a { 
+ *       key x; 
+ *       leaf x; 
+ *       leaf y;
+ *    }
+ * Maybe this type should be in cli_autocli.h
+ */
+enum autocli_listkw{
+    AUTOCLI_LISTKW_NONE,  /* No extra keywords, only <vars>: a <x> <y> */
+    AUTOCLI_LISTKW_NOKEY, /* Keywords on non-key variables: a <x> y <y> */
+    AUTOCLI_LISTKW_ALL,   /* Keywords on all variables: a x <x> y <y> */
+};
+typedef enum autocli_listkw autocli_listkw_t;
+
 /* 
  * Function Declarations 
  */
@@ -118,7 +137,7 @@ int expand_dbvar(void *h, char *name, cvec *cvv, cvec *argv,
 		  cvec *commands, cvec *helptexts);
 int cli_xml2file (cxobj *xn, int level, int prettyprint, clicon_output_cb *fn);
 int cli_xml2txt(cxobj *xn, clicon_output_cb *fn, int level);
-int cli_xml2cli(cxobj *xn, char *prepend, enum genmodel_type gt, clicon_output_cb  *fn);
+int xml2cli(clicon_handle h, FILE *f, cxobj *xn, char *prepend, clicon_output_cb  *fn);
 
 /* cli_show.c: CLIgen new vector arg callbacks */
 int show_yang(clicon_handle h, cvec *vars, cvec *argv);

example/main/clixon-example@2020-12-01.yang

@@ -15,6 +15,9 @@ module clixon-example {
     import ietf-datastores {
       prefix ds;
     }
+   import clixon-lib{
+      prefix cl;
+   }
     description
 	"Clixon example used as a part of the Clixon test suite.
          It can be used as a basis for making new Clixon applications.
@@ -52,6 +55,10 @@ module clixon-example {
 	    leaf value{
 		type string;
 	    }
+	    leaf hidden{
+		type string;
+		cl:autocli-op hide;
+	    }
 	    leaf stat{
 		description "Inline state data for example application";
 		config false;

example/main/example.xml.in

@@ -14,9 +14,6 @@
   <CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
   <CLICON_SOCK>/usr/local/var/example/example.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/example/example.pidfile</CLICON_BACKEND_PIDFILE>
-  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
   <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
   <CLICON_CLI_TAB_MODE>0</CLICON_CLI_TAB_MODE>
   <CLICON_XMLDB_DIR>/usr/local/var/example</CLICON_XMLDB_DIR>
@@ -24,5 +21,17 @@
   <CLICON_NACM_MODE>disabled</CLICON_NACM_MODE>
   <CLICON_STREAM_DISCOVERY_RFC5277>true</CLICON_STREAM_DISCOVERY_RFC5277>
   <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
-  <restconf><enable>true</enable><auth-type>none</auth-type><socket><namespace>default</namespace><address>0.0.0.0</address><port>80</port><ssl>false</ssl></socket></restconf>
+  <restconf>
+    <enable>true</enable>
+    <auth-type>none</auth-type>
+    <socket><namespace>default</namespace><address>0.0.0.0</address><port>80</port><ssl>false</ssl></socket>
+  </restconf>
+  <autocli>
+     <module-default>false</module-default>
+     <rule>
+       <name>include clixon-example</name>
+       <module-name>clixon-example</module-name>
+       <operation>enable</operation>
+     </rule>
+  </autocli>
 </clixon-config>

lib/clixon/clixon_data.h

@@ -96,7 +96,7 @@ cxobj *clicon_conf_xml(clicon_handle h);
 int clicon_conf_xml_set(clicon_handle h, cxobj *x);
 
 cxobj *clicon_conf_restconf(clicon_handle h);
-cxobj *clicon_conf_clispec(clicon_handle h);
+cxobj *clicon_conf_autocli(clicon_handle h);
 
 db_elmnt *clicon_db_elmnt_get(clicon_handle h, const char *db);
 int clicon_db_elmnt_set(clicon_handle h, const char *db, db_elmnt *xc);

lib/clixon/clixon_options.h

@@ -57,29 +57,6 @@
 /*
  * Types
  */
-/*! Controls how keywords a generated in CLI syntax / prints from object model
- * Example YANG: 
- * container c{
- *  list a {
- *    key x;
- *    leaf x;   
- *    leaf y;   
- *  }
- * }
- * NONE: c a <x> <y>;
- * VARS: c a <x> y <y>;
- * ALL:  c a x <x> y <y>;
- * HIDE: a x <x> y <y>;
- */
-enum genmodel_type{
-    GT_ERR =-1, /* Error  */
-    GT_NONE=0,  /* No extra keywords */
-    GT_VARS,    /* Keywords on non-key variables */
-    GT_ALL,     /* Keywords on all variables */
-    GT_HIDE,    /* Keywords on all variables and hide container around lists */
-    GT_OC_COMPRESS, /* OpenConfig */
-};
-typedef enum genmodel_type genmodel_type;
 
 /*! See clixon-config.yang type startup_mode */
 enum startup_mode_t{
@@ -189,9 +166,6 @@ static inline char *clicon_cli_mode(clicon_handle h){
 static inline int clicon_cli_tab_mode(clicon_handle h){
     return clicon_option_int(h, "CLICON_CLI_TAB_MODE");
 }
-static inline char *clicon_cli_model_treename(clicon_handle h){
-    return clicon_option_str(h, "CLICON_CLI_MODEL_TREENAME");
-}
 static inline char *clicon_sock_str(clicon_handle h){
     return clicon_option_str(h, "CLICON_SOCK");
 }
@@ -212,9 +186,6 @@ static inline char *clicon_nacm_recovery_user(clicon_handle h){
 }
 
 /*-- Specific option access functions for YANG options w type conversion--*/
-int   clicon_cli_genmodel(clicon_handle h);
-int   clicon_cli_genmodel_completion(clicon_handle h);
-enum genmodel_type clicon_cli_genmodel_type(clicon_handle h);
 int   clicon_cli_varonly(clicon_handle h);
 int   clicon_sock_family(clicon_handle h);
 int   clicon_sock_port(clicon_handle h);

lib/clixon/clixon_xml_map.h

@@ -52,8 +52,6 @@ typedef enum yang_class yang_class;
 int isxmlns(cxobj *x);
 int xml2txt_cb(FILE *f, cxobj *x, clicon_output_cb *fn);
 int xml2txt(FILE *f, cxobj *x, int level);
-int xml2cli_cb(FILE *f, cxobj *x, char *prepend, enum genmodel_type gt, clicon_output_cb *fn);
-int xml2cli(FILE *f, cxobj *x, char *prepend, enum genmodel_type gt);
 int xmlns_assign(cxobj *x);
 int xml2cvec(cxobj *xt, yang_stmt *ys, cvec **cvv0);
 int cvec2xml_1(cvec *cvv, char *toptag, cxobj *xp, cxobj **xt0);

lib/clixon/clixon_yang.h

@@ -229,7 +229,6 @@ int       yang_stats(yang_stmt *y, uint64_t *nrp, size_t *szp);
 yang_stmt *yspec_new(void);
 yang_stmt *ys_new(enum rfc_6020 keyw);
 yang_stmt *ys_prune(yang_stmt *yp, int i);
-
 int        ys_free1(yang_stmt *ys, int self);
 int        ys_free(yang_stmt *ys);
 int        ys_cp(yang_stmt *nw, yang_stmt *old);
@@ -238,6 +237,7 @@ int        yn_insert(yang_stmt *ys_parent, yang_stmt *ys_child);
 int        yn_insert1(yang_stmt *ys_parent, yang_stmt *ys_child);
 yang_stmt *yn_each(yang_stmt *yn, yang_stmt *ys);
 char      *yang_key2str(int keyword);
+int        yang_str2key(char *str);
 int        ys_module_by_xml(yang_stmt *ysp, struct xml *xt, yang_stmt **ymodp);
 yang_stmt *ys_module(yang_stmt *ys);
 int        ys_real_module(yang_stmt *ys, yang_stmt **ymod);
@@ -249,6 +249,7 @@ yang_stmt *yang_find_schemanode(yang_stmt *yn, char *argument);
 char      *yang_find_myprefix(yang_stmt *ys);
 char      *yang_find_mynamespace(yang_stmt *ys);
 int        yang_find_prefix_by_namespace(yang_stmt *ys, char *ns, char **prefix);
+int        yang_find_namespace_by_prefix(yang_stmt *ys, char *prefix, char **ns);
 yang_stmt *yang_myroot(yang_stmt *ys);
 yang_stmt *yang_choice(yang_stmt *y);
 int        yang_order(yang_stmt *y);
@@ -270,7 +271,6 @@ int        yang_config(yang_stmt *ys);
 int        yang_config_ancestor(yang_stmt *ys);
 int        yang_features(clicon_handle h, yang_stmt *yt);
 cvec      *yang_arg2cvec(yang_stmt *ys, char *delimi);
-int        yang_container_cli_hide(yang_stmt *ys, int gt);
 int        yang_key_match(yang_stmt *yn, char *name, int *lastkey);
 int        yang_type_cache_regexp_set(yang_stmt *ytype, int rxmode, cvec *regexps);
 int        yang_type_cache_get(yang_stmt *ytype, yang_stmt **resolved, int *options,
@@ -281,5 +281,6 @@ yang_stmt *yang_anydata_add(yang_stmt *yp, char *name);
 int        yang_extension_value(yang_stmt *ys, char *name, char *ns, int *exist, char **value);
 int        yang_sort_subelements(yang_stmt *ys);
 int        yang_init(clicon_handle h);
+int        yang_single_child_type(yang_stmt *ys, enum rfc_6020 subkeyw);
 
 #endif  /* _CLIXON_YANG_H_ */

lib/src/clixon_data.c

@@ -477,22 +477,22 @@ clicon_conf_restconf(clicon_handle h)
     return NULL;
 }
 
-/*! Get local YANG specification for Clixon-clispec.yang tree
+/*! Get clixon-autocli.yang part of the clixon config tree
  *
- * That is, get the XML of clixon-config/clispec container of clixon-config.yang
+ * That is, get the XML of clixon-config/autocli container of clixon-config.yang
  * @param[in]  h  Clicon handle
- * @retval     x  XML tree containing clispec xml node from clixon-clispec.yang
+ * @retval     x  XML tree containing clispec xml node from clixon-autoclu.yang
  * @code
- *    cxobj *xclispec = clicon_conf_clispec(h);
+ *    cxobj *xautocli = clicon_conf_autocli(h);
  * @endcode
  */
 cxobj *
-clicon_conf_clispec(clicon_handle h)
+clicon_conf_autocli(clicon_handle h)
 {
     cxobj  *xconfig = NULL;
 
     if ((xconfig = clicon_conf_xml(h)) != NULL)  	 /* Get local config */
-	return xpath_first(xconfig, NULL, "clispec");
+	return xpath_first(xconfig, NULL, "autocli");
     return NULL;
 }
 

lib/src/clixon_options.c

@@ -82,17 +82,6 @@
 #include "clixon_validate.h"
 #include "clixon_xml_map.h"
 
-/* Mapping between Cli generation from Yang string <--> constants, 
-   see clixon-config.yang type cli_genmodel_type */
-static const map_str2int cli_genmodel_map[] = {
-    {"NONE",                 GT_NONE},
-    {"VARS",                 GT_VARS},
-    {"ALL",                  GT_ALL},
-    {"HIDE",                 GT_HIDE},
-    {"OC_COMPRESS",          GT_OC_COMPRESS},
-    {NULL,                   -1}
-};
-
 /* Mapping between Clicon startup modes string <--> constants, 
    see clixon-config.yang type startup_mode */
 static const map_str2int startup_mode_map[] = {
@@ -113,7 +102,6 @@ static const map_str2int priv_mode_map[] = {
     {NULL,        -1}
 };
 
-
 /* Mapping between Clicon nacm user credential string <--> constants, 
  * see clixon-config.yang type nacm_cred_mode */
 static const map_str2int nacm_credentials_map[] = {
@@ -742,54 +730,6 @@ clicon_option_del(clicon_handle h,
  * But sometimes there are type conversions, etc which makes it more
  * convenient to make wrapper functions. Or not?
  *-----------------------------------------------------------------*/
-/*! Whether to generate CLIgen syntax from datamodel or not (0, 1 or 2)
- * Must be used with a previous clicon_option_exists().
- * @param[in] h     Clicon handle
- * @retval    flag  If set, generate CLI code from yang model, otherwise not
- * @see clixon-config@<date>.yang CLICON_CLI_GENMODEL
- */
-int
-clicon_cli_genmodel(clicon_handle h)
-{
-    char const *opt = "CLICON_CLI_GENMODEL";
-
-    if (clicon_option_exists(h, opt))
-	return clicon_option_int(h, opt);
-    else
-	return 0;
-}
-
-/*! Generate code for CLI completion of existing db symbols
- * @param[in] h     Clicon handle
- * @retval    flag  If set, generate auto-complete CLI specs
- * @see clixon-config@<date>.yang CLICON_CLI_GENMODEL_COMPLETION
- */
-int
-clicon_cli_genmodel_completion(clicon_handle h)
-{
-    char const *opt = "CLICON_CLI_GENMODEL_COMPLETION";
-
-    if (clicon_option_exists(h, opt))
-	return clicon_option_int(h, opt);
-    else
-	return 0;
-}
-
-/*! How to generate and show CLI syntax: VARS|ALL 
- * @param[in] h     Clicon handle
- * @retval    mode
- * @see clixon-config@<date>.yang CLICON_CLI_GENMODEL_TYPE
- */
-enum genmodel_type
-clicon_cli_genmodel_type(clicon_handle h)
-{
-    char *str;
-
-    if ((str = clicon_option_str(h, "CLICON_CLI_GENMODEL_TYPE")) == NULL)
-	return GT_VARS;
-    else
-	return clicon_str2int(cli_genmodel_map, str);
-}
 
 /*! Get "do not include keys in cvec" in cli vars callbacks
  * @param[in] h     Clicon handle

lib/src/clixon_string.c

@@ -357,9 +357,6 @@ uri_percent_decode(char  *enc,
  * @param[in]   ...    stdarg variable parameters
  * @retval      0      OK
  * @retval     -1      Error
- * @see https://www.w3.org/TR/2008/REC-xml-20081126/#syntax chapter 2.6
- * @see uri_percent_encode
- * @see AMPERSAND mode in clixon_xml_parse.l
  * @code
  *   char *encstr = NULL;
  *   if (xml_chardata_encode(&encstr, "fmtstr<>& %s", "substr<>") < 0)
@@ -368,12 +365,14 @@ uri_percent_decode(char  *enc,
  *      free(encstr);
  * @endcode
  * Essentially encode as follows:
- *     & -> "&amp; "   must
+ *     & -> "&amp;"   must
- *     < -> "&lt; "    must
+ *     < -> "&lt;"    must
- *     > -> "&gt; "    must for backward compatibility
+ *     > -> "&gt;"    must for backward compatibility
- *     ' -> "&apos; "  may
+ *     ' -> "&apos;"  may
- *     ' -> "&quot; "  may
+ *     " -> "&quot;"  may
- * Optionally >
+ * @see https://www.w3.org/TR/2008/REC-xml-20081126/#syntax chapter 2.6
+ * @see uri_percent_encode
+ * @see AMPERSAND mode in clixon_xml_parse.l, implicit decoding
  * @see xml_chardata_cbuf_append for a specialized version
  */
 int

lib/src/clixon_xml_map.c

@@ -211,151 +211,6 @@ xml2txt(FILE  *f,
     return xml2txt_recurse(f, x, fprintf, level);
 }
 
-/*! Translate from XML to CLI commands
- * Howto: join strings and pass them down. 
- * Identify unique/index keywords for correct set syntax.
- * @param[in] f        Where to print cli commands
- * @param[in] x        XML Parse-tree (to translate)
- * @param[in] prepend  Print this text in front of all commands.
- * @param[in] gt       option to steer cli syntax
- * @param[in] fn       Callback to make print function
- */
-int 
-xml2cli_recurse(FILE              *f, 
-		cxobj             *x, 
-		char              *prepend, 
-		enum genmodel_type gt,
-		clicon_output_cb  *fn)
-{
-    int              retval = -1;
-    cxobj           *xe = NULL;
-    cbuf            *cbpre = NULL;
-    yang_stmt       *ys;
-    int              match;
-    char            *body;
-
-    if (xml_type(x)==CX_ATTR)
-	goto ok;
-    if ((ys = xml_spec(x)) == NULL)
-	goto ok;
-    /* If leaf/leaf-list or presence container, then print line */
-    if (yang_keyword_get(ys) == Y_LEAF ||
-	yang_keyword_get(ys) == Y_LEAF_LIST){
-	if (prepend)
-	    (*fn)(f, "%s", prepend);
-	if (gt == GT_ALL || gt == GT_VARS || gt == GT_HIDE)
-	    (*fn)(f, "%s ", xml_name(x));
-	if ((body = xml_body(x)) != NULL){
-	    if (index(body, ' '))
-		(*fn)(f, "\"%s\"", body);
-	    else
-		(*fn)(f, "%s", body);
-	}
-	(*fn)(f, "\n");
-	goto ok;
-    }
-    /* Create prepend variable string */
-    if ((cbpre = cbuf_new()) == NULL){
-	clicon_err(OE_PLUGIN, errno, "cbuf_new");	
-	goto done;
-    }
-    if (prepend)
-	cprintf(cbpre, "%s", prepend);
-
-    /* If non-presence container && HIDE mode && only child is 
-     * a list, then skip container keyword
-     * See also yang2cli_container */
-    if (yang_container_cli_hide(ys, gt) == 0)
-	cprintf(cbpre, "%s ", xml_name(x));
-
-    /* If list then first loop through keys */
-    if (yang_keyword_get(ys) == Y_LIST){
-	xe = NULL;
-	while ((xe = xml_child_each(x, xe, -1)) != NULL){
-	    if ((match = yang_key_match(ys, xml_name(xe), NULL)) < 0)
-		goto done;
-	    if (!match)
-		continue;
-	    if (gt == GT_ALL)
-		cprintf(cbpre, "%s ", xml_name(xe));
-	    cprintf(cbpre, "%s ", xml_body(xe));
-	}
-    }
-    else if ((yang_keyword_get(ys) == Y_CONTAINER) &&
-	     yang_find(ys, Y_PRESENCE, NULL) != NULL){
-	/* If presence container, then print as leaf (but continue to children) */
-	if (prepend)
-	    (*fn)(f, "%s", prepend);
-	if (gt == GT_ALL || gt == GT_VARS || gt == GT_HIDE || gt == GT_OC_COMPRESS)
-	    (*fn)(f, "%s ", xml_name(x));
-	if ((body = xml_body(x)) != NULL){
-	    if (index(body, ' '))
-		(*fn)(f, "\"%s\"", body);
-	    else
-		(*fn)(f, "%s", body);
-	}
-	(*fn)(f, "\n");
-    }
-
-    /* Then loop through all other (non-keys) */
-    xe = NULL;
-    while ((xe = xml_child_each(x, xe, -1)) != NULL){
-	if (yang_keyword_get(ys) == Y_LIST){
-	    if ((match = yang_key_match(ys, xml_name(xe), NULL)) < 0)
-		goto done;
-	    if (match){
-		(*fn)(f, "%s\n", cbuf_get(cbpre));	
-		continue; /* Not key itself */
-	    }
-	}
-	if (xml2cli_recurse(f, xe, cbuf_get(cbpre), gt, fn) < 0)
-	    goto done;
-    }
- ok:
-    retval = 0;
- done:
-    if (cbpre)
-	cbuf_free(cbpre);
-    return retval;
-}
-
-/*! Translate from XML to CLI commands
- * Howto: join strings and pass them down. 
- * Identify unique/index keywords for correct set syntax.
- * @param[in] f        Where to print cli commands
- * @param[in] x        XML Parse-tree (to translate)
- * @param[in] prepend  Print this text in front of all commands.
- * @param[in] gt       option to steer cli syntax
- * @param[in] fn       Callback to make print function
- */
-int 
-xml2cli_cb(FILE              *f, 
-	   cxobj             *x, 
-	   char              *prepend, 
-	   enum genmodel_type gt,
-	   clicon_output_cb  *fn)
-{
-    return xml2cli_recurse(f, x, prepend, gt, fn);
-}
-
-/*! Translate from XML to CLI commands
- * Howto: join strings and pass them down. 
- * Identify unique/index keywords for correct set syntax.
- * Args:
- *  @param[in] f        Where to print cli commands
- *  @param[in] x        XML Parse-tree (to translate)
- *  @param[in] prepend  Print this text in front of all commands.
- *  @param[in] gt       option to steer cli syntax
- */
-int 
-xml2cli(FILE              *f, 
-	cxobj             *x, 
-	char              *prepend, 
-	enum genmodel_type gt)
-{
-    return xml2cli_recurse(f, x, prepend, gt, fprintf);
-}
-
 /*! Translate a single xml node to a cligen variable vector. Note not recursive 
  * @param[in]  xt   XML tree containing one top node
  * @param[in]  ys   Yang spec containing type specification of top-node of xt
@@ -1592,6 +1447,7 @@ assign_namespace(cxobj *x0, /* source */
     cvec      *nsc0 = NULL;
     cvec      *nsc = NULL;
     yang_stmt *y;
+    int        ret;
     
     /* 2a. Detect if namespace is declared in x1 target parent */
     if (xml2prefix(x1p, ns, &pexist) == 1){
@@ -1649,9 +1505,9 @@ assign_namespace(cxobj *x0, /* source */
 		    goto done;
 		}
 		/* Find local (imported) prefix for that module namespace */
-		if (yang_find_prefix_by_namespace(y, ns, &ptmp) < 0)
+		if ((ret = yang_find_prefix_by_namespace(y, ns, &ptmp)) < 0)
 		    goto done;
-		if ((prefix1 = strdup(ptmp)) == NULL){
+		if (ret == 1 && (prefix1 = strdup(ptmp)) == NULL){
 		    clicon_err(OE_UNIX, errno, "strdup");
 		    goto done;
 		}   

lib/src/clixon_yang.c

@@ -1267,23 +1267,27 @@ yang_find_mynamespace(yang_stmt *ys)
  * (global) namespace of a module, but you do not know the local prefix
  * used to access it in XML.
  * @param[in]  ys        Yang statement in module tree (or module itself)
- * @param[in]  ns		 Namspace URI as char* pointer into yang tree
+ * @param[in]  ns	 Namespace URI as char* pointer into yang tree
  * @param[out] prefix    Local prefix to access module with (direct pointer)
- * @retval     0         not found
+ * @retval    -1         Error
- * @retval    -1         found
+ * @retval     0         Not found
+ * @retval     1         Found
  * @note  prefix NULL is not returned, if own module, then return its prefix
  * @code
  *    char *prefix = NULL;
- *    if (yang_find_prefix_by_namespace(ys, "urn:example:clixon", &prefix) < 0)
+ *    if ((found = yang_find_prefix_by_namespace(ys, "urn:example:clixon", &prefix)) < 0)
  *      err;
+ *    if (found)
+ *      // use prefix
  * @endcode
+ * @see yang_find_module_by_namespace
  */
 int
 yang_find_prefix_by_namespace(yang_stmt *ys,
 			      char      *ns,
 			      char     **prefix)
 {
-    int        retval = 0; /* not found */
+    int        retval = -1;
     yang_stmt *my_ymod; /* My module */
     char      *myns;   /* My ns */
     yang_stmt *yspec;
@@ -1293,6 +1297,10 @@ yang_find_prefix_by_namespace(yang_stmt *ys,
     yang_stmt *yprefix; 
 
     clicon_debug(2, "%s", __FUNCTION__);
+    if (prefix == NULL){
+	clicon_err(OE_YANG, EINVAL, "prefix is NULL");
+	goto done;
+    }
     /* First check if namespace is my own module */
     myns = yang_find_mynamespace(ys);
     if (strcmp(myns, ns) == 0){
@@ -1316,10 +1324,55 @@ yang_find_prefix_by_namespace(yang_stmt *ys,
 	}
     }
  notfound:
+    retval = 0; /* not found */
+ done:
     return retval;
  found:
     assert(*prefix);
-    return 1;
+    retval = 1;
+    goto done;
+}
+
+/*! Given a yang statement and local prefi valid in module , find namespace
+ *
+ * @param[in]  ys        Yang statement in module tree (or module itself)
+ * @param[in]  prefix    Local prefix to access module with (direct pointer)
+ * @param[out] ns	 Namespace URI as char* pointer into yang tree
+ * @retval    -1         Error
+ * @retval     0         Not found
+ * @retval     1         Found
+ * @note  prefix NULL is not returned, if own module, then return its prefix
+ * @code
+ *    char *ns = NULL;
+ *    if ((found = yang_find_namespace_by_prefix(ys, "ex", &ns)) < 0)
+ *      err;
+ *    if (found)
+ *      // use ns *    
+ * @endcode
+ * @see yang_find_module_by_prefix
+ */
+int
+yang_find_namespace_by_prefix(yang_stmt *ys,
+			      char      *prefix,
+			      char     **ns)
+{
+    int        retval = -1; 
+    yang_stmt *ym;
+
+    if (ns == NULL){
+	clicon_err(OE_YANG, EINVAL, "ns is NULL");
+	goto done;
+    }
+    if ((ym = yang_find_module_by_prefix(ys, prefix)) == NULL)
+	goto notfound;
+    if ((*ns = yang_find_mynamespace(ym)) == NULL)
+	goto notfound;
+    retval = 1; /* found */
+ done:
+    return retval;
+ notfound:
+    retval = 0;
+    goto done;
 }
 
 /*! Return topmost yang root node directly under module/submodule
@@ -1505,6 +1558,12 @@ yang_key2str(int keyword)
     return (char*)clicon_int2str(ykmap, keyword);
 }
 
+int
+yang_str2key(char *str)
+{
+    return clicon_str2int(ykmap, str);
+}
+
 /*! Find top data node among all modules by namespace in xml tree
  * @param[in]  yspec    Yang specification
  * @param[in]  xt       XML node
@@ -3392,53 +3451,6 @@ yang_arg2cvec(yang_stmt *ys,
     return cvv;
 }
 
-/*! Check if yang is subject to generated cli GT_HIDE boolean
- * The yang should be:
- * 1) a non-presence container
- * 2) parent of a (single) list XXX: or could multiple lists work?
- * 3) no other data node children
- * @retval   0   No, does not satisfy the GT_HIDE condition
- * @retval   1   Yes, satisfies the GT_HIDE condition
- * @see clixon-config.yang HIDE enumeration type
- */
-int
-yang_container_cli_hide(yang_stmt         *ys,
-			enum genmodel_type gt)
-{
-    yang_stmt    *yc = NULL;
-    int           i;
-    enum rfc_6020 keyw;
-
-    keyw = yang_keyword_get(ys);
-    /* HIDE mode */
-    if (gt != GT_HIDE && gt != GT_OC_COMPRESS)
-	return 0;
-    /* A container */
-    if (yang_keyword_get(ys) != Y_CONTAINER)
-	return 0;
-    /* Non-presence */
-    if (yang_find(ys, Y_PRESENCE, NULL) != NULL)
-	return 0;
-    /* Ensure a single list child and no other data nodes */
-    i = 0; /* Number of list nodes */
-    while ((yc = yn_each(ys, yc)) != NULL) {
-	keyw = yang_keyword_get(yc);
-	/* case/choice could hide anything so disqualify those */
-	if (keyw == Y_CASE || keyw == Y_CHOICE)
-	    break;
-	if (!yang_datanode(yc)) /* Allowed, check next */
-	    continue;
-	if (keyw != Y_LIST) /* Another datanode than list */
-	    break;
-	if (i++>0) /* More than one list (or could this work?) */
-	    break;
-    }
-    if (yc != NULL) /* break from loop */
-	return 0;
-    if (i != 1) /* List found */
-	return 0;
-    return 1; /* Passed all tests: yes you can hide this keyword */
-}
 
 /*! Check if yang node yn has key-stmt as child which matches name
  *
@@ -3719,9 +3731,9 @@ yang_anydata_add(yang_stmt *yp,
 
 /*! Find extension argument and return if extension exists and its argument value
  *
- * @param[in]  ys     Yang statement
+ * @param[in]  ys     Yang statement where unknown statement may occur referncing to extension
  * @param[in]  name   Name of the extension 
- * @param[in]  ns     The namespace
+ * @param[in]  ns     The namespace of the module where the extension is defined
  * @param[out] exist  The extension exists.
  * @param[out] value  clispec operator (hide/none) - direct pointer into yang, dont free
  * @retval     0      OK: Look in exist and value for return value
@@ -3751,6 +3763,7 @@ yang_extension_value(yang_stmt *ys,
     cg_var    *cv;
     char      *prefix = NULL;
     cbuf      *cb = NULL;
+    int        ret;
 
     if ((cb = cbuf_new()) == NULL){
 	clicon_err(OE_UNIX, errno, "cbuf_new");
@@ -3762,7 +3775,9 @@ yang_extension_value(yang_stmt *ys,
 	    continue;
 	if ((ymod = ys_module(yext)) == NULL)
 	    continue;
-	if (yang_find_prefix_by_namespace(ymod, ns, &prefix) < 0)
+	if ((ret = yang_find_prefix_by_namespace(ymod, ns, &prefix)) < 0)
+	    goto done;
+	if (ret == 0) /* not found */
 	    goto ok;
 	cbuf_reset(cb);
 	cprintf(cb, "%s:%s", prefix, name);
@@ -3901,3 +3916,53 @@ yang_search_index_extension(clicon_handle h,
 }
 
 #endif /* XML_EXPLICIT_INDEX */
+
+/*! Check if yang node has a single child of specific type
+ *
+ * Mainly used for condition for CLI compression
+ * The yang should be:
+ * 1) If container it should be non-presence
+ * 2) parent of a (single) specified type
+ * 3) no other data node children
+ * @param[in] ys      Yang node
+ * @param[in] subkeyw Expected keyword of single child (typically Y_LIST)
+ * @retval    0       No, node does not have single child of specified type
+ * @retval    1       Yes, node has single child of specified type
+ * @see https://github.com/openconfig/ygot/blob/master/docs/design.md#openconfig-path-compression 2nd clause
+ */
+int
+yang_single_child_type(yang_stmt    *ys,
+		       enum rfc_6020 subkeyw)
+
+{
+    yang_stmt    *yc = NULL;
+    int           i;
+    enum rfc_6020 keyw;
+
+    /* Match parent */
+    /* If container, check it is Non-presence */
+    if (yang_keyword_get(ys) == Y_CONTAINER){
+	if (yang_find(ys, Y_PRESENCE, NULL) != NULL)
+	    return 0;
+    }
+    /* Ensure a single list child and no other data nodes */
+    i = 0; /* Number of list nodes */
+    while ((yc = yn_each(ys, yc)) != NULL) {
+	keyw = yang_keyword_get(yc);
+	/* case/choice could hide anything so disqualify those */
+	if (keyw == Y_CASE || keyw == Y_CHOICE)
+	    break;
+	if (!yang_datanode(yc)) /* Allowed, check next */
+	    continue;
+	if (keyw != subkeyw) /* Another datanode than subkeyw */
+	    break;
+	if (i++>0) /* More than one list (or could this work?) */
+	    break;
+    }
+    if (yc != NULL) /* break from loop */
+	return 0;
+    if (i != 1) /* List found */
+	return 0;
+    return 1; /* Passed all tests: yes you can hide this keyword */
+}
+

lib/src/clixon_yang_module.c

@@ -488,6 +488,7 @@ clixon_module_upgrade(clicon_handle    h,
  * @note Prefixes are relative to the module they are defined
  * @see yang_find_module_by_name
  * @see yang_find_module_by_namespace
+ * @see yang_find_namespace_by_prefix
  */
 yang_stmt *
 yang_find_module_by_prefix(yang_stmt *ys, 
@@ -564,6 +565,7 @@ yang_find_module_by_prefix_yspec(yang_stmt *yspec,
  * @retval     NULL       not found
  * @see yang_find_module_by_name
  * @see yang_find_module_by_prefix    module-specific prefix
+ * @see yang_find_prefix_by_namespace
  */
 yang_stmt *
 yang_find_module_by_namespace(yang_stmt *yspec, 

test/fuzz/cli/runfuzz.sh

@@ -24,11 +24,17 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
   <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
   <CLICON_CLI_TAB_MODE>0</CLICON_CLI_TAB_MODE>
+  <autocli>
+     <enable-autocli>true</enable-autocli>
+     <module-default>false</module-default>
+     <rule>
+       <name>include $APPNAME</name>
+       <operation>enable</operation>
+       <module-name>$APPNAME/module-name>
+     </rule>
+  </autocli>
 </clixon-config>
 EOF
 

test/fuzz/netconf/runfuzz.sh

@@ -24,9 +24,6 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
   <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
   <CLICON_CLI_TAB_MODE>0</CLICON_CLI_TAB_MODE>
 </clixon-config>

test/lib.sh

@@ -249,6 +249,35 @@ function restconf_config()
     fi
 }
 
+# Default autocli configuration
+# Can be placed in clixon-config
+# Exclude all modules instead as defined by arg1
+# Args:
+# 1: modname module name pattern to be included
+# 2: list-keyword
+# 3: treerefstate
+function autocli_config()
+{
+    modname=$1
+    listkw=$2
+    state=$3
+
+    TMP=$(cat <<EOF
+  <autocli>
+     <module-default>false</module-default>
+     <list-keyword-default>$listkw</list-keyword-default>
+     <treeref-state-default>$state</treeref-state-default>
+     <rule>
+       <name>include $modname</name>
+       <operation>enable</operation>
+       <module-name>$modname</module-name>
+     </rule>
+  </autocli>
+EOF
+   )
+   echo "${TMP}"
+}
+
 # Some tests may set owner of testdir to something strange and quit, need
 # to reset to me
 if [ ! -G $dir ]; then 

test/test_cli_apipath.sh

@@ -18,7 +18,8 @@ else
     mkdir $clidir
 fi
 
-# Use yang in example
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME} kw-all false)
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -29,11 +30,10 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <!-- ALL or VARS -->
-  <CLICON_CLI_GENMODEL_TYPE>ALL</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_cli_auto_extension.sh

@@ -22,6 +22,9 @@ fi
 
 # Use yang in example
 
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME}\* kw-nokey false)
+
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
@@ -33,17 +36,14 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL>2</CLICON_CLI_GENMODEL>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 
-
 cat <<EOF > $clidir/ex.cli
 CLICON_MODE="example";
 CLICON_PROMPT="%U@%H %W> ";

test/test_cli_auto_genmodel.sh

@@ -4,7 +4,7 @@
 # In particular setting a config, displaying as cli commands and reconfigure it
 # Tests:
 # Make a config in CLI. Show output as CLI, save it and ensure it is the same
-# Try the different GENMODEL settings
+# Try different list-keyword and compress settings (see clixon-autocli.yang)
 # NOTE this uses the "Old" autocli (eg cli_set()), see test_cli_auto.sh for "new" autocli using the cli_auto_*() API
 
 # Magic line must be first in script (see README.md)
@@ -32,27 +32,7 @@ if [ ! -d "$OPENCONFIG" ]; then
     echo "...skipped: OPENCONFIG not set"
     if [ "$s" = $0 ]; then exit 0; else return 0; fi
 fi
-
+OCDIR=$OPENCONFIG/release/models
-cat <<EOF > $cfg
-<clixon-config xmlns="http://clicon.org/config">
-  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
-  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
-  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
-  <CLICON_YANG_DIR>$OPENCONFIG/</CLICON_YANG_DIR>
-  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
-  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
-  <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
-  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
-  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL>2</CLICON_CLI_GENMODEL>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
-  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
-  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
-  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
-  <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
-</clixon-config>
-EOF
 
 cat <<EOF > $fyang
 module $APPNAME {
@@ -181,6 +161,79 @@ discard, discard_changes();
 
 EOF
 
+# Set config for CLI
+# 1. listkw   - either none, vars, all
+# 2 compress -   surrounding container entities are removed from list nodes
+# 3. openconfig - config and state containers are "compressed" out of the schema in openconfig modules
+function setconfig()
+{
+    listkw=$1
+    compress=$2
+    openconfig=$3
+
+        if $compress; then
+COMPRESS=$(cat <<EOF
+      <rule>
+         <name>compress</name>
+         <operation>compress</operation>
+	 <yang-keyword>container</yang-keyword>
+	 <yang-keyword-child>list</yang-keyword-child>
+      </rule>
+EOF
+)
+    else
+	COMPRESS=""
+    fi
+    if $openconfig; then
+OCOMPRESS=$(cat <<EOF
+      <rule>
+         <name>openconfig compress</name>
+	 <operation>compress</operation>
+	 <yang-keyword>container</yang-keyword>
+	 <schema-nodeid>config</schema-nodeid>
+	 <!--schema-nodeid>state</schema-nodeid-->
+	 <!--module-name>openconfig*</module-name-->
+	 <extension>oc-ext:openconfig-version</extension>
+      </rule>"
+EOF
+)
+    else
+	OCOMPRESS=""
+    fi
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$OCDIR/</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
+  <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+  <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
+  <autocli>
+     <module-default>false</module-default>
+     <list-keyword-default>${listkw}</list-keyword-default>
+     <treeref-state-default>true</treeref-state-default>
+     <rule>
+       <name>include ${APPNAME}</name>
+       <operation>enable</operation>
+       <module-name>${APPNAME}*</module-name>
+     </rule>
+     ${COMPRESS}
+     ${OCOMPRESS}
+  </autocli>
+</clixon-config>
+EOF
+} # setconfig
+
+new "Set config before backend start"
+setconfig kw-nokey false false
+
 new "test params: -f $cfg"
 if [ $BE -ne 0 ]; then
     new "kill old backend"
@@ -199,90 +252,112 @@ wait_backend
 # then deleting it, and reloading it
 # 1. mode - either VARS Keywords on non-key variables: a <x> y <y> or
 #                  ALL  Keywords on all variables: a x <x> y <y>
+#                  HIDE
+#                  OC_COMPRESS
+# 2. listkw   - either none, vars, all
+# 3. compress -   surrounding container entities are removed from list nodes
+# 4. openconfig - config and state containers are "compressed" out of the schema in openconfig modules
 function testrun()
 {
     mode=$1
-    if [ $mode = ALL ]; then
+    listkw=$2
-	table=" table"
+    compress=$3
+
+    if [ $listkw = kw-all ]; then
 	name=" name"
-    elif [ $mode = HIDE ]; then
+    else
-	table=
 	name=
-    elif [ $mode = OC_COMPRESS ]; then
+    fi
+    if $compress; then
 	table=
-	name=
     else
 	table=" table"
-	name=
     fi
 
     new "set a"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg set$table parameter$name a value x)" 0 ""
+    echo "$clixon_cli -1 -f $cfg set$table parameter$name a value x"
+    expectpart "$($clixon_cli -1 -f $cfg set$table parameter$name a value x)" 0 ""
     
     new "set b"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg set$table parameter$name b value y)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg set$table parameter$name b value y)" 0 ""
 
     new "reset b"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg set$table parameter$name b value z)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg set$table parameter$name b value z)" 0 ""
 
     new "show match a & b"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show config)" 0 "set$table parameter$name a" "set$table parameter$name a value x" "set$table parameter$name b" "set$table parameter$name b value z" --not-- "set$table parameter$name b value y"
+    expectpart "$($clixon_cli -1 -f $cfg show config)" 0 "set$table parameter$name a" "set$table parameter$name a value x" "set$table parameter$name b" "set$table parameter$name b value z" --not-- "set$table parameter$name b value y"
-SAVED=$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show config)
+
+SAVED=$($clixon_cli -1 -f $cfg show config)
 # awkward having pretty-printed xml in matching strings
 
     new "show match a & b xml"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show xml)" 0 "<table xmlns=\"urn:example:clixon\">" "<parameter>" "<name>a</name>" "<value>x</value>" "</parameter>" "<parameter>" "<name>b</name>" "<value>z</value>" "</parameter>" "</table>"
+    expectpart "$($clixon_cli -1 -f $cfg show xml)" 0 "<table xmlns=\"urn:example:clixon\">" "<parameter>" "<name>a</name>" "<value>x</value>" "</parameter>" "<parameter>" "<name>b</name>" "<value>z</value>" "</parameter>" "</table>"
 
     # https://github.com/clicon/clixon/issues/157
     new "delete a y expect fail"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg delete$table parameter$name a value y 2>&1)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg delete$table parameter$name a value y 2>&1)" 0 ""
 
     new "show match a & b xml" # Expect same
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show xml)" 0 "<table xmlns=\"urn:example:clixon\">" "<parameter>" "<name>a</name>" "<value>x</value>" "</parameter>" "<parameter>" "<name>b</name>" "<value>z</value>" "</parameter>" "</table>"
+    expectpart "$($clixon_cli -1 -f $cfg show xml)" 0 "<table xmlns=\"urn:example:clixon\">" "<parameter>" "<name>a</name>" "<value>x</value>" "</parameter>" "<parameter>" "<name>b</name>" "<value>z</value>" "</parameter>" "</table>"
 
     new "delete a x"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg delete$table parameter$name a value x)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg delete$table parameter$name a value x)" 0 ""
 
     new "show match a & b xml"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show xml)" 0 "<table xmlns=\"urn:example:clixon\">" "<parameter>" "<name>a</name>"  "</parameter>" "<parameter>" "<name>b</name>" "<value>z</value>" "</parameter>" "</table>" --not-- "<value>x</value>"
+    expectpart "$($clixon_cli -1 -f $cfg show xml)" 0 "<table xmlns=\"urn:example:clixon\">" "<parameter>" "<name>a</name>"  "</parameter>" "<parameter>" "<name>b</name>" "<value>z</value>" "</parameter>" "</table>" --not-- "<value>x</value>"
 
     new "delete a"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg delete$table parameter$name a)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg delete$table parameter$name a)" 0 ""
 
     new "show match b"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show config)" 0  "$table parameter$name b" "$table parameter$name b value z" --not-- "$table parameter$name a" "$table parameter$name a value x" "$table parameter$name b value y"
+    expectpart "$($clixon_cli -1 -f $cfg show config)" 0  "$table parameter$name b" "$table parameter$name b value z" --not-- "$table parameter$name a" "$table parameter$name a value x" "$table parameter$name b value y"
 
     new "discard"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg discard)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg discard)" 0 ""
 
     new "show match empty"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show config)" 0 --not-- "$table parameter$name b" "$table parameter$name b value z"  "$table parameter$name a" "$table parameter$name a value x" "$table parameter$name b value y"
+    expectpart "$($clixon_cli -1 -f $cfg show config)" 0 --not-- "$table parameter$name b" "$table parameter$name b value z"  "$table parameter$name a" "$table parameter$name a value x" "$table parameter$name b value y"
 
     new "load saved cli config"
-    expectpart "$(echo "$SAVED" | $clixon_cli -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg)" 0 ""
+    expectpart "$(echo "$SAVED" | $clixon_cli -f $cfg)" 0 ""
 
     new "show saved a & b"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg show config)" 0 "set$table parameter$name a" "set$table parameter$name a value x" "set$table parameter$name b" "set$table parameter$name b value z" --not-- "set$table parameter$name b value y"
+    expectpart "$($clixon_cli -1 -f $cfg show config)" 0 "set$table parameter$name a" "set$table parameter$name a value x" "set$table parameter$name b" "set$table parameter$name b value z" --not-- "set$table parameter$name b value y"
 
     new "discard"
-    expectpart "$($clixon_cli -1 -o CLICON_CLI_GENMODEL_TYPE=$mode -f $cfg discard)" 0 ""
+    expectpart "$($clixon_cli -1 -f $cfg discard)" 0 ""
 } # testrun
 
-new "keywords=HIDE"
+new "Config: Keywords on non-keys"
-testrun HIDE
+setconfig kw-nokey false false
 
-new "keywords=ALL"
+new "Keywords on non-keys"
-testrun ALL
+testrun VARS kw-nokey false
 
-new "keywords=OC_COMPRESS"
+new "Config: Keywords on all"
-testrun OC_COMPRESS
+setconfig kw-all false false
 
-new "keywords=VARS"
+new "Keywords on all"
-testrun VARS
+testrun ALL kw-all false
+
+new "Config: Keywords on non-keys, container compress"
+setconfig kw-nokey true false
+
+new "Keywords on non-keys, container compress"
+testrun HIDE kw-nokey true
+
+new "Config:Keywords on non-keys, container and openconfig compress"
+setconfig kw-nokey true true
+
+new "Keywords on non-keys, container and openconfig compress"
+testrun OC_COMPRESS kw-nokey true
+
+new "Config:default"
+setconfig kw-nokey false false
 
 # show state
 new "set a"
-expectpart "$($clixon_cli -1 -f $cfg set$table parameter a value x)" 0 ""
+expectpart "$($clixon_cli -1 -f $cfg set table parameter a value x)" 0 ""
 
 new "commit"
 expectpart "$($clixon_cli -1 -f $cfg commit)" 0 ""
@@ -295,23 +370,29 @@ expectpart "$($clixon_cli -1 -f $cfg show state exstate)" 0 "state sender x" --n
 
 #---- openconfig path compression
 
+new "Config:Openconfig compression"
+setconfig kw-nokey true true
+
 new "Openconfig: OC_COMPRESS+extension: compressed)"
-expectpart "$($clixon_cli -1 -f $cfg -o CLICON_CLI_GENMODEL_TYPE=OC_COMPRESS set interface e enabled true 2>&1)" 0 "^$"
+expectpart "$($clixon_cli -1 -f $cfg set interface e enabled true 2>&1)" 0 "^$"
 
 new "Openconfig: OC_COMPRESS+extension: no config (negative)"
-expectpart "$($clixon_cli -1 -f $cfg -o CLICON_CLI_GENMODEL_TYPE=OC_COMPRESS set interface e config enabled true 2>&1)" 255 "Unknown command"
+expectpart "$($clixon_cli -1 -f $cfg set interface e config enabled true 2>&1)" 255 "Unknown command"
 
 new "Openconfig: OC_COMPRESS+no-extension: no config"
-expectpart "$($clixon_cli -1 -f $cfg -o CLICON_CLI_GENMODEL_TYPE=OC_COMPRESS set interface2 e config enabled true 2>&1)" 0 "^$"
+expectpart "$($clixon_cli -1 -f $cfg set interface2 e config enabled true 2>&1)" 0 "^$"
 
 new "Openconfig: OC_COMPRESS+no-extension: no config (negative)"
-expectpart "$($clixon_cli -1 -f $cfg -o CLICON_CLI_GENMODEL_TYPE=OC_COMPRESS set interface2 e enabled true 2>&1)" 255 "Unknown command"
+expectpart "$($clixon_cli -1 -f $cfg set interface2 e enabled true 2>&1)" 255 "Unknown command"
+
+new "Config: default"
+setconfig kw-nokey false false
 
 new "Openconfig: OC_VARS+extension: no compresssion"
-expectpart "$($clixon_cli -1 -f $cfg -o CLICON_CLI_GENMODEL_TYPE=VARS set interfaces interface e config enabled true 2>&1)" 0 "^$"
+expectpart "$($clixon_cli -1 -f $cfg set interfaces interface e config enabled true 2>&1)" 0 "^$"
 
 new "Openconfig: OC_VARS+extension: no compresssion (negative)"
-expectpart "$($clixon_cli -1 -f $cfg -o CLICON_CLI_GENMODEL_TYPE=VARS set interfaces interface e enabled true 2>&1)" 255 "Unknown command"
+expectpart "$($clixon_cli -1 -f $cfg set interfaces interface e enabled true 2>&1)" 255 "Unknown command"
 
 new "Kill backend"
 # Check if premature kill

test/test_cli_auto_spec.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+# Test of clixon-clispec.yang and cli_clispec.[ch]
+
+# Magic line must be first in script (see README.md)
+s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
+
+APPNAME=example
+
+# include err() and new() functions and creates $dir
+
+cfg=$dir/conf_yang.xml
+fspec=$dir/automode.cli
+fin=$dir/in
+fstate=$dir/state.xml
+fyang=$dir/clixon-example.yang
+fyang2=$dir/clixon-example2.yang
+
+cat <<EOF > $fyang
+module clixon-example {
+    yang-version 1.1;
+    namespace "urn:example:clixon";
+    prefix ex;
+    /* Generic config data */
+    container table{
+	list parameter{
+	    key name;
+	    leaf name{
+		type string;
+	    }
+	    leaf value{
+		type string;
+	    }
+	    leaf stat{
+		description "Inline state data for example application";
+		config false;
+		type int32;
+	    }
+	}
+    }
+}
+EOF
+
+cat <<EOF > $fyang2
+module clixon-example2 {
+    yang-version 1.1;
+    namespace "urn:example:clixon2";
+    prefix ex2;
+    /* Alt config data */
+    container table2{
+	list parameter{
+	    key name;
+	    leaf name{
+		type string;
+	    }
+	    leaf value{
+		type string;
+	    }
+	    leaf stat{
+		description "Inline state data for example application";
+		config false;
+		type int32;
+	    }
+	}
+    }
+}
+EOF
+
+cat <<EOF > $fspec
+CLICON_MODE="example";
+CLICON_PROMPT="%U@%H %W> ";
+CLICON_PLUGIN="example_cli";
+
+# Autocli syntax tree operations
+edit @datamodelshow, cli_auto_edit("basemodel");
+up, cli_auto_up("basemodel");
+top, cli_auto_top("basemodel");
+set @datamodel, cli_auto_set();
+merge @datamodel, cli_auto_merge();
+create @datamodel, cli_auto_create();
+delete("Delete a configuration item") @datamodel, cli_auto_del();
+validate("Validate changes"), cli_validate();
+commit("Commit the changes"), cli_commit();
+quit("Quit"), cli_quit();
+show("Show a particular state of the system"){
+    configuration("Show configuration"), cli_auto_show("datamodel", "candidate", "text", true, false);{
+	    xml("Show configuration as XML"), cli_auto_show("datamodel", "candidate", "xml", false, false);
+	    cli("Show configuration as CLI commands"), cli_auto_show("datamodel", "candidate", "cli", false, false, "set ");
+	    netconf("Show configuration as netconf edit-config operation"), cli_auto_show("datamodel", "candidate", "netconf", false, false);
+	    text("Show configuration as text"), cli_auto_show("datamodel", "candidate", "text", false, false);
+	    json("Show configuration as JSON"), cli_auto_show("datamodel", "candidate", "json", false, false);
+    }
+    state("Show configuration and state"), cli_auto_show("datamodel", "running", "xml", false, true);
+}
+EOF
+
+cat <<EOF > $dir/startup_db
+<${DATASTORE_TOP}>
+  <table xmlns="urn:example:clixon">
+    <parameter>
+      <name>a</name>
+      <value>42</value>
+    </parameter>
+  </table>
+</${DATASTORE_TOP}>
+EOF
+
+# Add inline state
+cat <<EOF > $fstate
+  <table xmlns="urn:example:clixon">
+    <parameter>
+      <name>a</name>
+      <stat>99</stat>
+    </parameter>
+  </table>
+EOF
+
+# Make a new variant of clixon config file
+# Arg 1: autocli-spec
+function runconfig()
+{
+    AUTOCLI="$1"
+
+# Use yang in example
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
+  <CLICON_YANG_MAIN_DIR>$dir</CLICON_YANG_MAIN_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PRETTY>false</CLICON_XMLDB_PRETTY>
+  ${AUTOCLI}
+</clixon-config>
+EOF
+}
+
+new "backend config"
+runconfig ""
+
+new "test params: -f $cfg"
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -z -f $cfg
+    if [ $? -ne 0 ]; then
+	err
+    fi
+    new "start backend -s startup -f $cfg -- -sS $fstate"
+    start_backend -s startup -f $cfg -- -sS $fstate
+fi
+
+new "wait backend"
+wait_backend
+
+AUTOCLI=$(cat <<EOF
+  <autocli>
+     <module-default>false</module-default>
+  </autocli>
+EOF
+)
+new "disable autocli"
+runconfig "$AUTOCLI"
+
+new "set table expected fail"
+expectpart "$(echo "set table" | $clixon_cli -f $cfg 2>&1)" 255 "CLIgen tree 'datamodel' not found"
+
+AUTOCLI=$(cat <<EOF
+  <autocli>
+     <module-default>true</module-default>
+  </autocli>
+EOF
+)
+
+new "enable autocli"
+runconfig "$AUTOCLI"
+
+new "set table OK"
+expectpart "$(echo "set table" | $clixon_cli -f $cfg 2>&1)" 0 ""
+
+AUTOCLI=$(cat <<EOF
+  <autocli>
+     <module-default>false</module-default>
+     <rule>
+       <name>include example</name>
+       <operation>enable</operation>
+       <description>Include the example module for autocli generation</description>
+       <module-name>clixon-example</module-name>
+     </rule>
+  </autocli>
+EOF
+)
+
+new "exclude example2 using default"
+runconfig "$AUTOCLI"
+
+new "set table"
+expectpart "$(echo "set table" | $clixon_cli -f $cfg 2>&1)" 0 ""
+
+new "set table2 expect fail"
+expectpart "$(echo "set table2" | $clixon_cli -f $cfg 2>&1)" 0 "CLI syntax error" "Unknown command"
+
+if [ $BE -ne 0 ]; then
+    new "Kill backend"
+    # Check if premature kill
+    pid=$(pgrep -u root -f clixon_backend)
+    if [ -z "$pid" ]; then
+	err "backend already dead"
+    fi
+    # kill backend
+    stop_backend -f $cfg
+fi
+
+rm -rf $dir
+
+new "endtest"
+endtest

test/test_cli_auto_sub.sh

@@ -19,7 +19,8 @@ else
     mkdir $clidir
 fi
 
-# Use yang in example
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME}\* kw-nokey false)
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -32,13 +33,11 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL>2</CLICON_CLI_GENMODEL>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_cli_auto_treeref.sh

@@ -20,7 +20,8 @@ else
     mkdir $clidir
 fi
 
-# Use yang in example
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME}\* kw-nokey true)
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -32,13 +33,11 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL>2</CLICON_CLI_GENMODEL>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_cli_multikey.sh

@@ -12,7 +12,8 @@ APPNAME=example
 cfg=$dir/conf_yang.xml
 fyang=$dir/$APPNAME.yang
 
-# Use yang in example
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME} kw-all false)
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -23,11 +24,10 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <!-- ALL or VARS -->
-  <CLICON_CLI_GENMODEL_TYPE>ALL</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_cli_translate.sh

@@ -14,6 +14,8 @@ APPNAME=example
 cfg=$dir/conf_yang.xml
 
 fyang=$dir/clixon-example.yang
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME} kw-all false)
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -23,10 +25,10 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL_TYPE>ALL</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_openconfig.sh

@@ -23,24 +23,11 @@ if [ ! -d "$OPENCONFIG" ]; then
     if [ "$s" = $0 ]; then exit 0; else return 0; fi
 fi
 
-cat <<EOF > $cfg
+# OPENCONFIG dir has a small number of extra yangs apart from OCDIR
-<clixon-config xmlns="http://clicon.org/config">
+OCDIR=$OPENCONFIG/release/models
-  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+
-  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+files=$(find ${OPENCONFIG} -name "*.yang")
-  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
-  <CLICON_YANG_DIR>$OPENCONFIG</CLICON_YANG_DIR>
-  <CLICON_YANG_AUGMENT_ACCEPT_BROKEN>true</CLICON_YANG_AUGMENT_ACCEPT_BROKEN>
-  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
-  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
-  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
-  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
-  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
-  <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
-</clixon-config>
-EOF
 
-files=$(find $OPENCONFIG -name "*.yang")
 # Count nr of modules (exclude submodule) Assume "module" or "submodule"
 # first word on first line
 let ms=0; # Nr of modules
@@ -67,7 +54,29 @@ done
 new "Openconfig test: $clixon_cli -1f $cfg show version ($m modules)"
 for f in $files; do
     if [ -n "$(head -1 $f|grep '^module')" ]; then
-	new "$clixon_cli -D $DBG  -1f $cfg -y $f show version"
+	modname=$(basename $f | awk -F "." '{print $1}')
+	# Generate autocli for these modules
+	AUTOCLI=$(autocli_config $modname kw-nokey false)
+	
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>${OPENCONFIG}</CLICON_YANG_DIR>
+  <CLICON_YANG_AUGMENT_ACCEPT_BROKEN>true</CLICON_YANG_AUGMENT_ACCEPT_BROKEN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
+  ${AUTOCLI}
+</clixon-config>
+EOF
+
+	new "$clixon_cli -D $DBG  -1f $cfg -o CLICON_YANG_MAIN_FILE=$f show version"
 	expectpart "$($clixon_cli -D $DBG -1f $cfg -y $f show version)" 0 "${CLIXON_VERSION}"
     fi
 done

test/test_openconfig_interfaces.sh

@@ -19,6 +19,9 @@ fi
 
 OCDIR=$OPENCONFIG/release/models
 
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config clixon-example kw-nokey false)
+
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
@@ -31,11 +34,11 @@ cat <<EOF > $cfg
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf ietf-interfaces</CLICON_CLI_AUTOCLI_EXCLUDE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_openconfig_network_instance.sh

@@ -27,16 +27,29 @@ cat <<EOF > $cfg
   <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$OCDIR</CLICON_YANG_DIR>
   <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>	
-  <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf ietf-interfaces</CLICON_CLI_AUTOCLI_EXCLUDE>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
+  <autocli>
+     <module-default>false</module-default>
+     <list-keyword-default>kw-nokey</list-keyword-default>
+     <treeref-state-default>false</treeref-state-default>
+     <rule>
+       <name>openconfig1</name>
+       <operation>enable</operation>
+       <module-name>openconfig-bgp</module-name>
+     </rule>
+     <rule>
+       <name>openconfig2</name>
+       <operation>enable</operation>
+       <module-name>openconfig-network-instance</module-name>
+     </rule>
+  </autocli>
 </clixon-config>
 EOF
 
@@ -82,6 +95,25 @@ cat <<EOF > $dir/startup_db
 </config>
 EOF
 
+cat<<EOF > $dir/example_cli.cli
+# Clixon example specification
+CLICON_MODE="example";
+CLICON_PROMPT="%U@%H %W> ";
+CLICON_PLUGIN="example_cli";
+
+set @datamodel, cli_auto_set();
+save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename", "xml");{
+    cli("Save configuration as CLI commands"), save_config_file("candidate","filename", "cli");
+}
+show("Show a particular state of the system"){
+    configuration("Show configuration"), cli_auto_show("datamodel", "candidate", "xml", false, false);
+    version("Show version"), cli_show_version("candidate", "text", "/");
+}
+validate("Validate changes"), cli_validate();
+commit("Commit the changes"), cli_commit();
+quit("Quit"), cli_quit();
+EOF
+
 if [ $BE -ne 0 ]; then
     new "kill old backend"
     sudo clixon_backend -zf $cfg
@@ -101,14 +133,13 @@ new "$clixon_cli -D $DBG -1f $cfg show version"
 expectpart "$($clixon_cli -D $DBG -1f $cfg show version)" 0 "${CLIXON_VERSION}"
 
 new "$clixon_cli -D $DBG -1f $cfg save config as cli"
-expectpart "$($clixon_cli -D $DBG -1f $cfg save $dir/config.cli cli)" 0 "^$"
+expectpart "$($clixon_cli -D $DBG -1f $cfg save $dir/config.dump cli)" 0 "^$"
 
 new "Check saved config"
-expectpart "$(cat $dir/config.cli)" 0 "set network-instances network-instance default config type oc-ni-types:DEFAULT_INSTANCE" "set network-instances network-instance default config router-id 1.2.3.4"
+expectpart "$(cat $dir/config.dump)" 0 "set network-instances network-instance default config type oc-ni-types:DEFAULT_INSTANCE" "set network-instances network-instance default config router-id 1.2.3.4"
 
 new "load saved cli config"
-expectpart "$(cat $dir/config.cli | $clixon_cli -D $DBG -f $cfg 2>&1 > /dev/null)" 0 "^$"
+expectpart "$(cat $dir/config.dump | $clixon_cli -D $DBG -f $cfg 2>&1 > /dev/null)" 0 "^$"
-#time cat $dir/config.cli | $clixon_cli -D $DBG -f $cfg
 
 if [ $BE -ne 0 ]; then
     new "Kill backend"

test/test_openconfig_system.sh

@@ -22,6 +22,9 @@ fi
 
 OCDIR=$OPENCONFIG/release/models
 
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config clixon-example kw-nokey false)
+
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
@@ -33,11 +36,11 @@ cat <<EOF > $cfg
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf ietf-interfaces</CLICON_CLI_AUTOCLI_EXCLUDE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>true</CLICON_MODULE_LIBRARY_RFC7895>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

test/test_yang_when.sh

@@ -20,7 +20,8 @@ else
     mkdir $clidir
 fi
 
-# Use yang in example
+# Generate autocli for these modules
+AUTOCLI=$(autocli_config ${APPNAME}\*  kw-nokey false)
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -33,13 +34,11 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
-  <CLICON_CLI_GENMODEL>2</CLICON_CLI_GENMODEL>
-  <CLICON_CLI_GENMODEL_TYPE>VARS</CLICON_CLI_GENMODEL_TYPE>
   <CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
   <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
-  <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE>
+  ${AUTOCLI}
 </clixon-config>
 EOF
 

yang/clixon/Makefile.in

@@ -47,7 +47,7 @@ YANGSPECS	+= clixon-lib@2021-11-11.yang      # 5.4
 YANGSPECS	+= clixon-rfc5277@2008-07-01.yang
 YANGSPECS	+= clixon-xml-changelog@2019-03-21.yang
 YANGSPECS	+= clixon-restconf@2021-05-20.yang # 5.2
-YANGSPECS	+= clixon-clispec@2021-12-05.yang  # 5.5
+YANGSPECS	+= clixon-autocli@2021-12-05.yang  # 5.5
 
 all:	
 

yang/clixon/clixon-autocli@2021-12-05.yang

@@ -0,0 +1,202 @@
+module clixon-autocli{
+    yang-version 1.1;
+    namespace "http://clicon.org/autocli";
+    prefix autocli;
+
+    organization
+	"Clicon / Clixon";
+
+    contact
+	"Olof Hagsand <olof@hagsand.se>";
+
+    description
+	"Clixon CLIgen specification declarations, including autocli.
+         Design inspired by ietf-netconf-acm.yang
+      
+       ***** BEGIN LICENSE BLOCK *****
+       Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+       
+       This file is part of CLIXON
+
+       Licensed under the Apache License, Version 2.0 (the \"License\");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+            http://www.apache.org/licenses/LICENSE-2.0
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an \"AS IS\" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
+
+       Alternatively, the contents of this file may be used under the terms of
+       the GNU General Public License Version 3 or later (the \"GPL\"),
+       in which case the provisions of the GPL are applicable instead
+       of those above. If you wish to allow use of your version of this file only
+       under the terms of the GPL, and not to allow others to
+       use your version of this file under the terms of Apache License version 2, 
+       indicate your decision by deleting the provisions above and replace them with
+       the notice and other provisions required by the GPL. If you do not delete
+       the provisions above, a recipient may use your version of this file under
+       the terms of any one of the Apache License version 2 or the GPL.
+
+       ***** END LICENSE BLOCK *****";
+
+    revision 2021-12-05 {
+	description
+	    "Initial version";
+    }
+    typedef autocli-op {
+	description
+	    "Autocli rule-type operation, each rule use different fields as 
+             described in the individual enums below.";
+	type enumeration {
+	    enum enable {
+		description
+		    "Include a complete subtree to rendering of autocli. 
+                     Example: 
+                        <module-default>false</module-default>
+                        <rule>
+                           <name>wifi</name>
+                           <operation>enable</operation>
+                           <module-name>openconfig-wifi</module-name>
+                        </rule>
+                     Only on module-level and if module-default is false,
+                     Rule fields used: module-name";
+	    }
+	    enum compress {
+		description
+		"Skip a keyword from a command.
+                 Keep the command, only make it shorter by omitting a part.
+                 Example: compress containers if single list child
+                        <rule>
+                           <name>container compress</name>
+                           <operation>compress</operation>
+ 	                   <yang-keyword>container</yang-keyword>
+  	                   <yang-keyword-child>list</yang-keyword-child>
+                        </rule>
+                 Rule fields used: 
+                    module-name, yang-keyword, schema-nodeid, yang-keyword-child, extension";
+	    }
+	    /*--- NYI ----
+	    enum hide {
+		description
+		"A complete command (not just single keyword) is hidden from CLI query,
+                 help and completion, ie a user must type it manually. 
+                 Example: 'start shell'";
+	    }
+	    */
+	}
+    }
+    typedef list-keyword-type {
+	description
+	    "Autocli CLI keyword behaviour in YANG lists. 
+             With 'keyword' is meant CLIgen 'constants' rather than 'variables'.
+             Assume a YANG LIST: list a{ key x; leaf x; leaf y;} and how to generate
+             the autocli";
+	type enumeration {
+	    enum kw-none{
+		description "No extra keywords, only variables: a <x> <y>";
+	    }
+	    enum kw-nokey{
+		description "Keywords on non-key variables: a <x> y <y>";
+	    }
+	    enum kw-all{
+		description "Keywords on all variables: a x <x> y <y>";
+	    }
+	}
+    }
+    grouping clixon-autocli{
+	/* options */
+	leaf module-default {
+	    description
+		"Include YANG modules for generation of autocli.
+                 If true, all modules with a top-level datanode are generated, ie
+                 they get a top-level entry in the @basemodel tree.
+                 If false, you need to explicitly enable modules for autocli generation
+                 using 'enable' rules";
+	    type boolean;
+	    default true;
+	}
+	leaf list-keyword-default {
+	    description
+		"Autocli CLI keyword behaviour in YANG lists.";
+	    type list-keyword-type;
+	    default kw-nokey;
+	}
+	leaf treeref-state-default {
+	    description
+		"If 'true', generate CLI from YANG state/non-config statements as well, not only config data. 
+                 Many specs have very large state parts, for example openconfig has ca 10 times
+                 larger state than config parts, see for example openconfig-isis.yang.";
+	    type boolean;
+	    default false;
+	}
+	leaf completion-default {
+	    description
+		"Generate code for CLI completion of existing db symbols. 
+                 That is, check existing configure database for completion options.
+                 This is normally always enabled.";
+	    type boolean;
+	    default true;
+	}
+	/* rules */
+	list rule {
+	    description
+		"Represents a modification rule of a clixon clispec.";
+	    key name;
+	    leaf name {
+		description
+		    "Arbitrary name assigned for the rule, must be unique";
+		type string;
+	    }
+	    leaf description {
+		description
+		    "Rule description";
+		type string;
+	    }
+	    leaf operation {
+		description "Rule operation";
+		type autocli-op;
+	    }
+	    leaf module-name {
+		description
+		    "Name of the module associated with this rule.
+                     Wildchars '*' and '?' can be used (glob pattern).
+                     Revision and yang suffix are omitted
+                     Example: 'openconfig-*'";
+		type string;
+	    }
+	    leaf yang-keyword {
+		description
+		    "If present identifes a YANG keyword which the rule applies to
+                     Example: 'container'
+                     ";
+		type string;
+	    }
+	    leaf schema-nodeid {
+		description
+		    "path in the form of /<id>/<id> or just a single <id> identifying a YANG
+                     schema-node identifier as defined in RFC 7950 Sec 6.5
+                     Example: 'config', '/interfaces/interface'";
+		type string;
+	    }
+	    leaf yang-keyword-child {
+		description
+		    "The YANG statement has a single child, and the yang type of the child is the
+                     value of this option
+                     A (maybe too) specific property to cover openconfig compressions
+                     as defined here:
+                     https://github.com/openconfig/ygot/blob/master/docs/design.md#openconfig-path-compression";
+		type string;
+	    }
+	    leaf extension {
+		/* Consider making this a container with name/module/value instead */
+		description
+		    "The extension is set either in the node itself, or in this module
+                     Extension prefix must be set
+                     Example: oc-ext:openconfig-version";
+		type string;
+	    }
+	}
+    }
+}

yang/clixon/clixon-clispec@2021-12-05.yang

@@ -1,115 +0,0 @@
-module clixon-clispec{
-    yang-version 1.1;
-    namespace "http://clicon.org/clispec";
-    prefix ccli;
-
-    organization
-	"Clicon / Clixon";
-
-    contact
-	"Olof Hagsand <olof@hagsand.se>";
-
-    description
-	"Clixon CLIgen specification declarations, including autocli.
-      
-       ***** BEGIN LICENSE BLOCK *****
-       Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
-       
-       This file is part of CLIXON
-
-       Licensed under the Apache License, Version 2.0 (the \"License\");
-       you may not use this file except in compliance with the License.
-       You may obtain a copy of the License at
-            http://www.apache.org/licenses/LICENSE-2.0
-       Unless required by applicable law or agreed to in writing, software
-       distributed under the License is distributed on an \"AS IS\" BASIS,
-       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-       See the License for the specific language governing permissions and
-       limitations under the License.
-
-       Alternatively, the contents of this file may be used under the terms of
-       the GNU General Public License Version 3 or later (the \"GPL\"),
-       in which case the provisions of the GPL are applicable instead
-       of those above. If you wish to allow use of your version of this file only
-       under the terms of the GPL, and not to allow others to
-       use your version of this file under the terms of Apache License version 2, 
-       indicate your decision by deleting the provisions above and replace them with
-       the notice and other provisions required by the GPL. If you do not delete
-       the provisions above, a recipient may use your version of this file under
-       the terms of any one of the Apache License version 2 or the GPL.
-
-       ***** END LICENSE BLOCK *****";
-
-    revision 2021-12-05 {
-	description
-	    "Initial version";
-    }
-    
-    grouping clixon-clispec{
-	list rule {
-	    description
-		"Represents a modification rule of a clixon clispec.";
-	    ordered-by user; /* eg modules: omit * */
-	    key id;
-
-	    leaf id {
-		description
-		    "Arbitrary string index for the rule";
-		type string;
-	    }
-	    leaf description {
-		description
-		    "Rule description";
-		type string;
-	    }
-	    leaf-list node-id {
-		description
-		    "path in the form of <id>/<id> or just a single <id> identifying a YANG
-                     schema-node identifier as defined in RFC 7950 Sec 6.5
-                     Example: 'config', '/interfaces/interface'";
-		type string;
-	    }
-	    leaf-list module {
-		description
-		    "Constraints the rule to the module names identified by a glob.
-                     Wildchars '*' and '?' can be used.
-                     Revision and yang suffix are ommitted
-                     Note, this special rule is necessary since node-id cannot be used for
-                     modules.
-                     Example: 'openconfig-*'";
-		type string;
-	    }
-	    leaf-list yang-keyword {
-		description
-		    "If present identifes a YANG keyword which the rule applies to
-                     Example: 'container'";
-		type string;
-	    }
-	    leaf operation {
-		type enumeration {
-		    enum exclude {
-			description
-			    "Exclude a subtree from rendering of autocli. 
-                             Example: exclude: openconfig-* module";
-		    }
-		    enum include {
-			description
-			    "Include a subtree to rendering of autocli. 
-                             Example: include: openconfig-wifi module";
-		    }
-		    enum omit {
-			description
-			    "Remove a single keyword from a command. 
-                             Example: replace 'interfaces interface' with 'interface' ??";
-		    }
-		    enum hide {
-			description
-			"A complete command (not just single keyword) is hidden from CLI query,
-                         help and completion, ie a user must type it manually. 
-                         Example: 'start shell'";
-		    }
-		}
-	    }
-	}
-    }
-}

yang/clixon/clixon-config@2021-12-05.yang

@@ -6,8 +6,8 @@ module clixon-config {
     import clixon-restconf {
 	prefix clrc;
     }
-    import clixon-clispec {
+    import clixon-autocli {
-	prefix ccli;
+	prefix autocli;
     }    
     organization
 	"Clicon / Clixon";
@@ -48,11 +48,16 @@ module clixon-config {
 
     revision 2021-12-05 {
 	description
-	    "Added option:
+	    "Imported
-             Imported sub-spec:
+                    clixon-autocli.yang
-                    clixon-clispec.yang
+             Removed (previosly marked) obsolete options:
-             Removed obsolete options:
                     CLICON_YANG_LIST_CHECK
+             Marked as obsolete:
+                    CLICON_CLI_GENMODEL (use autocli/enable-autocli instead)
+                    CLICON_CLI_GENMODEL_TYPE (use autocli/list-keyword-default and compress rules instead)
+                    CLICON_CLI_GENMODEL_COMPLETION (use autocli/completion-default instead)
+                    CLICON_CLI_AUTOCLI_EXCLUDE (use autocli/exclude logic instead)
+                    CLICON_CLI_MODEL_TREENAME (use constant AUTOCLI_TREENAME instead)
              Released in Clixon 5.5";
     }
     revision 2021-11-11 {
@@ -375,8 +380,8 @@ module clixon-config {
 	container restconf {
 	    uses clrc:clixon-restconf;
 	}
-	container clispec {
+	container autocli {
-	    uses ccli:clixon-clispec;
+	    uses autocli:clixon-autocli;
 	}
 	leaf-list CLICON_FEATURE {
 	    description
@@ -639,7 +644,9 @@ module clixon-config {
                     This CLI tree can be accessed in CLI-spec files using the tree reference syntax (eg
                      @datamodel).
                  2: Same including state syntax in a tree called @datamodelstate and @datamodelshow
-                 See also CLICON_CLI_MODEL_TREENAME.";
+                 See also CLICON_CLI_MODEL_TREENAME.
+                 Obsolete, use clixon-autocli.yang enable-autocli instead";
+	    status obsolete;
 	}
 	leaf CLICON_CLI_MODEL_TREENAME {
 	    type string;
@@ -654,18 +661,21 @@ module clixon-config {
                  Example: set @datamodel, cli_set();
                           show @datamodelshow, cli_show_auto();
                           show state @datamodelstate, cli_show_auto_state();
+                 Obsolete, use constant AUTOCLI_TREENAME instead;
                  ";
+	    status obsolete;
 	}
 	leaf CLICON_CLI_GENMODEL_COMPLETION {
 	    type int32;
 	    default 1;
 	    description "Generate code for CLI completion of existing db symbols.
-                         (consider boolean)";
+                         Obsolete, use autocli/completion-default instead";
+	    status obsolete;
 	}
 	leaf CLICON_CLI_GENMODEL_TYPE {
 	    type cli_genmodel_type;
 	    default "VARS";
-	    description "How to generate and show auto CLI syntax: VARS|ALL|HIDE";
+	    description "How to generate and show auto CLI syntax: VARS|ALL|HIDE|OC_COMPRESS";
 	}
 	leaf CLICON_CLI_AUTOCLI_EXCLUDE {
 	    type string;
@@ -674,8 +684,9 @@ module clixon-config {
                  Example: 
                     <CLICON_CLI_AUTOCLI_EXCLUDE>clixon-restconf</CLICON_CLI_AUTOCLI_EXCLUDE> 
                  means generate autocli for all models except clixon-restconf.yang
-                 The value can be a list of space separated module names";
+                 The value can be a list of space separated module names
-	    default "clixon-restconf ietf-yang-library";
+                 Obsolete, use autocli/exclude logic instead";
+	    status obsolete;
 	}
 	leaf CLICON_CLI_VARONLY {
 	    type int32;