Text syntax parser/loader

Added support for list x y;
Uses a mechanism to parse as unknown XML body and post-parsing replace with list keys
Fixed example and test to work with new TEXT syntax

CHANGELOG.md

@@ -40,12 +40,23 @@ Planned: July 2022
 
 ### New features
 
-* Text syntax parser and loader
+* TEXT syntax parseable for loading files
-  * Added new text syntax parsing and loading from CLI
+  * Previously only supported output
-  * Text output format changed:
+  * TEXT output format changed (see API changes)
-    * Namespace/modulename added to top-level
+  * [Documentation](https://clixon-docs.readthedocs.io/en/latest/datastore.html#other-formats)
   * See [Support performant load_config_file(...) for TEXT format](https://github.com/clicon/clixon/issues/324)
 	
+### API changes on existing protocol/config features
+
+Users may have to change how they access the system
+
+* TEXT file format changed
+  * With new parsing of TEXT format, the output is changed
+    * Namespace/modulename added to top-level
+    * Leaf-list support: `a [ x y z ]`
+    * List key support: `a x y { ... }`
+    * See compile-time option `TEXT_LIST_KEYS`
+
 ### C/CLI-API changes on existing features
 
 Developers may need to change their code

apps/cli/cli_common.c

@@ -832,6 +832,7 @@ load_config_file(clicon_handle h,
     enum format_enum format = FORMAT_XML;
     yang_stmt       *yspec;
     cxobj           *xerr = NULL; 
+    char            *lineptr = NULL;
 
     if (cvec_len(argv) < 2 || cvec_len(argv) > 4){
 	clicon_err(OE_PLUGIN, EINVAL, "Received %d arguments. Expected: <dbname>,<varname>[,<format>]",
@@ -891,7 +892,10 @@ load_config_file(clicon_handle h,
 	}
 	break;
     case FORMAT_TEXT:
-	if ((ret = clixon_text_syntax_parse_file(fp, YB_NONE, yspec, &xt, &xerr)) < 0)
+	/* text parser requires YANG and since load/save files have a "config" top-level 
+	 * the yang-bind parameter must be YB_MODULE_NEXT
+	 */
+	if ((ret = clixon_text_syntax_parse_file(fp, YB_MODULE_NEXT, yspec, &xt, &xerr)) < 0)
 	    goto done;
 	if (ret == 0){
 	    clixon_netconf_error(xerr, "Loading", filename);
@@ -903,7 +907,6 @@ load_config_file(clicon_handle h,
 	    char         *mode = cli_syntax_mode(h);
 	    cligen_result result;            /* match result */
 	    int           evalresult = 0;    /* if result == 1, calback result */
-	    char         *lineptr = NULL;
 	    size_t        n;
 
 	    while(!cligen_exiting(cli_cligen(h))) {
@@ -952,6 +955,8 @@ load_config_file(clicon_handle h,
  ok:
     ret = 0;
  done:
+    if (lineptr)
+	free(lineptr); 
    if (xerr)
 	xml_free(xerr);
     if (xt)

example/main/example_cli.cli

@@ -108,16 +108,16 @@ save("Save candidate configuration to XML file") <filename:string>("Filename (lo
 }
 load("Load configuration from XML file") <filename:string>("Filename (local filename)"),load_config_file("filename", "replace");{
     replace("Replace candidate with file contents"), load_config_file("filename", "replace");{
-	cli("Replace candidate with file containing CLI commands"), load_config_file("","filename", "replace", "cli");
+	cli("Replace candidate with file containing CLI commands"), load_config_file("filename", "replace", "cli");
-	xml("Replace candidate with file containing XML"), load_config_file("","filename", "replace", "xml");
+	xml("Replace candidate with file containing XML"), load_config_file("filename", "replace", "xml");
-	json("Replace candidate with file containing JSON"), load_config_file("","filename", "replace", "json");
+	json("Replace candidate with file containing JSON"), load_config_file("filename", "replace", "json");
-	text("Replace candidate with file containing TEXT"), load_config_file("","filename", "replace", "text");
+	text("Replace candidate with file containing TEXT"), load_config_file("filename", "replace", "text");
     }
     merge("Merge file with existent candidate"), load_config_file("filename", "merge");{
-	cli("Merge candidate with file containing CLI commands"), load_config_file("","filename", "merge", "cli");
+	cli("Merge candidate with file containing CLI commands"), load_config_file("filename", "merge", "cli");
-	xml("Merge candidate with file containing XML"), load_config_file("","filename", "merge", "xml");
+	xml("Merge candidate with file containing XML"), load_config_file("filename", "merge", "xml");
-	json("Merge candidate with file containing JSON"), load_config_file("","filename", "merge", "json");
+	json("Merge candidate with file containing JSON"), load_config_file("filename", "merge", "json");
-	text("Merge candidate with file containing TEXT"), load_config_file("","filename", "merge", "text");
+	text("Merge candidate with file containing TEXT"), load_config_file("filename", "merge", "text");
     }
 }
 example("This is a comment") <var:int32>("Just a random number"), mycallback("myarg");

include/clixon_custom.h

@@ -165,8 +165,16 @@
 #define PROTO_RESTART_RECONNECT
 
 /*! Text output keys as identifiers instead of ordinary leafs
- * Ie:        list a { val 42; }      If defined
+ * That is, given list "list" with key value "a", if set, the output of show config or save
- * instead of list { keyname a; val 42; } If undefined
+ * as text command is:
- * Problem is, Parser is YANG unaware therefore doe not know "keyname"
+ *    list a { 
+ *       val 42; 
+ *    }
+ * If not set, the output is:
+ *    list { 
+ *       keyname a; 
+ *       val 42; 
+ *    }
+ * The TEXT parser (ie load) accepts both formats.
  */
-#undef TEXT_LIST_KEYS
+#define TEXT_LIST_KEYS

lib/clixon/clixon_xml.h

@@ -178,6 +178,7 @@ typedef struct clixon_xml_vec clixon_xvec; /* struct defined in clicon_xml_vec.c
 #define XML_FLAG_NONE      0x20 /* Node is added as NONE */
 #define XML_FLAG_DEFAULT   0x40 /* Added when a value is set as default @see xml_default */
 #define XML_FLAG_TOP       0x80 /* Top datastore symbol */
+#define XML_FLAG_BODYKEY  0x100 /* Text parsing key to be translated from body to key */
 
 /*
  * Prototypes

lib/src/clixon_text_syntax.c

@@ -274,6 +274,69 @@ clixon_txt2file(FILE             *f,
     return retval;
 }
 
+/*! Look for YANG lists nodes and convert bodies to keys
+ *
+ * This is a compromise between making the text parser (1) YANG aware or (2) not.
+ * (1) The reason for is that some constructs such as "list <keyval1> {" does not
+ * contain enough info (eg name of XML tag for <keyval1>)
+ * (2) The reason against is of principal of making the parser design simpler in a bottom-up mode
+ * The compromise between (1) and (2) is to first parse without YANG (2)  and then call a special
+ * function after YANG binding to populate key tags properly.
+ * @param[in]  x   XML node
+ * @see TEXT_LIST_KEYS which controls output/save, whereas this is parsing/input
+ * @see text_mark_bodies where marking of bodies made transformed here
+ */
+static int
+text_populate_list(cxobj *xn)
+{
+    int        retval = -1;
+    yang_stmt *yn;
+    yang_stmt *yc;
+    cxobj     *xc;
+    cxobj     *xb;
+    cvec      *cvk; /* vector of index keys */
+    cg_var    *cvi = NULL;
+    char      *namei;
+
+    if ((yn = xml_spec(xn)) == NULL)
+	goto ok;
+    if (yang_keyword_get(yn) == Y_LIST){
+	cvk = yang_cvec_get(yn);
+	/* Loop over bodies and keys and create key leafs 
+	 */
+	cvi = NULL;
+	xb = NULL;
+	while ((xb = xml_find_type(xn, NULL, NULL, CX_BODY)) != NULL) {
+	    if (!xml_flag(xb, XML_FLAG_BODYKEY))
+		continue;
+	    xml_flag_reset(xb, XML_FLAG_BODYKEY);
+	    if ((cvi = cvec_next(cvk, cvi)) == NULL){
+		clicon_err(OE_XML, 0, "text parser, key and body mismatch");
+		goto done;
+	    }
+	    namei = cv_string_get(cvi);
+	    if ((xc = xml_new(namei, xn, CX_ELMNT)) == NULL)
+		goto done;
+	    yc = yang_find(yn, Y_LEAF, namei);
+	    xml_spec_set(xc, yc);
+	    if ((xml_addsub(xc, xb)) < 0)
+		goto done;
+
+	}
+	if (xml_sort(xn) < 0)
+	    goto done;
+    }
+    xc = NULL;
+    while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL) {    
+	if (text_populate_list(xc) < 0)
+	    goto done;
+    }
+ ok:
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Parse a string containing text syntax and return an XML tree
  *
 
@@ -283,11 +346,11 @@ clixon_txt2file(FILE             *f,
  * @param[in]  yspec  Yang specification (if rfc 7951)
  * @param[out] xt     XML top of tree typically w/o children on entry (but created)
  * @param[out] xerr   Reason for invalid returned as netconf err msg 
- * 
- * @see _xml_parse for XML variant
  * @retval        1   OK and valid
  * @retval        0   Invalid (only if yang spec)
  * @retval       -1   Error with clicon_err called
+ * @see _xml_parse for XML variant
+ * @note Parsing requires YANG, which means yb must be YB_MODULE/_NEXT
  */
 static int 
 _text_syntax_parse(char      *str, 
@@ -302,8 +365,13 @@ _text_syntax_parse(char      *str,
     cxobj                  *x;
     cbuf                   *cberr = NULL;
     int                     failed = 0; /* yang assignment */
+    cxobj                  *xc;
     
     clicon_debug(1, "%s %d %s", __FUNCTION__, yb, str);
+    if (yb != YB_MODULE && yb != YB_MODULE_NEXT){
+	clicon_err(OE_YANG, EINVAL, "yb must be YB_MODULE or YB_MODULE_NEXT");
+	return -1;
+    }
     ts.ts_parse_string = str;
     ts.ts_linenum = 1;
     ts.ts_xtop = xt;
@@ -322,17 +390,6 @@ _text_syntax_parse(char      *str,
 	/* Populate, ie associate xml nodes with yang specs 
 	 */
 	switch (yb){
-	case YB_NONE:
-	    break;
-	case YB_PARENT:
-	    /* xt:n         Has spec
-	     * x:   <a> <-- populate from parent
-	     */
-	    if ((ret = xml_bind_yang0(x, YB_PARENT, NULL, xerr)) < 0)
-		goto done;
-	    if (ret == 0)
-		failed++;
-	    break;
 	case YB_MODULE_NEXT:
 	    if ((ret = xml_bind_yang(x, YB_MODULE, yspec, xerr)) < 0)
 		goto done;
@@ -348,19 +405,18 @@ _text_syntax_parse(char      *str,
 	    if (ret == 0)
 		failed++;
 	    break;
-	case YB_RPC:
+	default: /* shouldnt happen */
-	    if ((ret = xml_bind_yang_rpc(x, yspec, xerr)) < 0)
-		goto done;
-	    if (ret == 0){ /* Add message-id */
-		if (*xerr && clixon_xml_attr_copy(x, *xerr, "message-id") < 0)
-		    goto done;
-		failed++;
-	    }
 	    break; 
 	} /* switch */
+	/*! Look for YANG lists nodes and convert bodies to keys */
+	xc = NULL;
+	while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) 
+	    if (text_populate_list(xc) < 0)
+		goto done;
     }
     if (failed)
 	goto fail;
+
     /* Sort the complete tree after parsing. Sorting is not really meaningful if Yang 
        not bound */
     if (yb != YB_NONE)
@@ -439,6 +495,7 @@ clixon_text_syntax_parse_string(char      *str,
  * @note  you need to free the xml parse tree after use, using xml_free()
  * @note, If xt empty, a top-level symbol will be added so that <tree../> will be:  <top><tree.../></tree></top>
  * @note May block on file I/O
+ * @note Parsing requires YANG, which means yb must be YB_MODULE/_NEXT
  *
  * @retval        1     OK and valid
  * @retval        0     Invalid (only if yang spec) w xerr set
@@ -462,7 +519,7 @@ clixon_text_syntax_parse_file(FILE      *fp,
     char      ch;
     int       len = 0;
 
-    if (xt==NULL){
+    if (xt == NULL){
 	clicon_err(OE_XML, EINVAL, "xt is NULL");
 	return -1;
     }

lib/src/clixon_text_syntax_parse.y

@@ -45,7 +45,8 @@
 %type <stack>  stmt
 %type <stack>  id
 %type <string> value
-%type <stack>  leaflist
+%type <stack>  values
+%type <string> substr
 
 %start top
 
@@ -117,6 +118,8 @@ text_add_value(cxobj *xn,
     return xb;
 }
 
+/*! Create XML node prefix:id
+ */
 static cxobj*
 text_create_node(clixon_text_syntax_yacc *ts,
 		 char                    *name)
@@ -158,7 +161,7 @@ strjoin(char *str0,
     size_t len0;
     size_t len;
 
-    len0 = strlen(str0);
+    len0 = str0?strlen(str0):0;
     len = len0 + strlen(str1) + 1;
     if ((str0 = realloc(str0, len)) == NULL){
 	clicon_err(OE_YANG, errno, "realloc");
@@ -168,10 +171,10 @@ strjoin(char *str0,
     return str0;
 }
 
-/*! Given a vector of XML bodies, transform it to a vector of ELEMENT entries
+/*! Given a vector of XML bodies, transform it to a vector of ELEMENT entries copied from x1
  */
 static int
-text_leaflist_create(clixon_xvec *xvec0,
+text_element_create(clixon_xvec *xvec0,
 		     cxobj       *x1,
 		     clixon_xvec *xvec1)
 {
@@ -194,19 +197,33 @@ text_leaflist_create(clixon_xvec *xvec0,
     return retval;
 }
 
+/*! Special mechanism to mark bodies so they will not be filtered as whitespace
+ * @see strip_body_objects text_populate_list
+ */
+static int
+text_mark_bodies(clixon_xvec *xv)
+{
+    int    i;
+    cxobj *xb;
+    
+    for (i=0; i<clixon_xvec_len(xv); i++){
+	xb = clixon_xvec_i(xv, i);
+	xml_flag_set(xb, XML_FLAG_BODYKEY);
+    }
+    return 0;
+}
+ 
 %} 
  
 %%
 
 top        : stmt MY_EOF       { _PARSE_DEBUG("top->stmt");
-     //                                 if (xml_addsub(_TS->ts_xtop, $1) < 0) YYERROR;
 				 if (clixon_child_xvec_append(_TS->ts_xtop, $1) < 0) YYERROR;
 				 clixon_xvec_free($1);
                                  YYACCEPT; }
            ;
 
 stmts      : stmts stmt        { _PARSE_DEBUG("stmts->stmts stmt");
-     //                                 if (clixon_xvec_append($1, $2) < 0) YYERROR;
                                  if (clixon_xvec_merge($1, $2) < 0) YYERROR;
 				 clixon_xvec_free($2);
 				 $$ = $1;
@@ -216,28 +233,26 @@ stmts      : stmts stmt        { _PARSE_DEBUG("stmts->stmts stmt");
                                }
            ;
 
-stmt       : id value ';'      { _PARSE_DEBUG("stmt-> id value ;");
+stmt       : id values ';'     { _PARSE_DEBUG("stmt-> id value ;");
-                                 if (text_add_value($1, $2) == NULL) YYERROR;
+                                 text_mark_bodies($2);
-				 free($2);
+                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
+				 if (text_element_create($$, $1, $2) < 0) YYERROR;
+				 xml_free($1);
+				 clixon_xvec_free($2);
+                               }
+           | id values '{' stmts '}'  { _PARSE_DEBUG("stmt-> id values { stmts }");
+                                 text_mark_bodies($2);
+     				 if (clixon_child_xvec_append($1, $2) < 0) YYERROR;
+				 clixon_xvec_free($2);
+     				 if (clixon_child_xvec_append($1, $4) < 0) YYERROR;
+				 clixon_xvec_free($4);
                                  if (($$ = clixon_xvec_new()) == NULL) YYERROR;
 				 if (clixon_xvec_append($$, $1) < 0) YYERROR;
 	                       }
-           | id '"' value '"' ';' { _PARSE_DEBUG("stmt-> id \" value \" ;");
+           | id '[' values ']'
-                                    if (text_add_value($1, $3) == NULL) YYERROR;
+		               { _PARSE_DEBUG("stmt-> id [ values ]");
-				    free($3);
                                  if (($$ = clixon_xvec_new()) == NULL) YYERROR;
-				    if (clixon_xvec_append($$, $1) < 0) YYERROR;
+				 if (text_element_create($$, $1, $3) < 0) YYERROR;
-                                  }
-           | id '{' stmts '}'  { _PARSE_DEBUG("stmt-> id { stmts }");
-				 if (clixon_child_xvec_append($1, $3) < 0) YYERROR;
-				 clixon_xvec_free($3);
-                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
-				 if (clixon_xvec_append($$, $1) < 0) YYERROR;
- 	                       }
-           | id '[' leaflist ']'
-		               { _PARSE_DEBUG("stmt-> id [ leaflist ]");
-                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
-				 if (text_leaflist_create($$, $1, $3) < 0) YYERROR;
 				 xml_free($1);
 				 clixon_xvec_free($3);
 	                       }
@@ -249,25 +264,34 @@ id         : TOKEN             { _PARSE_DEBUG("id->TOKEN");
                                }
            ;
 
-value      : value TOKEN       { _PARSE_DEBUG("value->value TOKEN");
+/* Array of body objects, possibly empty */
-                                 $$ = strjoin($1, $2); free($2);}
+values     : values value      { _PARSE_DEBUG("values->values value");
-           | TOKEN             { _PARSE_DEBUG("value->TOKEN");
-                                 $$ = $1; } 
-           ;
-
-leaflist   : leaflist TOKEN    { _PARSE_DEBUG("leaflist->leaflist TOKEN");
                                  cxobj* x;
 				 if ((x = text_add_value(NULL, $2)) == NULL) YYERROR;;
 				 free($2);
                                  if (clixon_xvec_append($1, x) < 0) YYERROR;
 				 $$ = $1;
                                }
-           |                   { _PARSE_DEBUG("leaflist->");
+           |                   { _PARSE_DEBUG("values->value");
                                  if (($$ = clixon_xvec_new()) == NULL) YYERROR;
 	                       }
 	   ;
 
+/* Returns single string either as a single token or contained by double quotes  */
+value      : TOKEN             { _PARSE_DEBUG("value->TOKEN");
+                                 $$=$1;
+                               }
+           | '"' substr '"'  { _PARSE_DEBUG("value-> \" substr \"");
+                                 $$=$2;
+	                       }
+           ;
 
+/* Value within quotes merged to single string, has separate lexical scope  */
+substr     : substr TOKEN      { _PARSE_DEBUG("substr->substr TOKEN");
+                                 $$ = strjoin($1, $2); free($2);}
+           |                   { _PARSE_DEBUG("substr->");
+                                 $$ = NULL; } 
+           ;
 
 %%
 

lib/src/clixon_xml.c

@@ -1746,7 +1746,7 @@ xml_find_type(cxobj           *xt,
 	}
 	else
 	    pmatch = 1;
-	if (pmatch && strcmp(name, xml_name(x)) == 0)
+	if (pmatch && (name==NULL || strcmp(name, xml_name(x)) == 0))
 	    return x;
     }
     return NULL;
@@ -2010,7 +2010,7 @@ xml_dup(cxobj *x0)
  *     free(xvec);
  * @endcode
  * @see cxvec_prepend
- * @see clixon_cxvec_append  which is its own encapsulated xml vector datatype
+ * @see clixon_xvec_append  which is its own encapsulated xml vector datatype
  */
 int
 cxvec_append(cxobj   *x, 
@@ -2045,8 +2045,8 @@ cxvec_append(cxobj   *x,
  *  if (xvec)
  *     free(xvec);
  * @endcode
- * @see cxvec_prepend
+ * @see cxvec_append
- * @see clixon_cxvec_prepend  which is its own encapsulated xml vector datatype
+ * @see clixon_xvec_prepend  which is its own encapsulated xml vector datatype
  */
 int
 cxvec_prepend(cxobj   *x, 

lib/src/clixon_xml_bind.c

@@ -107,22 +107,28 @@ xml_bind_netconf_message_id_optional(int val)
 
 /*! After yang binding, bodies of containers and lists are stripped from XML bodies
  * May apply to other nodes?
+ * Exception for bodies marked with XML_FLAG_BODYKEY, see text syntax parsing
+ * @see text_mark_bodies
  */
 static int
-strip_whitespace(cxobj *xt)
+strip_body_objects(cxobj *xt)
 {
     yang_stmt    *yt;
     enum rfc_6020 keyword;
-    cxobj        *xc;
+    cxobj        *xb;
     
     if ((yt = xml_spec(xt)) != NULL){
 	keyword = yang_keyword_get(yt);
 	if (keyword == Y_LIST || keyword == Y_CONTAINER){
-	    xc = NULL;
+	    xb = NULL;
-	    while ((xc = xml_find_type(xt, NULL, "body", CX_BODY)) != NULL)
+	    /* Quits if marked object, assume all are same */
-		xml_purge(xc);
+	    while ((xb = xml_find_type(xt, NULL, "body", CX_BODY)) != NULL &&
+		   !xml_flag(xb, XML_FLAG_BODYKEY)){
+		xml_purge(xb);
 	    }
 	}
+    }
+
     return 0;
 }
 
@@ -367,7 +373,7 @@ xml_bind_yang(cxobj     *xt,
     cxobj *xc;         /* xml child */
     int    ret;
 
-    strip_whitespace(xt);
+    strip_body_objects(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	if ((ret = xml_bind_yang0(xc, yb, yspec, xerr)) < 0)
@@ -414,7 +420,7 @@ xml_bind_yang0_opt(cxobj     *xt,
 	goto fail;
     else if (ret == 2)     /* ret=2 for anyxml from parent^ */
     	goto ok;
-    strip_whitespace(xt);
+    strip_body_objects(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	/* It is xml2ns in populate_self_parent that needs improvement */
@@ -493,7 +499,7 @@ xml_bind_yang0(cxobj     *xt,
 	goto fail;
     else if (ret == 2)     /* ret=2 for anyxml from parent^ */
     	goto ok;
-    strip_whitespace(xt);
+    strip_body_objects(xt);
     xc = NULL;     /* Apply on children */
     while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
 	if ((ret = xml_bind_yang0_opt(xc, YB_PARENT, NULL, xerr)) < 0)

test/test_autocli_editmode.sh

@@ -300,7 +300,7 @@ edit table
 show config text
 EOF
 new "show config text"
-expectpart "$(cat $fin | $clixon_cli -f $cfg 2>&1)" 0 "parameter {" "name a;" "value 42;"
+expectpart "$(cat $fin | $clixon_cli -f $cfg 2>&1)" 0 "parameter a {" "value 42;"
 
 cat <<EOF > $fin
 edit table

test/test_cli.sh

@@ -72,10 +72,21 @@ show("Show a particular state of the system"){
     }
     configuration("Show configuration"), cli_auto_show("datamodel", "candidate", "text", true, false);{
 	    cli("Show configuration as CLI commands"), cli_auto_show("datamodel", "candidate", "cli", true, false, "set ");
+	    xml("Show configuration as XML"), cli_auto_show("datamodel", "candidate", "xml", true, false, "set ");
   }
 }
-save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename", "xml");
+save("Save candidate configuration to XML file") <filename:string>("Filename (local filename)"), save_config_file("candidate","filename", "xml"){
-load("Load configuration from XML file") <filename:string>("Filename (local filename)"),load_config_file("filename", "replace");
+    cli("Save configuration as CLI commands"), save_config_file("candidate","filename", "cli");
+    xml("Save configuration as XML"), save_config_file("candidate","filename", "xml");
+    json("Save configuration as JSON"), save_config_file("candidate","filename", "json");
+    text("Save configuration as TEXT"), save_config_file("candidate","filename", "text");
+}
+load("Load configuration from XML file") <filename:string>("Filename (local filename)"),load_config_file("filename", "replace");{
+	cli("Replace candidate with file containing CLI commands"), load_config_file("filename", "replace", "cli");
+	xml("Replace candidate with file containing XML"), load_config_file("filename", "replace", "xml");
+	json("Replace candidate with file containing JSON"), load_config_file("filename", "replace", "json");
+	text("Replace candidate with file containing TEXT"), load_config_file("filename", "replace", "text");
+}
 
 rpc("example rpc") <a:string>("routing instance"), example_client_rpc("");
 
@@ -154,7 +165,7 @@ new "cli success validate"
 expectpart "$($clixon_cli -1 -f $cfg -l o validate)" 0 "^$"
 
 new "cli compare diff"
-expectpart "$($clixon_cli -1 -f $cfg -l o show compare text)" 0 "+                ip 1.2.3.4;"
+expectpart "$($clixon_cli -1 -f $cfg -l o show compare text)" 0 "+            address 1.2.3.4"
 
 new "cli start shell"
 expectpart "$($clixon_cli -1 -f $cfg -l o shell echo foo)" 0 "foo" 
@@ -162,17 +173,19 @@ expectpart "$($clixon_cli -1 -f $cfg -l o shell echo foo)" 0 "foo"
 new "cli commit"
 expectpart "$($clixon_cli -1 -f $cfg -l o commit)" 0 "^$"
 
-new "cli save"
+for format in cli json text xml; do
-expectpart "$($clixon_cli -1 -f $cfg -l o save $dir/foo)" 0 "^$"
+    new "cli save $format"
+    expectpart "$($clixon_cli -1 -f $cfg -l o save $dir/foo $format)" 0 "^$"
 
-new "cli delete all"
+    new "cli delete all"
-expectpart "$($clixon_cli -1 -f $cfg -l o delete all)" 0 "^$"
+    expectpart "$($clixon_cli -1 -f $cfg -l o delete all)" 0 "^$"
 
-new "cli load"
+    new "cli load $format"
-expectpart "$($clixon_cli -1 -f $cfg -l o load $dir/foo)" 0 "^$"
+    expectpart "$($clixon_cli -1 -f $cfg -l o load $dir/foo $format)" 0 "^$"
 
-new "cli check load"
+    new "cli check load"
-expectpart "$($clixon_cli -1 -f $cfg -l o show conf cli)" 0 "interfaces interface eth/0/0 ipv4 enabled true"
+    expectpart "$($clixon_cli -1 -f $cfg -l o show conf cli)" 0 "interfaces interface eth/0/0 ipv4 enabled true"
+done
 
 new "cli debug set"
 expectpart "$($clixon_cli -1 -f $cfg -l o debug cli 1)" 0 "^$"

test/test_text_syntax.sh

@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-# Test: TEX syntax parser tests. 
-
-# Magic line must be first in script (see README.md)
-s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
-
-: ${clixon_util_text_syntax:=clixon_util_text_syntax}
-: ${clixon_util_xml:=clixon_util_xml}
-
-fyang=$dir/example.yang
-
-cat <<EOF > $fyang
-module example{
-   prefix ex;
-   namespace "urn:example:clixon";
-    /* Generic config data */
-   container table{
-      list parameter{
-         key "name index";
-	 leaf name{
-	    type string;
-	 }
-	 leaf index{
-	    type int32;
-	 }
-	 leaf value{
-	    type string;
-	 }
-	 leaf-list array{
-	    type string;
-         }
-      }
-   }
-}
-EOF
-
-cat <<EOF > $dir/x1.xml
-<table xmlns="urn:example:clixon">
-   <parameter>
-      <name>a</name>
-      <index>0</index>
-      <value>foo bar\n
-             description</value>
-   </parameter>
-   <parameter>
-      <name>b</name>
-      <index>17</index>
-      <value>bar:fie</value>
-      <array>bar</array>
-      <array>fie</array>
-      <array>foo</array>
-   </parameter>
-</table>
-EOF
-
-cat <<EOF > $dir/x1.txt
-example:table {
-    parameter {
-        name a;
-        index 0;
-        value "foo bar\n
-             description";
-    }
-    parameter {
-        name b;
-        index 17;
-        value bar:fie;
-        array [
-            bar
-            fie
-            foo
-       ]
-    }
-}
-EOF
-
-new "test params: -y $fyang"
-
-# No yang
-new "xml to txt"
-expectpart "$($clixon_util_xml -f $dir/x1.xml -y $fyang -oX -D $DBG > $dir/x2.txt)" 0 ""
-
-ret=$(diff $dir/x1.txt $dir/x2.txt)
-if [ $? -ne 0 ]; then
-    err1 "$ret" 
-fi
-
-new "txt to xml"
-expectpart "$($clixon_util_text_syntax -f $dir/x1.txt -y $fyang -D $DBG > $dir/x2.xml)" 0 ""
-
-ret=$(diff $dir/x1.xml $dir/x2.xml)
-if [ $? -ne 0 ]; then
-    err1 "XML" "$ret"
-fi
-
-rm -rf $dir
-
-# unset conditional parameters 
-unset clixon_util_text_syntax
-unset clixon_util_xml
-
-new "endtest"
-endtest

util/Makefile.in

@@ -85,7 +85,6 @@ LIBDEPS	   += $(top_srcdir)/lib/src/$(CLIXON_LIB)
 APPSRC    = clixon_util_xml.c
 APPSRC   += clixon_util_xml_mod.c
 APPSRC   += clixon_util_json.c
-APPSRC   += clixon_util_text_syntax.c
 APPSRC   += clixon_util_yang.c
 APPSRC   += clixon_util_xpath.c
 APPSRC   += clixon_util_path.c
@@ -153,9 +152,6 @@ clixon_util_validate: clixon_util_validate.c $(LIBDEPS)
 clixon_util_dispatcher: clixon_util_dispatcher.c $(LIBDEPS) 
 	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ -l clixon_backend -o $@ $(LIBS)
 
-clixon_util_text_syntax: clixon_util_text_syntax.c $(LIBDEPS) 
-	$(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $^ $(LIBS) -o $@
-
 ifdef with_restconf
 clixon_util_stream: clixon_util_stream.c $(LIBDEPS)
 	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -lcurl -o $@

util/clixon_util_text_syntax.c

@@ -1,168 +0,0 @@
-/*
- *
-  ***** BEGIN LICENSE BLOCK *****
- 
-  Copyright (C) 2022 Olof Hagsand and Rubicon Communications, LLC (Netgate)
-
-  This file is part of CLIXON.
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-
-  Alternatively, the contents of this file may be used under the terms of
-  the GNU General Public License Version 3 or later (the "GPL"),
-  in which case the provisions of the GPL are applicable instead
-  of those above. If you wish to allow use of your version of this file only
-  under the terms of the GPL, and not to allow others to
-  use your version of this file under the terms of Apache License version 2, 
-  indicate your decision by deleting the provisions above and replace them with
-  the  notice and other provisions required by the GPL. If you do not delete
-  the provisions above, a recipient may use your version of this file under
-  the terms of any one of the Apache License version 2 or the GPL.
-
-  ***** END LICENSE BLOCK *****
-
-  * Text syntax utility command
- */
-
-#ifdef HAVE_CONFIG_H
-#include "clixon_config.h" /* generated by config & autoconf */
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <string.h>
-#include <ctype.h>
-#include <limits.h>
-#include <stdint.h>
-#include <syslog.h>
-#include <signal.h>
-
-/* cligen */
-#include <cligen/cligen.h>
-
-/* clixon */
-#include "clixon/clixon.h"
-
-/* Command line options passed to getopt(3) */
-#define UTIL_TEXT_SYNTAX_OPTS "hD:f:tl:y:"
-
-/*
- * Text syntax parse and pretty print test program
- * Example compile:
-*/
-static int
-usage(char *argv0)
-{
-    fprintf(stderr, "usage:%s [options] JSON as input on stdin\n"
-	    "where options are\n"
-            "\t-h \t\tHelp\n"
-    	    "\t-D <level> \tDebug\n"
-	    "\t-f <file>\tTEXT syntax input file (overrides stdin)\n"
-	    "\t-t \t\tOutput as TEXT syntax (default is as XML)\n"
-	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n"
-	    "\t-y <filename> \tyang filename to parse (must be stand-alone)\n"	    ,
-	    argv0);
-    exit(0);
-}
-
-int
-main(int    argc,
-     char **argv)
-{
-    int        retval = -1;
-    cxobj     *xt = NULL;
-    cbuf      *cb = cbuf_new();
-    int        c;
-    int        logdst = CLICON_LOG_STDERR;
-    int        text_syntax_output = 0;
-    char      *yang_filename = NULL;
-    yang_stmt *yspec = NULL;
-    cxobj     *xerr = NULL; /* malloced must be freed */
-    int        ret;
-    int        dbg = 0;
-    char      *input_filename = NULL;
-    FILE      *fp = stdin; /* base file, stdin */
-    
-    optind = 1;
-    opterr = 0;
-    while ((c = getopt(argc, argv, UTIL_TEXT_SYNTAX_OPTS)) != -1)
-	switch (c) {
-	case 'h':
-	    usage(argv[0]);
-	    break;
-    	case 'D':
-	    if (sscanf(optarg, "%d", &dbg) != 1)
-		usage(argv[0]);
-	    break;
-	case 'f':
-	    input_filename = optarg;
-	    break;
-	case 't':
-	    text_syntax_output++;
-	    break;
-	case 'l': /* Log destination: s|e|o|f */
-	    if ((logdst = clicon_log_opt(optarg[0])) < 0)
-		usage(argv[0]);
-	    break;
-	case 'y':
-	    yang_filename = optarg;
-	    break;
-	default:
-	    usage(argv[0]);
-	    break;
-	}
-    clicon_log_init(__FILE__, dbg?LOG_DEBUG:LOG_INFO, logdst);
-    clicon_debug_init(dbg, NULL);
-
-    if (yang_filename){
-	if ((yspec = yspec_new()) == NULL)
-	    goto done;
-	if (yang_parse_filename(yang_filename, yspec) == NULL){
-	    fprintf(stderr, "yang parse error %s\n", clicon_err_reason);
-	    return -1;
-	}
-    }
-    if (input_filename){
-	if ((fp = fopen(input_filename, "r")) == NULL){
-	    clicon_err(OE_YANG, errno, "open(%s)", input_filename);	
-	    goto done;
-	}
-    }
-    if ((ret = clixon_text_syntax_parse_file(fp, yspec?YB_MODULE:YB_NONE, yspec, &xt, &xerr)) < 0)
-	goto done;
-    if (ret == 0){
-	xml_print(stderr, xerr);
-	goto done;
-    }
-    if (text_syntax_output){
-	if (clixon_txt2file(stdout, xt, 0, fprintf, 1) < 0)
-	    goto done;
-    }
-    else{
-	if (clixon_xml2cbuf(cb, xt, 0, 1, -1, 1) < 0)
-	    goto done;
-	fprintf(stdout, "%s", cbuf_get(cb));
-    }
-    fflush(stdout);
-    retval = 0;
- done:
-    if (yspec)
-	ys_free(yspec);
-    if (xt)
-	xml_free(xt);
-    if (cb)
-	cbuf_free(cb);
-    return retval;
-}