Text syntax parser/loader

Leaf-list syntax using: "id [ leaf leaf ...]" syntax for output and input
Variables using \n\r in text
Added clixon_xvec_merge()

CHANGELOG.md

@@ -40,7 +40,7 @@ Planned: July 2022
 
 ### New features
 
-* Text syntax parser/loader
+* Text syntax parser and loader
   * Added new text syntax parsing and loading from CLI
   * Text output format changed:
     * Namespace/modulename added to top-level

include/clixon_custom.h

@@ -163,3 +163,10 @@
  * If not set, client will exit
  */
 #define PROTO_RESTART_RECONNECT
+
+/*! Text output keys as identifiers instead of ordinary leafs
+ * Ie:        list a { val 42; }      If defined
+ * instead of list { keyname a; val 42; } If undefined
+ * Problem is, Parser is YANG unaware therefore doe not know "keyname"
+ */
+#undef TEXT_LIST_KEYS

lib/clixon/clixon_xml_vec.h

@@ -54,6 +54,7 @@ cxobj       *clixon_xvec_i(clixon_xvec *xv, int i);
 int          clixon_xvec_extract(clixon_xvec *xv, cxobj ***xvcec, int *xlen, int *xmax);
 int          clixon_xvec_append(clixon_xvec *xv, cxobj *x);
 int          clixon_xvec_prepend(clixon_xvec *xv, cxobj *x);
+int          clixon_xvec_merge(clixon_xvec *xv0, clixon_xvec *xv1);
 int          clixon_xvec_insert_pos(clixon_xvec *xv, cxobj *x, int i);
 int          clixon_xvec_rm_pos(clixon_xvec *xv, int i);
 int          clixon_xvec_print(FILE *f, clixon_xvec *xv);

lib/src/clixon_text_syntax.c

@@ -97,28 +97,36 @@ tleaf(cxobj *x)
 }
 
 /*! Translate XML to a "pseudo-code" textual format using a callback - internal function
- * @param[in]  xn     XML object to print
+ * @param[in]     xn       XML object to print
- * @param[in]  fn     Callback to make print function
+ * @param[in]     fn       Callback to make print function
- * @param[in]  f      File to print to
+ * @param[in]     f        File to print to
- * @param[in]  level  print 4 spaces per level in front of each line
+ * @param[in]     level    Print 4 spaces per level in front of each line
- * @see xml2txt  XXX why are these different?
+ * @param[in,out] leaflist Leaflist state for [] 
+ * leaflist state:
+ * 0: No leaflist
+ * 1: In leaflist
  */
-int
+static int
-xml2txt(cxobj            *xn,
+xml2txt1(cxobj            *xn,
-	    clicon_output_cb *fn,
+	 clicon_output_cb *fn,
-	    FILE             *f, 
+	 FILE             *f, 
-	    int               level)
+	 int               level,
+	 int              *leaflist)
 {
     cxobj     *xc = NULL;
     int        children=0;
     int        retval = -1;
     int        exist = 0;
     yang_stmt *yn;
-    yang_stmt *yp;
+    yang_stmt *yp = NULL;
     yang_stmt *ymod;
     yang_stmt *ypmod;
     char      *prefix = NULL;
     char      *value;
+#ifdef TEXT_LIST_KEYS
+    cg_var    *cvi;
+    cvec      *cvk = NULL; /* vector of index keys */
+#endif
 
     if (xn == NULL || fn == NULL){
 	clicon_err(OE_XML, EINVAL, "xn or fn is NULL");
@@ -141,40 +149,84 @@ xml2txt(cxobj            *xn,
 	}
 	else
 	    prefix = yang_argument_get(ymod);
+#ifdef TEXT_LIST_KEYS
+	if (yang_keyword_get(yn) == Y_LIST){
+	    if ((cvk = yang_cvec_get(yn)) == NULL){
+		clicon_err(OE_YANG, 0, "No keys");
+		goto done;
+	    }
+	}
+#endif
     }
     xc = NULL;     /* count children (elements and bodies, not attributes) */
     while ((xc = xml_child_each(xn, xc, -1)) != NULL)
 	if (xml_type(xc) == CX_ELMNT || xml_type(xc) == CX_BODY)
 	    children++;
-    if (!children){ /* If no children print line */
+    if (children == 0){ /* If no children print line */
 	switch (xml_type(xn)){
 	case CX_BODY:
 	    value = xml_value(xn);
 	    /* Add quotes if string contains spaces */
-	    if (index(value, ' ') != NULL)
+	    if (*leaflist)
+		(*fn)(f, "%*s%s\n", 4*level, "", xml_value(xn));
+	    else if (index(value, ' ') != NULL)
 		(*fn)(f, "\"%s\";\n", xml_value(xn));
 	    else
 		(*fn)(f, "%s;\n", xml_value(xn));
 	    break;
 	case CX_ELMNT:
-	    (*fn)(f, "%*s%s;\n", 4*level, "", xml_name(xn));
+	    (*fn)(f, "%*s%s", 4*level, "", xml_name(xn));
+#ifdef TEXT_LIST_KEYS
+	    cvi = NULL; 	    /* Lists only */
+	    while ((cvi = cvec_each(cvk, cvi)) != NULL) {
+		if ((xc = xml_find_type(xn, NULL, cv_string_get(cvi), CX_ELMNT)) != NULL)
+		    (*fn)(f, " %s", xml_body(xc));
+	    }
+#endif
+	    (*fn)(f, ";\n");
 	    break;
 	default:
 	    break;
 	}
 	goto ok;
     }
-    (*fn)(f, "%*s", 4*level, "");
+    if (*leaflist == 0){
-    if (prefix)
+	(*fn)(f, "%*s", 4*level, "");
-	(*fn)(f, "%s:", prefix);
+	if (prefix)
-    (*fn)(f, "%s ", xml_name(xn));
+	    (*fn)(f, "%s:", prefix);
-    if (!tleaf(xn))
+	(*fn)(f, "%s", xml_name(xn));
-	(*fn)(f, "{\n");
+    }
+#ifdef TEXT_LIST_KEYS
+    cvi = NULL; 	/* Lists only */
+    while ((cvi = cvec_each(cvk, cvi)) != NULL) {
+	if ((xc = xml_find_type(xn, NULL, cv_string_get(cvi), CX_ELMNT)) != NULL)
+	    (*fn)(f, " %s", xml_body(xc));
+    }
+#endif
+    if (yn && yang_keyword_get(yn) == Y_LEAF_LIST && *leaflist)
+	;
+    else if (yn && yang_keyword_get(yn) == Y_LEAF_LIST && *leaflist == 0){
+	*leaflist = 1;
+	(*fn)(f, " [\n");
+    }
+    else if (!tleaf(xn))
+	(*fn)(f, " {\n");
+    else
+	(*fn)(f, " ");
     xc = NULL;
     while ((xc = xml_child_each(xn, xc, -1)) != NULL){
-	if (xml_type(xc) == CX_ELMNT || xml_type(xc) == CX_BODY)
+	if (xml_type(xc) == CX_ELMNT || xml_type(xc) == CX_BODY){
-	    if (xml2txt(xc, fn, f, level+1) < 0)
+#ifdef TEXT_LIST_KEYS
+	    if (yang_key_match(yn, xml_name(xc), NULL))
+		continue; /* Skip keys, already printed */
+#endif
+	    if (xml2txt1(xc, fn, f, level+1, leaflist) < 0)
 		break;
+	}
+    }
+    if (yn && yang_keyword_get(yn) != Y_LEAF_LIST && *leaflist != 0){
+	*leaflist = 0;
+	(*fn)(f, "%*s\n", 4*(level+1), "]");
     }
     if (!tleaf(xn))
 	(*fn)(f, "%*s}\n", 4*level, "");
@@ -184,6 +236,23 @@ xml2txt(cxobj            *xn,
     return retval;
 }
 
+/*! Translate XML to a "pseudo-code" textual format using a callback
+ * @param[in]     xn       XML object to print
+ * @param[in]     fn       Callback to make print function
+ * @param[in]     f        File to print to
+ * @param[in]     level    Print 4 spaces per level in front of each line
+ */
+int
+xml2txt(cxobj            *xn,
+	clicon_output_cb *fn,
+	FILE             *f, 
+	int               level)
+{
+    int leaflist = 0;
+
+    return xml2txt1(xn, fn, f, level, &leaflist);
+}
+
 /*! Parse a string containing text syntax and return an XML tree
  *
 

lib/src/clixon_text_syntax_parse.l

@@ -92,7 +92,7 @@ int clixon_text_syntax_parsewrap(void)
 <INITIAL>\;               { return *yytext; }
 <INITIAL>\"               { _TS->ts_lex_state =INITIAL; BEGIN(STRING);  return *yytext; }
 <INITIAL>[^\n\r \t\[\]\{\}\;\"]+  {
-                          clixon_text_syntax_parselval.string = strdup(yytext);
+                            clixon_text_syntax_parselval.string = strdup(yytext);
                             return TOKEN; }
 <INITIAL>.                { return -1; }
 
@@ -100,10 +100,13 @@ int clixon_text_syntax_parsewrap(void)
 <COMMENT><<EOF>>          { return MY_EOF; }
 <COMMENT>[^\n]+
 
+<STRING>\n                { _TS->ts_linenum++;
+                            clixon_text_syntax_parselval.string = strdup(yytext);
+                            return TOKEN;}
 <STRING>\"                { BEGIN(_TS->ts_lex_state); return *yytext; }
-<STRING>[^\n\r\"]+        { clixon_text_syntax_parselval.string = strdup(yytext);
+<STRING>[^\n\"]+          { clixon_text_syntax_parselval.string = strdup(yytext);
                             return TOKEN; }
-<STRING>.                { return -1; }
+<STRING>.                 { return -1; }
 
 %%
 

lib/src/clixon_text_syntax_parse.y

@@ -45,6 +45,7 @@
 %type <stack>  stmt
 %type <stack>  id
 %type <string> value
+%type <stack>  leaflist
 
 %start top
 
@@ -80,7 +81,7 @@
 #include "clixon_text_syntax_parse.h"
 
 /* Enable for debugging, steals some cycles otherwise */
-#if 0
+#if 1
 #define _PARSE_DEBUG(s) clicon_debug(1,(s))
 #else
 #define _PARSE_DEBUG(s)
@@ -99,22 +100,21 @@ clixon_text_syntax_parseerror(void *arg,
     return;
 }
 
-static int
+static cxobj*
 text_add_value(cxobj *xn,
-	       char   *value)
+	       char  *value)
 {
-    int    retval = -1;
     cxobj *xb = NULL;
 
     if ((xb = xml_new("body", xn, CX_BODY)) == NULL)
 	goto done;
     if (xml_value_set(xb,  value) < 0){
 	xml_free(xb);
+	xb = NULL;
 	goto done;
     }
-    retval = 0;
  done:
-    return retval;
+    return xb;
 }
 
 static cxobj*
@@ -151,17 +151,64 @@ text_create_node(clixon_text_syntax_yacc *ts,
     return xn;
 }
 
+static char*
+strjoin(char *str0,
+        char *str1)
+{
+    size_t len0;
+    size_t len;
+
+    len0 = strlen(str0);
+    len = len0 + strlen(str1) + 1;
+    if ((str0 = realloc(str0, len)) == NULL){
+	clicon_err(OE_YANG, errno, "realloc");
+	return NULL;
+    }
+    strcpy(str0+len0, str1);
+    return str0;
+}
+
+/*! Given a vector of XML bodies, transform it to a vector of ELEMENT entries
+ */
+static int
+text_leaflist_create(clixon_xvec *xvec0,
+		     cxobj       *x1,
+		     clixon_xvec *xvec1)
+{
+    int    retval = -1;
+    cxobj *xb;
+    cxobj *x2;
+    int    i;
+
+    for (i=0; i<clixon_xvec_len(xvec1); i++){
+	xb = clixon_xvec_i(xvec1, i);
+	if ((x2 = xml_dup(x1)) == NULL)
+	    goto done;
+	if (xml_addsub(x2, xb) < 0)
+	    goto done;
+	if (clixon_xvec_append(xvec0, x2) < 0)
+	    goto done;
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+ 
 %} 
  
 %%
 
 top        : stmt MY_EOF       { _PARSE_DEBUG("top->stmt");
-                                 if (xml_addsub(_TS->ts_xtop, $1) < 0) YYERROR;
+     //                                 if (xml_addsub(_TS->ts_xtop, $1) < 0) YYERROR;
+				 if (clixon_child_xvec_append(_TS->ts_xtop, $1) < 0) YYERROR;
+				 clixon_xvec_free($1);
                                  YYACCEPT; }
            ;
 
 stmts      : stmts stmt        { _PARSE_DEBUG("stmts->stmts stmt");
-                                 if (clixon_xvec_append($1, $2) < 0) YYERROR;
+     //                                 if (clixon_xvec_append($1, $2) < 0) YYERROR;
+                                 if (clixon_xvec_merge($1, $2) < 0) YYERROR;
+				 clixon_xvec_free($2);
 				 $$ = $1;
                                } 
            |                   { _PARSE_DEBUG("stmts->stmt");
@@ -170,34 +217,57 @@ stmts      : stmts stmt        { _PARSE_DEBUG("stmts->stmts stmt");
            ;
 
 stmt       : id value ';'      { _PARSE_DEBUG("stmt-> id value ;");
-                                 if (text_add_value($1, $2) < 0) YYERROR;
+                                 if (text_add_value($1, $2) == NULL) YYERROR;
-				 $$ = $1;
+				 free($2);
+                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
+				 if (clixon_xvec_append($$, $1) < 0) YYERROR;
                                }
            | id '"' value '"' ';' { _PARSE_DEBUG("stmt-> id \" value \" ;");
-                                 if (text_add_value($1, $3) < 0) YYERROR;
+                                    if (text_add_value($1, $3) == NULL) YYERROR;
-				 $$ = $1;
+				    free($3);
-                               }
+                                    if (($$ = clixon_xvec_new()) == NULL) YYERROR;
+				    if (clixon_xvec_append($$, $1) < 0) YYERROR;
+                                  }
            | id '{' stmts '}'  { _PARSE_DEBUG("stmt-> id { stmts }");
 				 if (clixon_child_xvec_append($1, $3) < 0) YYERROR;
 				 clixon_xvec_free($3);
-				 $$ = $1;
+                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
+				 if (clixon_xvec_append($$, $1) < 0) YYERROR;
  	                       }
-
+           | id '[' leaflist ']'
-           | id '[' values ']' { _PARSE_DEBUG("stmt-> id [ values ]"); }
+		               { _PARSE_DEBUG("stmt-> id [ leaflist ]");
+                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
+				 if (text_leaflist_create($$, $1, $3) < 0) YYERROR;
+				 xml_free($1);
+				 clixon_xvec_free($3);
+	                       }
            ;
 
 id         : TOKEN             { _PARSE_DEBUG("id->TOKEN");
-                                  if (($$ = text_create_node(_TS, $1)) == NULL) YYERROR;;
+                                 if (($$ = text_create_node(_TS, $1)) == NULL) YYERROR;
+				 free($1);
                                }
            ;
 
-values     : values TOKEN      { _PARSE_DEBUG("values->values TOKEN"); } 
+value      : value TOKEN       { _PARSE_DEBUG("value->value TOKEN");
-           |                   { _PARSE_DEBUG("values->"); } 
+                                 $$ = strjoin($1, $2); free($2);}
+           | TOKEN             { _PARSE_DEBUG("value->TOKEN");
+                                 $$ = $1; } 
            ;
 
-value      : TOKEN             { _PARSE_DEBUG("value->TOKEN"); $$ = $1; } 
+leaflist   : leaflist TOKEN    { _PARSE_DEBUG("leaflist->leaflist TOKEN");
+                                 cxobj* x;
+				 if ((x = text_add_value(NULL, $2)) == NULL) YYERROR;;
+				 free($2);
+                                 if (clixon_xvec_append($1, x) < 0) YYERROR;
+				 $$ = $1;
+                               } 
+           |                   { _PARSE_DEBUG("leaflist->");
+                                 if (($$ = clixon_xvec_new()) == NULL) YYERROR;
+                               } 
            ;
 
 
+
 %%
 

lib/src/clixon_xml.c

@@ -1056,7 +1056,6 @@ clixon_child_xvec_append(cxobj       *xn,
     for (i=0; i<clixon_xvec_len(xv); i++){
 	xc = clixon_xvec_i(xv, i);
 	if (xml_addsub(xn, xc) < 0)
-	//	if (xml_child_append(xn, xc) < 0)
 	    goto done;
     }
     retval = 0;

lib/src/clixon_xml_vec.c

@@ -261,6 +261,36 @@ clixon_xvec_append(clixon_xvec *xv,
     return retval;
 }
 
+/*! Append a second clixon-xvec into a first
+ *
+ * @param[in,out] xv0   XML tree vector
+ * @param[in]     xv1   XML tree (append this to vector)
+ * @retval        0     OK, with xv0 with new entries from xv1
+ * @retval       -1     Error
+ * @code
+ *  if (clixon_xvec_merge(xv0, xv1) < 0) 
+ *     err;
+ * @endcode
+ */
+int
+clixon_xvec_merge(clixon_xvec *xv0,
+		  clixon_xvec *xv1)
+{
+    int    retval = -1;
+    cxobj *x;
+    int    i;
+
+    for (i=0; i<clixon_xvec_len(xv1); i++){
+	x = clixon_xvec_i(xv1, i);
+	if (clixon_xvec_inc(xv0) < 0)
+	    goto done;
+	xv0->xv_vec[xv0->xv_len-1] = x;
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Prepend a new xml tree to an existing xml vector first in the list
  *
  * @param[in]  xv    XML tree vector

test/test_text_syntax.sh

@@ -16,15 +16,21 @@ module example{
     /* Generic config data */
    container table{
       list parameter{
-         key name;
+         key "name index";
-	    leaf name{
+	 leaf name{
-	       type string;
+	    type string;
-	    }
+	 }
-	    leaf value{
+	 leaf index{
-		type string;
+	    type int32;
-	    }
+	 }
-        }
+	 leaf value{
-    }
+	    type string;
+	 }
+	 leaf-list array{
+	    type string;
+         }
+      }
+   }
 }
 EOF
 
@@ -32,11 +38,17 @@ cat <<EOF > $dir/x1.xml
 <table xmlns="urn:example:clixon">
    <parameter>
       <name>a</name>
-      <value>foo bar</value>
+      <index>0</index>
+      <value>foo bar\n
+             description</value>
    </parameter>
    <parameter>
       <name>b</name>
+      <index>17</index>
       <value>bar:fie</value>
+      <array>bar</array>
+      <array>fie</array>
+      <array>foo</array>
    </parameter>
 </table>
 EOF
@@ -45,11 +57,19 @@ cat <<EOF > $dir/x1.txt
 example:table {
     parameter {
         name a;
-        value "foo bar";
+        index 0;
+        value "foo bar\n
+             description";
     }
     parameter {
         name b;
+        index 17;
         value bar:fie;
+        array [
+            bar
+            fie
+            foo
+       ]
     }
 }
 EOF

util/clixon_util_xml.c

@@ -326,14 +326,6 @@ main(int    argc,
 	}
     }
 
-    /* Dump data structures (for debug) */
-    if (clicon_debug_get()){
-	cbuf_reset(cb);
-	xmltree2cbuf(cb, xt, 0);       
-	fprintf(stderr, "%s\n", cbuf_get(cb));
-	cbuf_reset(cb);
-    }
-
     /* 3. Validate data (if yspec) */
     if (validate){
 	if (validate_tree(h, xt, yspec) < 0)