Removing newlines from XML data [https://github.com/clicon/clixon/issues/65]

CHANGELOG.md

@@ -134,6 +134,7 @@
   * <!DOCTYPE (ie DTD) is not supported.
 
 ### Corrected Bugs
+* Removing newlines from XML data [https://github.com/clicon/clixon/issues/65]
 * [ietf-netconf-notification@2008-07-01.yang validation problem #62](https://github.com/clicon/clixon/issues/62)
 * Ignore CR(\r) in yang files for DOS files
 * Keyword "min" (not only "max") can be used in built-in types "range" and "length" statements.

lib/src/clixon_xml.c

@@ -1483,7 +1483,7 @@ _xml_parse(const char  *str,
 {
     int                       retval = -1;
     struct xml_parse_yacc_arg ya = {0,};
-
+    cxobj                    *x;
     if (strlen(str) == 0)
 	return 0; /* OK */
     if (xt == NULL){
@@ -1501,6 +1501,9 @@ _xml_parse(const char  *str,
 	goto done;    
     if (clixon_xml_parseparse(&ya) != 0)  /* yacc returns 1 on error */
 	goto done;
+    x = NULL;
+    while ((x = xml_find_type(xt, NULL, "body", CX_BODY)) != NULL)
+	xml_purge(x);
     /* Verify namespaces after parsing */
     if (xml_apply0(xt, CX_ELMNT, xml_localname_check, NULL) < 0)
     	goto done;

lib/src/clixon_xml_parse.l

@@ -105,8 +105,8 @@ ncname     {namestart}{namechar}*
 %%
 
 <START,TEXTDECL>[ \t]   ;
-<START,STATEA,CMNT,TEXTDECL>\n   { _YA->ya_linenum++; }
+<START,CMNT,TEXTDECL>\n   { _YA->ya_linenum++; }
-<START,STATEA,CMNT,TEXTDECL>\r
+<START,CMNT,TEXTDECL>\r
 
 <START>{ncname}        { clixon_xml_parselval.string = strdup(yytext);
                             return NAME; /* rather be catch-all */
@@ -133,6 +133,10 @@ ncname     {namestart}{namechar}*
 <STATEA>"<?"          { BEGIN(PIDECL); return BQMARK; }
 <STATEA>\<            { BEGIN(START); return *clixon_xml_parsetext; }
 <STATEA>&             { _YA->ya_lex_state =STATEA;BEGIN(AMPERSAND);}
+<STATEA>[ \t]         { clixon_xml_parselval.string = yytext;return WHITESPACE; }
+<STATEA>\r\n          { clixon_xml_parselval.string = "\n";return WHITESPACE; }
+<STATEA>\r            { clixon_xml_parselval.string = "\n";return WHITESPACE; }
+<STATEA>\n            { clixon_xml_parselval.string = yytext; _YA->ya_linenum++;return WHITESPACE; }
 <STATEA>.             { clixon_xml_parselval.string = yytext; return CHARDATA; }
 
 	/* @see xml_chardata_encode */

lib/src/clixon_xml_parse.y

@@ -41,7 +41,7 @@
 
 %start document
 
-%token <string> NAME CHARDATA STRING
+%token <string> NAME CHARDATA WHITESPACE STRING
 %token MY_EOF
 %token VER ENC SD
 %token BSLASH ESLASH
@@ -357,12 +357,13 @@ prolog      : xmldcl misclist
             ;
 
 misclist    : misclist misc { clicon_debug(2, "misclist->misclist misc"); }
-            |      { clicon_debug(2, "misclist->"); }
+            |     { clicon_debug(2, "misclist->"); }
             ;
 
 /* [27]	Misc ::=  Comment | PI | S */
-misc        : comment { clicon_debug(2, "misc->comment"); }
+misc        : comment    { clicon_debug(2, "misc->comment"); }
-	    | pi      { clicon_debug(2, "misc->pi"); }
+	    | pi         { clicon_debug(2, "misc->pi"); }
+            | WHITESPACE { clicon_debug(2, "misc->white space"); }
 	    ;
 
 xmldcl      : BXMLDCL verinfo encodingdecl sddecl EQMARK
@@ -424,6 +425,8 @@ content     : element      { clicon_debug(2, "content -> element"); }
             | pi           { clicon_debug(2, "content -> pi"); }
             | CHARDATA     { if (xml_parse_content(_YA, $1) < 0) YYABORT;  
                              clicon_debug(2, "content -> CHARDATA %s", $1); }
+            | WHITESPACE   { if (xml_parse_content(_YA, $1) < 0) YYABORT;  
+                             clicon_debug(2, "content -> WHITESPACE %s", $1); }
             |              { clicon_debug(2, "content -> "); }
             ;
 

test/test_xml.sh

@@ -3,14 +3,20 @@
 #  @see https://www.w3.org/TR/2008/REC-xml-20081126
 #       https://www.w3.org/TR/2009/REC-xml-names-20091208
 #PROG="valgrind --leak-check=full --show-leak-kinds=all ../util/clixon_util_xml"
-PROG=../util/clixon_util_xml
+
 
 # include err() and new() functions and creates $dir
 . ./lib.sh
 
+
+PROG="../util/clixon_util_xml -D $DBG"
+
 new "xml parse"
 expecteof "$PROG" 0 "<a><b/></a>" "^<a><b/></a>$"
 
+new "xml parse to json"
+expecteof "$PROG -j" 0 "<a><b/></a>" '^{"a": {"b": null}}$'
+
 new "xml parse strange names"
 expecteof "$PROG" 0 "<_-><b0.><c-.-._/></b0.></_->" "^<_-><b0.><c-.-._/></b0.></_->$"
 
@@ -23,6 +29,14 @@ expecteof "$PROG" 255 "<9/>" ""
 new "xml parse name errors"
 expecteof "$PROG" 255 "<a%/>" ""
 
+LF='
+'
+new "xml parse content with CR LF -> LF, CR->LF (see https://www.w3.org/TR/REC-xml/#sec-line-ends)"
+ret=$(echo "<x>a
b${LF}c
${LF}d</x>" | $PROG)
+if [ "$ret" != "<x>a${LF}b${LF}c${LF}d</x>" ]; then
+     err '<x>a$LFb$LFc</x>' "$ret"
+fi
+
 XML=$(cat <<EOF
 <a><description>An example of escaped CENDs</description>
 <sometext>
@@ -38,6 +52,7 @@ XML=$(cat <<EOF
 </a>
 EOF
 )
+
 new "xml CDATA"
 expecteof "$PROG" 0 "$XML" "^<a><description>An example of escaped CENDs</description><sometext>
 <![CDATA[ They're saying \"x < y\" & that \"z > y\" so I guess that means that z > x ]]>

util/clixon_util_xml.c

@@ -72,6 +72,7 @@ usage(char *argv0)
 	    "where options are\n"
             "\t-h \t\tHelp\n"
     	    "\t-D <level> \tDebug\n"
+	    "\t-j \tOutput as JSON\n"
 	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n",
 	    argv0);
     exit(0);
@@ -87,10 +88,11 @@ main(int    argc,
     int   retval = -1;
     int   c;
     int   logdst = CLICON_LOG_STDERR;
+    int   json = 0;
 
     optind = 1;
     opterr = 0;
-    while ((c = getopt(argc, argv, "hD:l:")) != -1)
+    while ((c = getopt(argc, argv, "hD:jl:")) != -1)
 	switch (c) {
 	case 'h':
 	    usage(argv[0]);
@@ -99,6 +101,9 @@ main(int    argc,
 	    if (sscanf(optarg, "%d", &debug) != 1)
 		usage(argv[0]);
 	    break;
+	case 'j':
+	    json++;
+	    break;
 	case 'l': /* Log destination: s|e|o|f */
 	    if ((logdst = clicon_log_opt(optarg[0])) < 0)
 		usage(argv[0]);
@@ -114,7 +119,10 @@ main(int    argc,
     }
     xc = NULL;
     while ((xc = xml_child_each(xt, xc, -1)) != NULL) 
-	clicon_xml2cbuf(cb, xc, 0, 0); /* print xml */
+	if (json)
+	    xml2json_cbuf(cb, xc, 0); /* print xml */
+	else
+	    clicon_xml2cbuf(cb, xc, 0, 0); /* print xml */
     fprintf(stdout, "%s", cbuf_get(cb));
     fflush(stdout);
 #if 0