Extended util/clixon_util_xml with yang and validate functionality so it can be used as a stand-alone utility for validating XML/JSON files

CHANGELOG.md

@@ -178,6 +178,7 @@
 
 ### Minor changes
 
+* Extended `util/clixon_util_xml` with yang and validate functionality so it can be used as a stand-alone utility for validating XML/JSON files
 * JSON parse and print improvements
   * Integrated parsing with namespace translation and yang spec lookup
 * Added CLIgen tab-modes in config option CLICON_CLI_TAB_MODE, which means you can control the behaviour of `<tab>` in the CLI.

lib/src/clixon_options.c

@@ -243,6 +243,7 @@ parse_configfile(clicon_handle  h,
 
 /*! Add configuration option overriding file setting
  * Add to clicon_options hash, and to clicon_conf_xml tree
+ * Assumes clicon_conf_xml_set has been called
  * @param[in]  h      Clicon handle
  * @param[in]  name   Name of configuration option (see clixon-config.yang)
  * @param[in]  value  String value

lib/src/clixon_xml_map.c

@@ -1280,8 +1280,8 @@ xml_yang_validate_all(clicon_handle h,
  */
 int
 xml_yang_validate_all_top(clicon_handle h,
-			  cxobj   *xt, 
+			  cxobj        *xt, 
-			  cbuf    *cbret)
+			  cbuf         *cbret)
 {
     int    ret;
     cxobj *x;
@@ -2053,7 +2053,7 @@ xml_tree_prune_flagged(cxobj *xt,
  */
 int
 xml_default(cxobj *xt, 
-	   void   *arg)
+	    void   *arg)
 {
     int        retval = -1;
     yang_stmt *ys;

test/all.sh

@@ -2,6 +2,9 @@
 # Run, eg as:
 # ./all.sh 2>&1 | tee test.log # break on first test
 
+# Pattern to run tests, default is all, but you may want to narrow it down
+: ${pattern:=test_*.sh}
+
 if [ $# -gt 0 ]; then 
     echo "usage: $0 # detailed logs and stopon first error"
     exit -1
@@ -9,10 +12,10 @@ fi
 
 err=0
 testnr=0
-for test in test_*.sh; do
+for test in $pattern; do
     if [ $testnr != 0 ]; then echo; fi
     testfile=$test
-    . ./$test 
+    ret=$(./$test) #    . ./$test 
     errcode=$?
     if [ $errcode -ne 0 ]; then
 	err=1

test/mem.sh

@@ -2,7 +2,7 @@
 # Run valgrind leak test for cli, restconf, netconf or background.
 # Stop on first error
     
-# Pattern to run tests
+# Pattern to run tests, default is all, but you may want to narrow it down
 : ${pattern:=test_*.sh}
 
 # Run valgrindtest once, args:

test/test_xml.sh

@@ -6,7 +6,7 @@
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
-: ${clixon_util_xml:=clixon_util_xml}
+: ${clixon_util_xml:=clixon_util_xml -o} # -o is output
 
 new "xml parse"
 expecteof "$clixon_util_xml" 0 "<a><b/></a>" "^<a><b/></a>$"

util/clixon_util_json.c

@@ -59,7 +59,7 @@
 #include "clixon/clixon.h"
 
 /*
- * Turn this on to get a json parse and pretty print test program
+ * JSON parse and pretty print test program
  * Usage: xpath
  * read json from input
  * Example compile:

util/clixon_util_xml.c

@@ -34,6 +34,11 @@
  * XML support functions.
  * @see https://www.w3.org/TR/2008/REC-xml-20081126
  *      https://www.w3.org/TR/2009/REC-xml-names-20091208
+ * The function can do yang validation, process xml and json, etc.
+ * On success, nothing is printed and exitcode 0
+ * On failure, an error is printed on stderr and exitcode != 0
+ * Failure error prints are different, it would be nice to make them more
+ * uniform. (see clicon_rpc_generate_error)
  */
 
 #ifdef HAVE_CONFIG_H
@@ -49,7 +54,9 @@
 #include <stdint.h>
 #include <syslog.h>
 #include <assert.h>
+#include <fcntl.h>
 #include <sys/time.h>
+#include <sys/stat.h>
 
 /* cligen */
 #include <cligen/cligen.h>
@@ -69,12 +76,20 @@
 static int
 usage(char *argv0)
 {
-    fprintf(stderr, "usage:%s [options]\n"
+    fprintf(stderr, "usage:%s [options] with xml on stdin\n"
 	    "where options are\n"
             "\t-h \t\tHelp\n"
     	    "\t-D <level> \tDebug\n"
+	    "\t-f <file>\tXML input file (overrides stdin)\n"
+	    "\t-J \t\tInput as JSON\n"
 	    "\t-j \t\tOutput as JSON\n"
-	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n",
+	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n"
+	    "\t-o \t\tOutput the file\n"
+	    "\t-v \t\tValidate the result in terms of Yang model (requires -y)\n"
+	    "\t-p \t\tPretty-print output\n"
+	    "\t-y <filename> \tYang filename or dir (load all files)\n"
+    	    "\t-Y <dir> \tYang dirs (can be several)\n"
+	    ,
 	    argv0);
     exit(0);
 }
@@ -83,17 +98,39 @@ int
 main(int    argc,
      char **argv)
 {
-    int   retval = -1;
+    int           retval = -1;
-    cxobj *xt = NULL;
+    cxobj        *xt = NULL;
-    cxobj *xc;
+    cxobj        *xc;
-    cbuf  *cb = cbuf_new();
+    cbuf         *cb = cbuf_new();
-    int   c;
+    int           c;
-    int   logdst = CLICON_LOG_STDERR;
+    int           logdst = CLICON_LOG_STDERR;
-    int   json = 0;
+    int           jsonin = 0;
+    int           jsonout = 0;
+    char         *input_filename = NULL;
+    char         *yang_file_dir = NULL;
+    yang_stmt    *yspec = NULL;
+    cxobj        *xerr = NULL; /* malloced must be freed */
+    int           ret;
+    int           pretty = 0;
+    int           validate = 0;
+    int           output = 0;
+    clicon_handle h;
+    struct stat   st;
+    int           fd = 0; /* stdin */
+    cxobj        *xcfg = NULL;
+    cbuf         *cberr;
 
+    /* In the startup, logs to stderr & debug flag set later */
+    clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR); 
+
+    if ((h = clicon_handle_init()) == NULL)
+	goto done;
+
+    xcfg = xml_new("clixon-config", NULL, NULL);
+    clicon_conf_xml_set(h, xcfg);
     optind = 1;
     opterr = 0;
-    while ((c = getopt(argc, argv, "hD:jl:")) != -1)
+    while ((c = getopt(argc, argv, "hD:f:Jjl:pvoy:Y:")) != -1)
 	switch (c) {
 	case 'h':
 	    usage(argv[0]);
@@ -102,35 +139,129 @@ main(int    argc,
 	    if (sscanf(optarg, "%d", &debug) != 1)
 		usage(argv[0]);
 	    break;
+	case 'f':
+	    input_filename = optarg;
+	    break;
+	case 'J':
+	    jsonin++;
+	    break;
 	case 'j':
-	    json++;
+	    jsonout++;
 	    break;
 	case 'l': /* Log destination: s|e|o|f */
 	    if ((logdst = clicon_log_opt(optarg[0])) < 0)
 		usage(argv[0]);
 	    break;
+	case 'o':
+	    output++;
+	    break;
+	case 'v':
+	    validate++;
+	    break;
+	case 'p':
+	    pretty++;
+	    break;
+	case 'y':
+	    yang_file_dir = optarg;
+	    break;
+	case 'Y':
+	    if (clicon_option_add(h, "CLICON_YANG_DIR", optarg) < 0)
+		goto done;
+	    break;
 	default:
 	    usage(argv[0]);
 	    break;
 	}
-    clicon_log_init(__FILE__, debug?LOG_DEBUG:LOG_INFO, logdst);
+    if (validate && !yang_file_dir){
-    if (xml_parse_file(0, "</config>", NULL, &xt) < 0){
+	fprintf(stderr, "-v requires -y\n");
-	fprintf(stderr, "xml parse error %s\n", clicon_err_reason);
+	usage(argv[0]);
-	goto done;
+    }
+    clicon_log_init(__FILE__, debug?LOG_DEBUG:LOG_INFO, logdst);
+    /* 1. Parse yang */
+    if (yang_file_dir){
+	if ((yspec = yspec_new()) == NULL)
+	    goto done;
+	if (stat(yang_file_dir, &st) < 0){
+	    clicon_err(OE_YANG, errno, "%s not found", yang_file_dir);
+	    goto done;
+	}
+	if (S_ISDIR(st.st_mode)){
+	    if (yang_spec_load_dir(h, yang_file_dir, yspec) < 0)
+		goto done;
+	}
+	else{
+	    if (yang_spec_parse_file(h, yang_file_dir, yspec) < 0)
+		goto done;
+	}
+    }
+    if (input_filename){
+	if ((fd = open(input_filename, O_RDONLY)) < 0){
+	    clicon_err(OE_YANG, errno, "open(%s)", input_filename);	
+	    goto done;
+	}
+    }
+    /* 2. Parse data (xml/json) */
+    if (jsonin){
+	if ((ret = json_parse_file(fd, yspec, &xt, &xerr)) < 0)
+	    goto done;
+	if (ret == 0){
+	    xml_print(stderr, xerr);
+	    goto done;
+	}
+    }
+    else{
+	if (xml_parse_file(fd, "</config>", NULL, &xt) < 0){
+	    fprintf(stderr, "xml parse error %s\n", clicon_err_reason);
+	    goto done;
+	}
+    }
+
+    /* Dump data structures (for debug) */
+    if (debug){
+	cbuf_reset(cb);
+	xmltree2cbuf(cb, xt, 0);       
+	fprintf(stderr, "%s\n", cbuf_get(cb));
+	cbuf_reset(cb);
+    }
+
+    /* 3. Validate data (if yspec) */
+    if (validate){
+	xc = xml_child_i(xt, 0);
+	if ((cberr = cbuf_new()) == NULL){
+	    clicon_err(OE_UNIX, errno, "cbuf_new");
+	    goto done;
+	}
+	/* Populate */
+	if (xml_apply0(xc, CX_ELMNT, xml_spec_populate, yspec) < 0)
+	    goto done;
+	/* Sort */
+	if (xml_apply0(xc, CX_ELMNT, xml_sort, h) < 0)
+	    goto done;
+	/* Add default values */
+	if (xml_apply(xc, CX_ELMNT, xml_default, h) < 0)
+	    goto done;
+	if (xml_apply0(xc, -1, xml_sort_verify, h) < 0)
+	    clicon_log(LOG_NOTICE, "%s: sort verify failed", __FUNCTION__);
+	if ((ret = xml_yang_validate_all_top(h, xc, cberr)) < 0) 
+	    goto done;
+	if (ret > 0 && (ret = xml_yang_validate_add(h, xc, cberr)) < 0)
+	    goto done;
+	if (ret == 0){
+	    fprintf(stderr, "%s", cbuf_get(cberr));
+	    goto done;
+	}
+    }
+    /* 4. Output data (xml/json) */
+    if (output){
+	xc = NULL;
+	while ((xc = xml_child_each(xt, xc, -1)) != NULL) 
+	    if (jsonout)
+		xml2json_cbuf(cb, xc, pretty); /* print xml */
+	    else
+		clicon_xml2cbuf(cb, xc, 0, pretty); /* print xml */
+	fprintf(stdout, "%s", cbuf_get(cb));
+	fflush(stdout);
     }
-    xc = NULL;
-    while ((xc = xml_child_each(xt, xc, -1)) != NULL) 
-	if (json)
-	    xml2json_cbuf(cb, xc, 0); /* print xml */
-	else
-	    clicon_xml2cbuf(cb, xc, 0, 0); /* print xml */
-    fprintf(stdout, "%s", cbuf_get(cb));
-    fflush(stdout);
-#if 0
-    cbuf_reset(cb);
-    xmltree2cbuf(cb, xt, 0);       /* dump data structures */
-    fprintf(stderr, "%s\n", cbuf_get(cb));
-#endif
     retval = 0;
  done:
     if (xt)