* Yang parser is stricter (see cardinality below) which may break parsing of slack yang specs.

* YANG parser cardinality checked (only modules level yet)
  * See https://github.com/clicon/clixon/issues/48

CHANGELOG.md

@@ -7,7 +7,15 @@
 
 ### Major New features
 ### API changes on existing features (you may need to change your code)
+* Yang parser is stricter (see cardinality below) which may break parsing of slack yang specs.
 ### Minor changes
+* YANG parser cardinality checked (only modules level yet)
+  * See https://github.com/clicon/clixon/issues/48
+* XML parser conformance to W3 spec
+  * Names lexically correct (NCName)
+  * Syntactically Correct handling of '<?' (processing instructions) and '<?xml' (XML declaration)
+  * XML prolog syntax for 'well-formed' XML
+  * <!DOCTYPE (ie DTD) is not supported.
 ### Corrected Bugs
 ### Known issues
 

README.md

@@ -24,6 +24,7 @@ support.
   * [Clixon project page](http://www.clicon.org)
   * [Tests](test/)
   * [Docker](docker/)
+  * [Roadmap](ROADMAP.md)
   * [Reference manual](http://www.clicon.org/doxygen/index.html) (Note: the link may not be up-to-date. It is better to build your own: `cd doc; make doc`)
   
 Background

example/example.yang

@@ -1,4 +1,6 @@
 module example {
+    yang-version 1.1;
+    namespace "urn:example:clixon";
     prefix ex;
     import ietf-interfaces {
 	prefix if;

lib/clixon/clixon_yang.h

@@ -55,9 +55,11 @@
  * Wanted to unify these (K_ and Y_) but gave up for several reasons:
  * - Dont want to expose a generated yacc file to the API
  * - Cant use the symbols in this file because yacc needs token definitions
+ * - Use 0 as no keyword --> therefore start enumeration with 1.
  */
 enum rfc_6020{
-    Y_ACTION = 0,
+    Y_ACTION = 1,
+    Y_ANYDATA,
     Y_ANYXML,
     Y_ARGUMENT,
     Y_AUGMENT,
@@ -252,6 +254,7 @@ yang_stmt *ys_module(yang_stmt *ys);
 yang_spec *ys_spec(yang_stmt *ys);
 yang_stmt *yang_find_module_by_prefix(yang_stmt *ys, char *prefix);
 yang_stmt *yang_find(yang_node *yn, int keyword, char *argument);
+int        yang_match(yang_node *yn, int keyword, char *argument);
 yang_stmt *yang_find_datanode(yang_node *yn, char *argument);
 yang_stmt *yang_find_schemanode(yang_node *yn, char *argument);
 yang_stmt *yang_find_topnode(yang_spec *ysp, char *name, yang_class class);

lib/src/Makefile.in

@@ -70,6 +70,7 @@ SRC     = clixon_sig.c clixon_log.c clixon_err.c clixon_event.c \
 	  clixon_string.c clixon_handle.c  \
 	  clixon_xml.c clixon_xml_sort.c clixon_xml_map.c clixon_file.c \
 	  clixon_json.c clixon_yang.c clixon_yang_type.c clixon_yang_module.c \
+          clixon_yang_cardinality.c \
 	  clixon_hash.c clixon_options.c clixon_plugin.c \
 	  clixon_proto.c clixon_proto_client.c \
 	  clixon_xpath.c clixon_xpath_ctx.c clixon_sha1.c \

lib/src/clixon_yang.c

@@ -76,12 +76,17 @@
 #include "clixon_options.h"
 #include "clixon_yang_type.h"
 #include "clixon_yang_parse.h"
+#include "clixon_yang_cardinality.h"
 
 /* Size of json read buffer when reading from file*/
 #define BUFLEN 1024
 
+/*
+ * Local variables
+ */
 /* Mapping between yang keyword string <--> clicon constants */
 static const map_str2int ykmap[] = {
+    {"anydata",          Y_ANYDATA}, 
     {"anyxml",           Y_ANYXML}, 
     {"argument",         Y_ARGUMENT}, 
     {"augment",          Y_AUGMENT}, 
@@ -393,9 +398,11 @@ yn_each(yang_node *yn,
  * @param[in]  yn         Yang node, current context node.
  * @param[in]  keyword    if 0 match any keyword
  * @param[in]  argument   String compare w argument. if NULL, match any.
+ * @retval     ys         Yang statement, if any
  * This however means that if you actually want to match only a yang-stmt with 
  * argument==NULL you cannot, but I have not seen any such examples.
  * @see yang_find_datanode
+ * @see yang_match  returns number of matches
  */
 yang_stmt *
 yang_find(yang_node *yn, 
@@ -420,6 +427,38 @@ yang_find(yang_node *yn,
     }
     return match ? ys : NULL;
 }
+
+/*! Count number of children that matches keyword and argument
+ *
+ * @param[in]  yn         Yang node, current context node.
+ * @param[in]  keyword    if 0 match any keyword
+ * @param[in]  argument   String compare w argument. if NULL, match any.
+ * @retval     n          Number of matches
+ * This however means that if you actually want to match only a yang-stmt with 
+ * argument==NULL you cannot, but I have not seen any such examples.
+ * @see yang_find
+ */
+int
+yang_match(yang_node *yn, 
+	   int        keyword, 
+	   char      *argument)
+{
+    yang_stmt *ys = NULL;
+    int        i;
+    int        match = 0;
+
+    for (i=0; i<yn->yn_len; i++){
+	ys = yn->yn_stmt[i];
+	if (keyword == 0 || ys->ys_keyword == keyword){
+	    if (argument == NULL)
+		match++;
+	    else
+		if (ys->ys_argument && strcmp(argument, ys->ys_argument) == 0)
+		    match++;
+	}
+    }
+    return match;
+}
 #ifdef NOTYET
 /*! Prototype more generic than yang_find_datanode and yang_find_schemanode
  */
@@ -2140,6 +2179,11 @@ yang_parse(clicon_handle h,
     if (yang_parse_recurse(ymod, dir, ysp) < 0)
 	goto done;
 
+    /* Check cardinality maybe this should be done after grouping/augment */
+    for (i=modnr; i<ysp->yp_len; i++) /* XXX */
+	if (yang_cardinality(h, ysp->yp_stmt[i], ysp->yp_stmt[i]->ys_argument) < 0)
+	    goto done;
+
     /* Step 2: check features: check if enabled and remove disabled features */
     for (i=modnr; i<ysp->yp_len; i++) /* XXX */
 	if (yang_features(h, ysp->yp_stmt[i]) < 0)

lib/src/clixon_yang_parse.l

@@ -112,6 +112,7 @@ clixon_yang_parsewrap(void)
 
  /* RFC 6020 keywords */
 <KEYWORD>action           { BEGIN(ARGUMENT); return K_ACTION; }
+<KEYWORD>anydata          { BEGIN(ARGUMENT); return K_ANYDATA; }
 <KEYWORD>anyxml           { BEGIN(ARGUMENT); return K_ANYXML; }
 <KEYWORD>argument         { BEGIN(ARGUMENT); return K_ARGUMENT; }
 <KEYWORD>augment          { BEGIN(ARGUMENT); return K_AUGMENT; }

lib/src/clixon_yang_parse.y

@@ -65,6 +65,7 @@
  * - Cant use the symbols in this file because yacc needs token definitions
  */
 %token K_ACTION
+%token K_ANYDATA
 %token K_ANYXML
 %token K_ARGUMENT
 %token K_AUGMENT

test/lib.sh

@@ -142,11 +142,10 @@ EOF
       echo -e "\e[0m:"
       exit -1
   fi
-  # If error dont match output strings
-  if [ $r != 0 ]; then
-      return
-  fi
-
+  # If error dont match output strings (why not?)
+#  if [ $r != 0 ]; then
+#      return
+#  fi
   # Match if both are empty string
   if [ -z "$ret" -a -z "$expect" ]; then
       return

test/test_datastore.sh

@@ -10,6 +10,9 @@ datastore=../datastore/datastore_client
 
 cat <<EOF > $fyang
 module ietf-ip{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix ip;
    container x {
     list y {
       key "a b";

test/test_feature.sh

@@ -28,6 +28,8 @@ EOF
 
 cat <<EOF > $fyang
 module $APPNAME{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
    prefix ex;
    import ietf-routing {
 	prefix rt;
@@ -93,7 +95,7 @@ expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><validate><source><candid
 new "netconf disabled feature"
 expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><edit-config><target><candidate/></target><config><A>foo</A></config></edit-config></rpc>]]>]]>" '^<rpc-reply><rpc-error><error-tag>operation-failed</error-tag><error-type>protocol</error-type><error-severity>error</error-severity><error-message>XML node config/A has no corresponding yang specification (Invalid XML or wrong Yang spec?'
 
-# This test has been broken up into all differetn modules instead of one large
+# This test has been broken up into all different modules instead of one large
 # reply since the modules change so often
 new "netconf schema resource, RFC 7895"
 ret=$($clixon_netconf -qf $cfg -y $fyang<<EOF 
@@ -101,7 +103,7 @@ ret=$($clixon_netconf -qf $cfg -y $fyang<<EOF
 EOF
 )
 new "netconf module A"
-expect="<module><name>example</name><revision/><namespace/><feature>A</feature><conformance-type>implement</conformance-type></module>"
+expect="<module><name>example</name><revision/><namespace>urn:example:clixon</namespace><feature>A</feature><conformance-type>implement</conformance-type></module>"
 match=`echo "$ret" | grep -GZo "$expect"`
 if [ -z "$match" ]; then
       err "$expect" "$ret"

test/test_leafref.sh

@@ -24,6 +24,8 @@ EOF
 
 cat <<EOF > $fyang
 module example{
+    yang-version 1.1;
+    namespace "urn:example:clixon";
     prefix ex;
     import ietf-interfaces {
 	prefix if;

test/test_list.sh

@@ -26,6 +26,9 @@ EOF
 
 cat <<EOF > $fyang
 module $APPNAME{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
+  prefix ex;
   container c{
     presence true;
     list a0{

test/test_nacm.sh

@@ -31,6 +31,8 @@ EOF
 
 cat <<EOF > $fyang
 module $APPNAME{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
   prefix ex;
   import ietf-netconf-acm {
 	prefix nacm;

test/test_nacm_ext.sh

@@ -37,6 +37,8 @@ EOF
 
 cat <<EOF > $fyang
 module $APPNAME{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
   prefix ex;
   container authentication {
 	description "Example code for enabling www basic auth and some example 

test/test_netconf.sh

@@ -30,6 +30,8 @@ EOF
 
 cat <<EOF > $fyang
 module example{
+    yang-version 1.1;
+    namespace "urn:example:clixon";
     prefix ex;
     import ietf-interfaces {
 	prefix if;

test/test_order.sh

@@ -41,6 +41,9 @@ EOF
 
 cat <<EOF > $fyang
 module example{
+    yang-version 1.1;
+    namespace "urn:example:clixon";
+    prefix ex;
     container c{
       leaf d{
          type string;

test/test_perf.sh

@@ -24,6 +24,9 @@ fconfig=$dir/config
 
 cat <<EOF > $fyang
 module ietf-ip{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix ip;
    container x {
     list y {
       key "a";

test/test_restconf.sh

@@ -30,6 +30,8 @@ EOF
 
 cat <<EOF > $fyang
 module example{
+    yang-version 1.1;
+    namespace "urn:example:clixon";
     prefix ex;
     import ietf-interfaces {
 	prefix if;

test/test_restconf2.sh

@@ -22,6 +22,9 @@ EOF
 
 cat <<EOF > $fyang
 module example{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix ex;
    container cont1{
       list interface{
         key name;

test/test_type.sh

@@ -26,6 +26,9 @@ EOF
 
 cat <<EOF > $fyang
 module example{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
+  prefix ex;
   typedef ab {
        type string {
          pattern

test/test_when_must.sh

@@ -27,6 +27,8 @@ EOF
 
 cat <<EOF > $fyang
 module $APPNAME{
+  yang-version 1.1;
+  namespace "urn:example:clixon";
   prefix ex;
   identity routing-protocol {
      description

test/test_xml.sh

@@ -79,13 +79,13 @@ new "XMLdecl no version"
 expecteof "$PROG" 255 '<?xml ?><a/>' ''
 
 new "XMLdecl misspelled version"
-expecteof "$PROG" 255 '<?xml verion="1.0"?><a/>' '<a/>'
+expecteof "$PROG -l o" 255 '<?xml verion="1.0"?><a/>' 'yntax error: at or before: v'
 
 new "XMLdecl version + encoding"
 expecteof "$PROG" 0 '<?xml version="1.0" encoding="UTF-16"?><a/>' '<a/>'
 
 new "XMLdecl version + misspelled encoding"
-expecteof "$PROG" 255 '<?xml version="1.0" encding="UTF-16"?><a/>' '<a/>'
+expecteof "$PROG -l o" 255 '<?xml version="1.0" encding="UTF-16"?><a/>' 'syntax error: at or before: e'
 
 new "XMLdecl version + standalone"
 expecteof "$PROG" 0 '<?xml version="1.0" standalone="yes"?><a/>' '<a/>'

test/test_yang.sh

@@ -27,7 +27,9 @@ EOF
 
 cat <<EOF > $fyang
 module $APPNAME{
+   yang-version 1.1;
    prefix ex;
+   namespace "urn:example:clixon";
    extension c-define {
       description "Example from RFC 6020";
       argument "name";
@@ -89,6 +91,8 @@ EOF
 # This yang definition uses an extension which is not defined. Error when loading
 cat <<EOF > $fyangerr
 module $APPNAME{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
    prefix ex;
    extension c-define {
       description "Example from RFC 6020";
@@ -126,7 +130,9 @@ expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "<rpc><get-config><source><cand
 new "netconf discard-changes"
 expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
-new "cli not defined extension"
+#new "cli not defined extension"
+#new "netconf not defined extension"
+#expecteof "$clixon_netconf -qf $cfg -l o" 0 "$YANG" "Extension ex:not-defined not found"
 # This text yields an error, but the test cannot detect the error message yet
 #expectfn "$clixon_cli -1f $cfg -y $fyangerr show version" 0 "Yang error: Extension ex:not-defined not found"
 

test/test_yang_parse.sh

@@ -1,41 +1,26 @@
 #!/bin/bash
 # Test: YANG parser tests
 # First an example yang, second all openconfig yangs
+# Problem with this is that util only parses single file. it should
+# call yang_parse().
 #PROG="valgrind --leak-check=full --show-leak-kinds=all ../util/clixon_util_yang"
 PROG=../util/clixon_util_yang
 OPENCONFIG=~/syssrc/openconfig
-
+exit 0 # nyi
 # include err() and new() functions and creates $dir
 . ./lib.sh
 
-YANG=$(cat <<EOF
-module test{
-   prefix ex;
-   extension c-define {
-      description "Example from RFC 6020";
-      argument "name";
-   }
-   ex:not-defined ARGUMENT;
-}
-EOF
-)
-
-new "yang parse"
-#expecteof "$PROG" 0 "$YANG" "^$YANG$"
-
-if [ ! -d $OPENCONFIG ]; then
-    echo "$OPENCONFIG not found. Do git clone https://github.com/openconfig/public and point DIR to it to run these tests"
-    rm -rf $dir
-    exit 0
-fi
-
 # Openconfig
+# Files not parseable:
+# - openconfig-access-points.yang
+# - openconfig-access-points.yang
 new "Openconfig"
 files=$(find $OPENCONFIG -name "*.yang")
 for f in $files; do
     new "$f"
     YANG=$(cat $f)
- #   expecteof "$PROG" 0 "$YANG" "module"
+ # NYI
+  expecteof "$PROG" 0 "$YANG" "module"
 done
 rm -rf $dir
 

util/clixon_util_xml.c

@@ -46,7 +46,6 @@
 #include <errno.h>
 #include <string.h>
 #include <limits.h>
-#include <fnmatch.h>
 #include <stdint.h>
 #include <syslog.h>
 #include <assert.h>
@@ -72,7 +71,8 @@ usage(char *argv0)
     fprintf(stderr, "usage:%s [options]\n"
 	    "where options are\n"
             "\t-h \t\tHelp\n"
-    	    "\t-D <level> \tDebug\n",
+    	    "\t-D <level> \tDebug\n"
+	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n",
 	    argv0);
     exit(0);
 }
@@ -86,11 +86,11 @@ main(int    argc,
     cbuf  *cb = cbuf_new();
     int   retval = -1;
     char  c;
+    int   logdst = CLICON_LOG_STDERR;
 
-    clicon_log_init("xpath", LOG_DEBUG, CLICON_LOG_STDERR); 
     optind = 1;
     opterr = 0;
-    while ((c = getopt(argc, argv, "hD:")) != -1)
+    while ((c = getopt(argc, argv, "hD:l:")) != -1)
 	switch (c) {
 	case 'h':
 	    usage(argv[0]);
@@ -99,10 +99,15 @@ main(int    argc,
 	    if (sscanf(optarg, "%d", &debug) != 1)
 		usage(argv[0]);
 	    break;
+	case 'l': /* Log destination: s|e|o|f */
+	    if ((logdst = clicon_log_opt(optarg[0])) < 0)
+		usage(argv[0]);
+	    break;
 	default:
 	    usage(argv[0]);
 	    break;
 	}
+    clicon_log_init("clixon_util_xml", debug?LOG_DEBUG:LOG_INFO, logdst);
     if (xml_parse_file(0, "</config>", NULL, &xt) < 0){
 	fprintf(stderr, "xml parse error %s\n", clicon_err_reason);
 	goto done;

util/clixon_util_yang.c

@@ -40,6 +40,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <unistd.h>
 #include <limits.h>
 #include <ctype.h>
 #define __USE_GNU /* strverscmp */
@@ -65,7 +66,12 @@
 static int
 usage(char *argv0)
 {
-    fprintf(stderr, "usage:%s.\n\tInput on stdin\n", argv0);
+    fprintf(stderr, "usage:%s [options]\n"
+	    "where options are\n"
+            "\t-h \t\tHelp\n"
+    	    "\t-D <level> \tDebug\n"
+	    "\t-l <s|e|o> \tLog on (s)yslog, std(e)rr, std(o)ut (stderr is default)\n",
+	    argv0);
     exit(0);
 }
 
@@ -73,12 +79,29 @@ int
 main(int argc, char **argv)
 {
     yang_spec *yspec = NULL;
+    char      c;
+    int       logdst = CLICON_LOG_STDERR;
     
-    if (argc != 1){
+    optind = 1;
+    opterr = 0;
+    while ((c = getopt(argc, argv, "hD:l:")) != -1)
+	switch (c) {
+	case 'h':
 	    usage(argv[0]);
-	return -1;
+	    break;
+    	case 'D':
+	    if (sscanf(optarg, "%d", &debug) != 1)
+		usage(argv[0]);
+	    break;
+	case 'l': /* Log destination: s|e|o|f */
+	    if ((logdst = clicon_log_opt(optarg[0])) < 0)
+		usage(argv[0]);
+	    break;
+	default:
+	    usage(argv[0]);
+	    break;
 	}
-    clicon_log_init("clixon_util_yang", LOG_INFO, CLICON_LOG_STDERR);
+    clicon_log_init("clixon_util_yang", debug?LOG_DEBUG:LOG_INFO, logdst);
     if ((yspec = yspec_new()) == NULL)
 	goto done;
     if (yang_parse_file(0, "yang test", yspec) == NULL){

yang/clixon-config@2018-10-21.yang

@@ -1,5 +1,6 @@
 module clixon-config {
-
+    yang-version 1.1;
+    namespace "http://clicon.org";
     prefix cc;
 
     organization