* Optional yangs for testing have been removed from the Clixon repo

* These were included for testing
  * If you want to run the Clixon test suite you need to point `YANGMODELS`, see test/README.md
  * The following configure options have been removed:
    * `configure --with-opt-yang-installdir=DIR`
    * `configure   --enable-optyangs`
  * You may need to specify standard YANGs using configure option `--with-yang-standard-dir=DIR`
* Updated yang ietf models with fetures for tet
* Added option `CLICON_YANG_AUGMENT_ACCEPT_BROKEN` to accept broken yangmodels.
  * This is a debug option for CI testcases where standard YANG models are broken

configure

@@ -621,8 +621,6 @@ ac_includes_default="\
 
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
-OPT_YANG_INSTALLDIR
-YANG_INSTALLDIR
 EGREP
 GREP
 LEXLIB
@@ -635,8 +633,9 @@ CXXFLAGS
 CXX
 CPP
 wwwdir
+YANG_STANDARD_DIR
+YANG_INSTALLDIR
 CLIXON_YANG_PATCH
-enable_optyangs
 with_libxml2
 HAVE_LIBNGHTTP2
 HAVE_LIBEVHTP
@@ -719,7 +718,6 @@ ac_user_opts='
 enable_option_checking
 enable_debug
 with_cligen
-enable_optyangs
 enable_yang_patch
 enable_publish
 with_restconf
@@ -729,7 +727,7 @@ with_configfile
 with_libxml2
 with_sigaction
 with_yang_installdir
-with_opt_yang_installdir
+with_yang_standard_installdir
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1369,8 +1367,6 @@ Optional Features:
   --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
   --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
   --enable-debug          Build with debug symbols, default: no
-  --enable-optyangs       Include optional yang files for examples and testing
-                          in clixon install, default: no
   --enable-yang-patch     Enable YANG patch, RFC 8072, default: no
   --enable-publish        Enable publish of notification streams using SSE and
                           curl
@@ -1393,9 +1389,9 @@ Optional Packages:
   --with-yang-installdir=DIR
                           Install Clixon yang files here (default:
                           ${prefix}/share/clixon)
-  --with-opt-yang-installdir=DIR
+  --with-yang-standard-dir=DIR
-                          Install optional yang files here (default:
+                          Where standard IETF/IEEE YANGs are (default:
-                          ${prefix}/share/clixon)
+                          ${prefix}/share/yang/standard)
 
 Some influential environment variables:
   CC          C compiler command
@@ -3395,7 +3391,9 @@ HAVE_LIBNGHTTP2=false
  # consider using neutral constant such as with-http2
 
 
+# Where Clixon installs its YANG specs
 
+# Examples require standard IETF YANGs. You need to provide these for example and tests
 
 
 # Home dir for web user, such as nginx fcgi sockets
@@ -4590,26 +4588,6 @@ if test -d "${with_cligen}"; then
   test -d "$with_cligen" && CLIGEN_PREFIX="$with_cligen"
 fi
 
-# Disable/enable standard Yang files.
-# If enable - include yang/standard/*.yang in clixon yang files (default)
-# If disable - get standard yang files from elsewhere
-# Check whether --enable-optyangs was given.
-if test "${enable_optyangs+set}" = set; then :
-  enableval=$enable_optyangs;
-	  if test "$enableval" = no; then
-	      enable_optyangs=no
-	  else
-	      enable_optyangs=yes
-          fi
-
-else
-   enable_optyangs=no
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: optyangs is $enable_optyangs" >&5
-$as_echo "optyangs is $enable_optyangs" >&6; }
-
 # Disable/enable yang patch
 # Check whether --enable-yang-patch was given.
 if test "${enable_yang_patch+set}" = set; then :
@@ -5674,26 +5652,23 @@ else
 
 fi
 
-
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: Clixon yang files are installed in ${YANG_INSTALLDIR}" >&5
 $as_echo "Clixon yang files are installed in ${YANG_INSTALLDIR}" >&6; }
 
-# OPT_YANG_INSTALLDIR is where clixon installs standard yang files
+# YANG_STANDARD_DIR is where clixon assumes standard IETF are
-# (  the files in in yang/standard)
+# This is NOT installed by Clixon and is not needed for core system
-# that Clixon needs to run (or examples rely on). These may be retreived from
+# However, it is required by the main example and some of the tests
-# elsewhere (eg yangmodels repo)
 
-# Check whether --with-opt-yang-installdir was given.
+# Check whether --with-yang-standard-installdir was given.
-if test "${with_opt_yang_installdir+set}" = set; then :
+if test "${with_yang_standard_installdir+set}" = set; then :
-  withval=$with_opt_yang_installdir; OPT_YANG_INSTALLDIR="$withval"
+  withval=$with_yang_standard_installdir; YANG_STANDARD_DIR="$withval"
 else
-  OPT_YANG_INSTALLDIR="${prefix}/share/clixon"
+  YANG_STANDARD_DIR="${prefix}/share/yang/standard"
 
 fi
 
-
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Standard YANG files should be in ${YANG_STANDARD_DIR}" >&5
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Optional yang files are installed in ${OPT_YANG_INSTALLDIR} (if enabled)" >&5
+$as_echo "Standard YANG files should be in ${YANG_STANDARD_DIR}" >&6; }
-$as_echo "Optional yang files are installed in ${OPT_YANG_INSTALLDIR} (if enabled)" >&6; }
 
 # Default location for config file
 
@@ -5704,7 +5679,7 @@ _ACEOF
 
 
 
-ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/config.sh test/cicd/Makefile test/vagrant/Makefile"
+ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile example/main/example.xml extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/config.sh test/cicd/Makefile test/vagrant/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -6412,6 +6387,7 @@ do
     "etc/clixonrc") CONFIG_FILES="$CONFIG_FILES etc/clixonrc" ;;
     "example/Makefile") CONFIG_FILES="$CONFIG_FILES example/Makefile" ;;
     "example/main/Makefile") CONFIG_FILES="$CONFIG_FILES example/main/Makefile" ;;
+    "example/main/example.xml") CONFIG_FILES="$CONFIG_FILES example/main/example.xml" ;;
     "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;;
     "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;;
     "docker/main/Makefile") CONFIG_FILES="$CONFIG_FILES docker/main/Makefile" ;;

configure.ac

@@ -118,9 +118,11 @@ AC_SUBST(with_restconf) # Set to native or fcgi -> compile apps/restconf
 AC_SUBST(HAVE_LIBEVHTP,false)   # consider using neutral constant such as with-http1
 AC_SUBST(HAVE_LIBNGHTTP2,false) # consider using neutral constant such as with-http2
 AC_SUBST(with_libxml2)  
-AC_SUBST(enable_optyangs)
 AC_SUBST(CLIXON_YANG_PATCH)
-
+# Where Clixon installs its YANG specs
+AC_SUBST(YANG_INSTALLDIR) 
+# Examples require standard IETF YANGs. You need to provide these for example and tests
+AC_SUBST(YANG_STANDARD_DIR)
 
 # Home dir for web user, such as nginx fcgi sockets
 AC_SUBST(wwwdir,/www-data)
@@ -173,20 +175,6 @@ if test -d "${with_cligen}"; then
   test -d "$with_cligen" && CLIGEN_PREFIX="$with_cligen"
 fi
 
-# Disable/enable standard Yang files.
-# If enable - include yang/standard/*.yang in clixon yang files (default)
-# If disable - get standard yang files from elsewhere
-AC_ARG_ENABLE(optyangs, AS_HELP_STRING([--enable-optyangs],[Include optional yang files for examples and testing in clixon install, default: no]),[
-	  if test "$enableval" = no; then
-	      enable_optyangs=no
-	  else	      
-	      enable_optyangs=yes
-          fi
-        ],
-	[ enable_optyangs=no])
-
-AC_MSG_RESULT(optyangs is $enable_optyangs)
-
 # Disable/enable yang patch
 AC_ARG_ENABLE(yang-patch, AS_HELP_STRING([--enable-yang-patch],[Enable YANG patch, RFC 8072, default: no]),[
 	  if test "$enableval" = no; then
@@ -347,20 +335,17 @@ AC_ARG_WITH(yang-installdir,
 	    [YANG_INSTALLDIR="$withval"],
     	    [YANG_INSTALLDIR="${prefix}/share/clixon"]		
 	    )
-AC_SUBST(YANG_INSTALLDIR)
 AC_MSG_RESULT(Clixon yang files are installed in ${YANG_INSTALLDIR})	
 
-# OPT_YANG_INSTALLDIR is where clixon installs standard yang files
+# YANG_STANDARD_DIR is where clixon assumes standard IETF are
-# (  the files in in yang/standard)
+# This is NOT installed by Clixon and is not needed for core system
-# that Clixon needs to run (or examples rely on). These may be retreived from
+# However, it is required by the main example and some of the tests
-# elsewhere (eg yangmodels repo)
+AC_ARG_WITH(yang-standard-installdir,
-AC_ARG_WITH(opt-yang-installdir,
+            [AS_HELP_STRING([--with-yang-standard-dir=DIR],[Where standard IETF/IEEE YANGs are (default: ${prefix}/share/yang/standard)])],
-	    [AS_HELP_STRING([--with-opt-yang-installdir=DIR],[Install optional yang files here (default: ${prefix}/share/clixon)])],
+	    [YANG_STANDARD_DIR="$withval"],
-	    [OPT_YANG_INSTALLDIR="$withval"],
+    	    [YANG_STANDARD_DIR="${prefix}/share/yang/standard"]		
-    	    [OPT_YANG_INSTALLDIR="${prefix}/share/clixon"]		
 	    )
-AC_SUBST(OPT_YANG_INSTALLDIR)
+AC_MSG_RESULT(Standard YANG files should be in ${YANG_STANDARD_DIR})	
-AC_MSG_RESULT(Optional yang files are installed in ${OPT_YANG_INSTALLDIR} (if enabled))	
 
 # Default location for config file
 AC_DEFINE_UNQUOTED(CLIXON_DEFAULT_CONFIG,"${CLIXON_DEFAULT_CONFIG}",[Location for apps to find default config file])
@@ -381,6 +366,7 @@ AC_OUTPUT(Makefile
 	  etc/clixonrc
 	  example/Makefile
   	  example/main/Makefile
+    	  example/main/example.xml
 	  extras/rpm/Makefile
 	  docker/Makefile
   	  docker/main/Makefile

doc/FAQ.md

@@ -79,11 +79,10 @@ One of the examples is [a hello world example](https://github.com/clicon/clixon-
 ## How do you build and install Clixon?
 Clixon: 
 ```
-	./configure --enable-optyang;
+	./configure
 	make; 
 	sudo make install; 
 ```
-(note: optyang enable only if you need to run the main example, otherwise it is not necessary).
 
 The main example:
 ```

docker/main/Dockerfile

@@ -39,6 +39,13 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison curl-dev
 
+# Checkut standard YANG models for tests (note >1G for full repo)
+WORKDIR /usr/local/share
+RUN mkdir yang; cd yang; git init; git remote add -f origin https://github.com/YangModels/yang; git config core.sparseCheckout true; echo "standard/" >> .git/info/sparse-checkout; echo "experimental/" >> .git/info/sparse-checkout
+
+WORKDIR /usr/local/share/yang
+RUN git pull origin master
+
 # Create a directory to hold source-code, dependencies etc
 RUN mkdir /clixon
 RUN mkdir /clixon/build
@@ -59,7 +66,7 @@ WORKDIR /clixon/clixon
 COPY clixon .
 
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --enable-optyangs --without-restconf
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --without-restconf
 RUN make
 RUN make install
 
@@ -100,6 +107,7 @@ RUN apk add --update sudo curl procps grep make bash # iproute2 # contains ip
 RUN adduser -D -H clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
 
 # Log to stderr.
 CMD /usr/local/bin/startsystem.sh

docker/main/Dockerfile.fcgi

@@ -38,6 +38,13 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison fcgi-dev curl-dev
 
+# Checkut standard YANG models for tests (note >1G for full repo)
+WORKDIR /usr/local/share
+RUN mkdir yang; cd yang; git init; git remote add -f origin https://github.com/YangModels/yang; git config core.sparseCheckout true; echo "standard/" >> .git/info/sparse-checkout; echo "experimental/" >> .git/info/sparse-checkout
+
+WORKDIR /usr/local/share/yang
+RUN git pull origin master
+
 # Create a directory to hold source-code, dependencies etc
 RUN mkdir /clixon
 RUN mkdir /clixon/build
@@ -63,7 +70,7 @@ RUN adduser -D -H -G www-data www-data
 RUN apk add --update nginx
 
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --enable-optyangs --with-restconf=fcgi
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --with-restconf=fcgi
 RUN make
 RUN make install
 
@@ -114,6 +121,7 @@ RUN adduser nginx clicon
 RUN adduser www-data clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
 
 # Manually created
 RUN mkdir /www-data

docker/main/Dockerfile.native

@@ -38,15 +38,30 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison curl-dev
 
-# Create a directory to hold source-code, dependencies etc
-RUN mkdir /clixon
-
 # evhtp dependencies
 RUN apk add --update libevent libevent-dev
 
 # nghttp2 dependencies
 RUN apk add --update nghttp2
 
+# Checkut models
+WORKDIR /usr/local/share/
+
+# Checkut standard YANG models for tests (note >1G for full repo)
+RUN mkdir yang; cd yang; git init; git remote add -f origin https://github.com/YangModels/yang; git config core.sparseCheckout true; echo "standard/" >> .git/info/sparse-checkout; echo "experimental/" >> .git/info/sparse-checkout
+
+WORKDIR /usr/local/share/yang
+RUN git pull origin master
+
+RUN mkdir /usr/local/share/openconfig
+WORKDIR /usr/local/share/openconfig
+
+# Checkut Openconfig models for tests
+RUN git clone https://github.com/openconfig/public
+
+# Create a directory to hold source-code, dependencies etc
+RUN mkdir /clixon
+
 # clone libevhtp
 WORKDIR /clixon
 
@@ -79,7 +94,7 @@ WORKDIR /clixon/clixon
 COPY clixon .
 
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --enable-optyangs --with-restconf=native --enable-nghttp2 --enable-evhtp
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --with-restconf=native --enable-nghttp2 --enable-evhtp
 
 RUN make
 RUN make install
@@ -134,6 +149,9 @@ RUN adduser -D -H clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
 COPY --from=0 /usr/local/lib/libevhtp.so* /usr/local/lib/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/experimental/
+COPY --from=0 /usr/local/share/openconfig/* /usr/local/share/openconfig/
 
 # Manually created
 RUN mkdir /www-data

docker/main/startsystem.sh

@@ -61,9 +61,13 @@ echo "$STORE" > /usr/local/var/example/running_db
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
 SKIPLIST="test_api.sh test_c++.sh test_install.sh test_privileges.sh test_augment.sh test_choice.sh test_identity.sh test_nacm_datanode_read.sh test_nacm_datanode.sh test_nacm_datanode_write.sh test_nacm_default.sh test_nacm_ext.sh test_nacm_module_read.sh test_nacm_module_write.sh test_nacm_protocol.sh test_nacm.sh test_nacm_recovery.sh test_perf.sh test_perf_state_only.sh test_perf_state.sh test_restconf2.sh test_restconf_err.sh test_restconf_jukebox.sh test_restconf_listkey.sh test_restconf_patch.sh test_restconf.sh test_restconf_startup.sh test_rpc.sh test_ssl_certs.sh test_stream.sh test_submodule.sh test_upgrade_auto.sh test_upgrade_interfaces.sh test_upgrade_repair.sh test_yang_namespace.sh"
-#IETFRFC=
+# Parse yangmodels from https://github.com/YangModels/yang
+YANGMODELS=/usr/local/share/yang
 EOF
 
+# Patch yang syntax errors
+sed -i s/=\ olt\'/=\ \'olt\'/g /usr/local/share/yang/standard/ieee/published/802.3/ieee802-ethernet-pon.yang
+
 # Workaround for this error output:
 # sudo: setrlimit(RLIMIT_CORE): Operation not permitted
 echo "Set disable_coredump false" > /etc/sudo.conf

docker/main/startsystem_fcgi.sh

@@ -90,9 +90,13 @@ cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
 SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh"
 RCPROTO=http # Because nginx
-#IETFRFC=
+# Parse yangmodels from https://github.com/YangModels/yang
+YANGMODELS=/usr/local/share/yang
 EOF
 
+# Patch yang syntax errors
+sed -i s/=\ olt\'/=\ \'olt\'/g /usr/local/share/yang/standard/ieee/published/802.3/ieee802-ethernet-pon.yang
+
 # Workaround for this error output:
 # sudo: setrlimit(RLIMIT_CORE): Operation not permitted
 echo "Set disable_coredump false" > /etc/sudo.conf

docker/main/startsystem_native.sh

@@ -59,9 +59,13 @@ echo "$STORE" > /usr/local/var/example/running_db
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
 SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh"
-#IETFRFC=
+# Parse yangmodels from https://github.com/YangModels/yang
+YANGMODELS=/usr/local/share/yang
 EOF
 
+# Patch yang syntax errors
+sed -i s/=\ olt\'/=\ \'olt\'/g /usr/local/share/yang/standard/ieee/published/802.3/ieee802-ethernet-pon.yang
+
 # Workaround for this error output:
 # sudo: setrlimit(RLIMIT_CORE): Operation not permitted
 echo "Set disable_coredump false" > /etc/sudo.conf

example/main/example.xml.in

@@ -4,6 +4,7 @@
   <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE>
   <CLICON_FEATURE>clixon-restconf:fcgi</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>@YANG_STANDARD_DIR@</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>clixon-example</CLICON_YANG_MODULE_MAIN>
   <CLICON_CLI_MODE>example</CLICON_CLI_MODE>
   <CLICON_BACKEND_DIR>/usr/local/lib/example/backend</CLICON_BACKEND_DIR>

lib/src/clixon_yang_parse_lib.c

@@ -242,23 +242,24 @@ yang_augment_node(clicon_handle h,
 	goto done;
 
     if (ytarget == NULL){
-#if 1
+	if (clicon_option_bool(h, "CLICON_YANG_AUGMENT_ACCEPT_BROKEN")){
-	/* Fail with fatal error if augment target not found 
+	    /* Log a warning and proceed if augment target not found 
-	 * This is "correct"
+	     * This may be necessary with some broken models
-	 */
+	     */
-	clicon_err(OE_YANG, 0, "Augment failed in module %s: target node %s not found",
+	    clicon_log(LOG_WARNING, "Warning: Augment failed in module %s: target node %s not found",
-		   yang_argument_get(ys_module(ys)),
+		       yang_argument_get(ys_module(ys)),
-		   schema_nodeid);
+		       schema_nodeid);
-	goto done; 
+	    goto ok;
-#else
+	}
-	/* Log a warning and proceed if augment target not found 
+	else{
-	 * This may be necessary with some broken models
+	    /* Fail with fatal error if augment target not found 
-	 */
+	     * This is "correct"
-	clicon_log(LOG_WARNING, "Warning: Augment failed in module %s: target node %s not found",
+	     */
-		   yang_argument_get(ys_module(ys)),
+	    clicon_err(OE_YANG, 0, "Augment failed in module %s: target node %s not found",
-		   schema_nodeid);
+		       yang_argument_get(ys_module(ys)),
-	goto ok;
+		       schema_nodeid);
-#endif
+	    goto done; 
+	}
     }
     /* The target node MUST be either a container, list, choice, case, input, output, or notification node.
      * which means it is slightly different than a schema-nodeid ? */

test/README.md

@@ -7,10 +7,9 @@ automatically run as part of the all.sh, sum.sh tests etc. The scripts
 need to follow some rules to work properly, please look at one or two
 to get the idea.
 
-Note that configure must ensure optional yangs be installed:
+Note that some IETF yangs need to be available, by default these are
-```
+available in https://github.com/YangModels/yang. Just ensure the YANGMODELS in site.sh points
-  ./configure --enable-optyangs
+to a locally checked out dir.
-```
 
 See also the [site.sh](#site-sh) for example for skipping tests or setting some site-specific variables.
 

test/lib.sh

@@ -138,17 +138,6 @@ fi
 # Start restconf user, can be root which is dropped to wwwuser
 : ${wwwstartuser:=root} 
 
-# Parse yangmodels from https://github.com/YangModels/yang
-# Recommended: checkout yangmodels elsewhere in the tree and set the env
-# to that
-#: ${YANGMODELS=$(pwd)/yang} # just skip if not set
-
-# Parse yang openconfig models from https://github.com/openconfig/public
-#: ${OPENCONFIG=$(pwd)/public} # just skip if not set
-
-# Standard IETF RFC yang files. 
-: ${IETFRFC=../yang/mandatory}
-
 # Some restconf tests can run IPv6, but its complicated because:
 # - docker by default does not run IPv6
 : ${IPv6:=false}
@@ -187,6 +176,11 @@ if [ -f ./site.sh ]; then
     done
 fi
 
+# Standard IETF RFC yang files. 
+if [ ! -z ${YANGMODELS} ]; then
+    : ${IETFRFC=$YANGMODELS/standard/ietf/RFC}
+fi
+
 # Check sanity between --with-restconf setting and if nginx is started by systemd or not
 # This check is optional because some installs, such as vagrant make a non-systemd/direct
 # start

test/test_cli_auto.sh

@@ -14,7 +14,6 @@ fin=$dir/in
 fstate=$dir/state.xml
 
 # Use yang in example
-
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>

test/test_cli_translate.sh

@@ -19,6 +19,7 @@ cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>clixon-example</CLICON_YANG_MODULE_MAIN>
   <CLICON_CLISPEC_DIR>$dir</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>

test/test_feature.sh

@@ -221,7 +221,7 @@ fi
 fi # false
 
 new "netconf module ietf-inet-types"
-expect="<module><name>ietf-inet-types</name><revision>2020-07-06</revision><namespace>urn:ietf:params:xml:ns:yang:ietf-inet-types</namespace><conformance-type>implement</conformance-type></module>"
+expect="<module><name>ietf-inet-types</name><revision>2021-02-22</revision><namespace>urn:ietf:params:xml:ns:yang:ietf-inet-types</namespace><conformance-type>implement</conformance-type></module>"
 match=`echo "$ret" | grep --null -Go "$expect"`
 if [ -z "$match" ]; then
       err "$expect" "$ret"

test/test_openconfig.sh

@@ -29,6 +29,7 @@ cat <<EOF > $cfg
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$OPENCONFIG</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>

test/test_openconfig_interfaces.sh

@@ -24,6 +24,7 @@ cat <<EOF > $cfg
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$OCDIR</CLICON_YANG_DIR>
   <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>	
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>

test/test_openconfig_network_instance.sh

@@ -24,6 +24,7 @@ cat <<EOF > $cfg
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$OCDIR</CLICON_YANG_DIR>
   <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>	
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>

test/test_openconfig_system.sh

@@ -27,6 +27,7 @@ cat <<EOF > $cfg
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$OCDIR</CLICON_YANG_DIR>
   <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>	
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>

test/test_privileges.sh

@@ -24,6 +24,7 @@ cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>clixon-example</CLICON_YANG_MODULE_MAIN>
   <CLICON_SOCK>$dir/$APPNAME.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/var/tmp/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>

test/test_sock.sh

@@ -32,6 +32,7 @@ cat <<EOF > $cfg
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>*:*</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>clixon-example</CLICON_YANG_MODULE_MAIN>
   <CLICON_SOCK_FAMILY>$family</CLICON_SOCK_FAMILY>
   <CLICON_SOCK_PORT>4535</CLICON_SOCK_PORT>

test/test_yang_models_ietf.sh

@@ -4,26 +4,7 @@
 # - Only a simple smoketest (CLI check) is made, essentially YANG parsing. A full system may not work
 # - Env variable YANGMODELS should point to checkout place. (define it in site.sh for example)
 # - Some FEATURES are set to make it work
-# - Some DIFFs are necessary in yangmodels
+# - Some YANGmodels are broken, therefore CLICON_YANG_AUGMENT_ACCEPT_BROKEN is true
-#     - standard/ietf/RFC/ietf-mud@2019-01-28.yang
-#           -      + "/acl:l4/acl:tcp/acl:tcp" {
-#           +      + "/acl:l4/acl:tcp" {
-#     - standard/ietf/RFC/ietf-acldns@2019-01-28.yang
-#                augment "/acl:acls/acl:acl/acl:aces/acl:ace/acl:matches"
-#            -        + "/acl:l3/acl:ipv4/acl:ipv4" {
-#            +        + "/acl:l3/acl:ipv4" {
-#                 description
-#                   "Adding domain names to matching.";
-#            +    if-feature acl:match-on-ipv4;
-#                 uses dns-matches;
-#               }
-#               augment "/acl:acls/acl:acl/acl:aces/acl:ace/acl:matches"
-#            -        + "/acl:l3/acl:ipv6/acl:ipv6" {
-#            +        + "/acl:l3/acl:ipv6" {
-#                 description
-#                   "Adding domain names to matching.";
-#            +    if-feature acl:match-on-ipv6;
-
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@@ -46,9 +27,13 @@ cat <<EOF > $cfg
   <CLICON_FEATURE>ietf-subscribed-notifications:configured</CLICON_FEATURE>
   <CLICON_FEATURE>ietf-subscribed-notifications:replay</CLICON_FEATURE>
   <CLICON_FEATURE>ietf-access-control-list:match-on-tcp</CLICON_FEATURE>
+  <CLICON_FEATURE>ietf-te-topology:template</CLICON_FEATURE>
+  <CLICON_FEATURE>ietf-te-topology:te-topology-hierarchy</CLICON_FEATURE>
+  <CLICON_FEATURE>ietf-te-types:path-optimization-metric</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$YANGMODELS/standard/ieee/published/802.1</CLICON_YANG_DIR> 
   <CLICON_YANG_DIR>$YANGMODELS/standard/ietf/RFC</CLICON_YANG_DIR>	
+  <CLICON_YANG_AUGMENT_ACCEPT_BROKEN>true</CLICON_YANG_AUGMENT_ACCEPT_BROKEN>
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>

yang/README.md

@@ -1,12 +1,11 @@
 # Yang files
 
-There are three classes of Yang files
+There are two classes of Yang files
   * Clixon yang files.
   * Mandatory: "Standard" yang files necessary for clixon lib/client/backend to run
-  * Optional: "Standard" yang files for examples and tests 
 
-The first two (clixon and mandatory) are always installed. If you want
+Both are always installed. If you want to change where the are
-to change where the are installed, configure with: `--with-yang-installdir=DIR`
+installed, configure with: `--with-yang-installdir=DIR`
 
 The third (optional) is only installed if configure flag
 `--enable-optyang` is set. Further, the optional yang files are

yang/clixon/Makefile.in

@@ -38,7 +38,6 @@ bindir 		= @bindir@
 includedir	= @includedir@
 datarootdir     = @datarootdir@
 
-# See also OPT_YANG_INSTALLDIR for the standard yang files
 YANG_INSTALLDIR   = @YANG_INSTALLDIR@
 
 YANGSPECS	 = clixon-config@2021-07-11.yang   # 5.3

yang/clixon/clixon-config@2021-11-11.yang

@@ -47,6 +47,7 @@ module clixon-config {
 	description
 	    "Added option:
                     CLICON_PLUGIN_CALLBACK_CHECK
+                    CLICON_YANG_AUGMENT_ACCEPT_BROKEN
 	     Modified options:
                     CLICON_CLI_GENMODEL_TYPE: added OC_COMPRESS enum
                     CLICON_YANG_DIR: recursive search
@@ -936,7 +937,8 @@ module clixon-config {
 	    type boolean;
 	    default false;
 	    description
-		"If enabled, make a check of resources before and after each plugin callback code
+		"Debug option.
+                 If enabled, make a check of resources before and after each plugin callback code
                  to check if the plugin violated resources.
                  This is primarily intended for development and debugging but may also be enabled 
                  in a running system.
@@ -950,6 +952,21 @@ module clixon-config {
                  as well as the CLIgen callbacks.
                  See https://clixon-docs.readthedocs.io/en/latest/backend.html#plugin-callback-guidelines";
 	}
+	leaf CLICON_YANG_AUGMENT_ACCEPT_BROKEN {
+	    type boolean;
+	    default false;
+	    description
+		"Debug option. If enabled, accept broken augments on the form:
+                    augment <target> { ... }
+                 where <target> is an XPath which MUST be an existing node but for many
+                 yangmodels do not.
+                 There are several cases why this may be the case:
+                 - syntax errors,
+                 - features that need to be enabled
+                 - wrong XPaths, etc
+                 This option should be enabled only for passing some testcases it should 
+                 normally never be enabled in system YANGs that are used in a system.";
+	}
 	leaf CLICON_NAMESPACE_NETCONF_DEFAULT {
 	    type boolean;
 	    default false;

yang/mandatory/Makefile.in

@@ -42,7 +42,7 @@ datarootdir     = @datarootdir@
 # See also YANG_INSTALLDIR for the clixon-specific yang files
 YANG_INSTALLDIR   = @YANG_INSTALLDIR@
 
-YANGSPECS += ietf-inet-types@2020-07-06.yang
+YANGSPECS += ietf-inet-types@2021-02-22.yang
 YANGSPECS += ietf-netconf@2011-06-01.yang
 YANGSPECS += ietf-netconf-acm@2018-02-14.yang
 YANGSPECS += ietf-restconf@2017-01-26.yang

yang/mandatory/ietf-inet-types@2021-02-22.yang

@@ -23,7 +23,7 @@ module ietf-inet-types {
     described in BCP 14 (RFC 2119) (RFC 8174) when, and only when,
     they appear in all capitals, as shown here.
 
-    Copyright (c) 2020 IETF Trust and the persons identified as
+    Copyright (c) 2021 IETF Trust and the persons identified as
     authors of the code.  All rights reserved.
 
     Redistribution and use in source and binary forms, with or
@@ -35,23 +35,26 @@ module ietf-inet-types {
 
     This version of this YANG module is part of RFC XXXX;
     see the RFC itself for full legal notices.";
-  revision 2020-07-06 {
+
+  revision 2021-02-22 {
     description
      "This revision adds the following new data types:
-      - ip-address-and-prefix
+      - inet:ip-address-and-prefix
-      - ipv4-address-and-prefix
+      - inet:ipv4-address-and-prefix
-      - ipv6-address-and-prefix
+      - inet:ipv6-address-and-prefix
-      - email-address";
+      - inet:host-name
+      - inet:email-address
+      The inet:host union was changed to use inet:host-name instead
+      of inet:domain-name.";
     reference
      "RFC XXXX: Common YANG Data Types";
   }
-
   revision 2013-07-15 {
     description
      "This revision adds the following new data types:
-      - ip-address-no-zone
+      - inet:ip-address-no-zone
-      - ipv4-address-no-zone
+      - inet:ipv4-address-no-zone
-      - ipv6-address-no-zone";
+      - inet:ipv6-address-no-zone";
     reference
      "RFC 6991: Common YANG Data Types";
   }
@@ -94,7 +97,6 @@ module ietf-inet-types {
       RFC 2460: Internet Protocol, Version 6 (IPv6) Specification
       RFC 4001: Textual Conventions for Internet Network Addresses";
   }
-
   typedef dscp {
     type uint8 {
       range "0..63";
@@ -143,7 +145,6 @@ module ietf-inet-types {
       Note that the port number value zero is reserved by IANA.  In
       situations where the value zero does not make sense, it can
       be excluded by subtyping the port-number type.
-
       In the value set and its semantics, this type is equivalent
       to the InetPortNumber textual convention of the SMIv2.";
     reference
@@ -323,6 +324,7 @@ module ietf-inet-types {
       A prefix length value of n corresponds to an IP address
       mask that has n contiguous 1-bits from the most
       significant bit (MSB) and all other bits set to 0.
+
       The canonical format of an IPv4 prefix has all bits of
       the IPv4 address set to zero that are not part of the
       IPv4 prefix.
@@ -450,10 +452,9 @@ module ietf-inet-types {
       for current practice in domain name use, and some possible
       future expansion.  Note that Internet host names have a
       stricter syntax (described in RFC 952) than the DNS
-      recommendations in RFCs 1034 and 1123, and that systems
+      recommendations in RFCs 1034 and 1123. Schema nodes
-      that want to store host names in schema node instances
+      representing host names should use the host-name type
-      using the domain-name type are recommended to adhere to
+      instead of the domain-type.
-      this stricter standard to ensure interoperability.
 
       The encoding of DNS names in the DNS protocol is limited
       to 255 characters.  Since the encoding consists of labels
@@ -486,21 +487,39 @@ module ietf-inet-types {
                 (IDNA): Definitions and Document Framework";
   }
 
+  typedef host-name {
+    type domain-name {
+      pattern '[a-zA-Z0-9\-\.]+';
+      length "2..max";
+    }
+    description
+     "The host-name type represents (fully qualified) host names.
+      Host names must be at least two characters long (see RFC 952)
+      and they are restricted to labels consisting of letters, digits
+      and hyphens separated by dots (see RFC1123 and RFC 952).";
+    reference
+     "RFC  952: DoD Internet Host Table Specification
+      RFC 1123: Requirements for Internet Hosts: Application and Support";
+  }
+
   typedef host {
     type union {
       type inet:ip-address;
-      type inet:domain-name;
+      type inet:host-name;
     }
     description
-     "The host type represents either an IP address or a DNS
+     "The host type represents either an IP address or a (fully
-      domain name.";
+      qualified) host name.";
   }
 
   /*
    * DISCUSS:
-   * - Lada suggested to replace the inet:domain-name usage in
+   * - It was discussed to define int-domain-name and int-host-name
-   *   the union with a new host-name definition that follows
+   *   that use U-labels instead of A-labels and to add int-host-name
-   *   the NR-LDH definition in RFC 5890.
+   *   to the inet:host union, perhaps all gated by an inet:idna-aware
+   *   feature.
+   * - It is not clear how inet:idna-aware affects inet:email-address
+   *   and inet:uri - do we also need int-uri and int-email-address?
    */
 
   typedef uri {
@@ -560,15 +579,6 @@ module ietf-inet-types {
 
   /*
    * DISCUSS:
-   * - It was suggested to add email types following RFC 5322
-   *   email-address        (addr-spec, per Section 3.4.1)
-   *   named-email-address  (name-addr, per Section 3.4)
-   * - This sounds useful but the devil is in the details,
-   *   in particular name-addr is a quite complex construct;
-   *   perhaps addr-spec is sufficient, this is also the
-   *   format allowed in mailto: URIs (mailto: seems to use
-   *   only a subset of addr-spec which may be good enough
-   *   here as well).
    * - Need to define a pattern that has a meaningful trade-off
    *   between precision and complexity (there are very tight
    *   pattern that are very long and complex). The current
@@ -576,14 +586,4 @@ module ietf-inet-types {
    *   domain-literal, obs-domain.
    */
 
-  /*
-   * DISCUSS:
-   * - There was a request to add types for URI fields (scheme,
-   *   authority, path, query, fragment) but it is not clear how
-   *   commonly useful these types are, the WG was pretty silent
-   *   about this proposal. On the technical side, it is unclear
-   *   whether data is represented with percent escapes resolved
-   *   or not. (Mahesh's proposal does not spell this out, the
-   *   pattern does not allow the % character, which may be wrong.)
-   */
 }