Clixon 5.4.0 release

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Clixon Changelog
 
-* [5.4.0](#540) Expected: November
+* [5.4.0](#540) 30 November 2021
 * [5.3.0](#530) 27 September 2021
 * [5.2.0](#520) 1 July 2021
 * [5.1.0](#510) 15 April 2021
@@ -32,9 +32,9 @@
 * [3.3.1](#331) June 7 2017
 
 ## 5.4.0
-Expected: November, 2021
+30 November, 2021
 
-Thanks Netgate for providing the dispatcher code (used in the pagination)!
+This release features lots of minor updates and bugfixes, an updated list pagination and optimized auto-cli for large yang-specs. Thanks Netgate for providing the dispatcher code used in the new pagination API!
 
 ### New features
 
@@ -124,7 +124,7 @@ Developers may need to change their code
   * Check blocked signals and signal handlers
   * Check termios settings
   * Any changes to context are logged at loglevel WARNING
-  * New option: `CLICON_PLUGIN_CALLBACK_CHECK`: enable it to for checks (default false)
+  * New option: `CLICON_PLUGIN_CALLBACK_CHECK`: set to 1 to get checks, 2, to abort on failure (default 0)
 * Added: [OpenConfig Path Compression Support](https://github.com/clicon/clixon/issues/274)
   * PR: [OpenConfig path compression](https://github.com/clicon/clixon/pull/276)
 * C API: Added set/get pointer API to clixon_data:

configure

@@ -1370,10 +1370,10 @@ Optional Features:
   --enable-yang-patch     Enable YANG patch, RFC 8072, default: no
   --enable-publish        Enable publish of notification streams using SSE and
                           curl
-  --disable-evhtp         Disable evhtp for native restconf http/1, default:
+  --disable-evhtp         Disable evhtp for native restconf http/1, ie http/2
-                          yes
+                          only
-  --disable-nghttp2       Disable nghttp2 for native restconf http/2, default:
+  --disable-nghttp2       Disable nghttp2 for native restconf http/2, ie
-                          yes
+                          http/1 only
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -1390,8 +1390,8 @@ Optional Packages:
                           Install Clixon yang files here (default:
                           ${prefix}/share/clixon)
   --with-yang-standard-dir=DIR
-                          Where standard IETF/IEEE YANGs are (default:
+                          Directory of standard IETF/IEEE YANG specs (default:
-                          ${prefix}/share/yang/standard)
+                          $prefix/share/yang/standard)
 
 Some influential environment variables:
   CC          C compiler command
@@ -2233,7 +2233,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 : ${INSTALLFLAGS="-s"}
 : ${LINKAGE=dynamic}
 
-
 # where autotool scripts are: install-sh, config.sub, config.guess
 ac_aux_dir=
 for ac_dir in config-aux "$srcdir"/config-aux; do
@@ -2268,7 +2267,7 @@ ac_configure="$SHELL $ac_aux_dir/configure"  # Please don't use this var.
 CLIXON_VERSION_MAJOR="5"
 CLIXON_VERSION_MINOR="4"
 CLIXON_VERSION_PATCH="0"
-CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\""
+CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\""
 
 # Debug flag
 # Check whether --enable-debug was given.

configure.ac

@@ -44,14 +44,13 @@ AC_INIT(lib/clixon/clixon.h.in)
 : ${INSTALLFLAGS="-s"}
 : ${LINKAGE=dynamic}
 
-
 # where autotool scripts are: install-sh, config.sub, config.guess
 AC_CONFIG_AUX_DIR(config-aux)
 
 CLIXON_VERSION_MAJOR="5"
 CLIXON_VERSION_MINOR="4"
 CLIXON_VERSION_PATCH="0"
-CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\""
+CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\""
 
 # Debug flag
 AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[Build with debug symbols, default: no]),[
@@ -232,7 +231,7 @@ elif test "x${with_restconf}" == xnative; then
    AC_CHECK_LIB(ssl, OPENSSL_init_ssl ,, AC_MSG_ERROR([libssl missing]))
    AC_CHECK_LIB(crypto, CRYPTO_new_ex_data, , AC_MSG_ERROR([libcrypto missing])) 
    # Check if evhtp is enabled for http/1
-   AC_ARG_ENABLE(evhtp, AS_HELP_STRING([--disable-evhtp],[Disable evhtp for native restconf http/1, default: yes]),[
+   AC_ARG_ENABLE(evhtp, AS_HELP_STRING([--disable-evhtp],[Disable evhtp for native restconf http/1, ie http/2 only]),[
    	  if test "$enableval" = no; then
 	      ac_enable_evhtp=no
 	  else	      
@@ -255,7 +254,7 @@ elif test "x${with_restconf}" == xnative; then
    fi		  
 
    # Check if nghttp2 is enabled for http/2
-   AC_ARG_ENABLE(nghttp2, AS_HELP_STRING([--disable-nghttp2],[Disable nghttp2 for native restconf http/2, default: yes]),[
+   AC_ARG_ENABLE(nghttp2, AS_HELP_STRING([--disable-nghttp2],[Disable nghttp2 for native restconf http/2, ie http/1 only]),[
 	  if test "$enableval" = no; then
 	      ac_enable_nghttp2=no
 	  else	      
@@ -341,7 +340,7 @@ AC_MSG_RESULT(Clixon yang files are installed in ${YANG_INSTALLDIR})
 # This is NOT installed by Clixon and is not needed for core system
 # However, it is required by the main example and some of the tests
 AC_ARG_WITH(yang-standard-installdir,
-            [AS_HELP_STRING([--with-yang-standard-dir=DIR],[Where standard IETF/IEEE YANGs are (default: ${prefix}/share/yang/standard)])],
+            [AS_HELP_STRING([--with-yang-standard-dir=DIR],[Directory of standard IETF/IEEE YANG specs (default: $prefix/share/yang/standard)])],
 	    [YANG_STANDARD_DIR="$withval"],
     	    [YANG_STANDARD_DIR="${prefix}/share/yang/standard"]		
 	    )

docker/README.md

@@ -3,5 +3,5 @@
 This directory contains sub-directories with examples of Clixon docker images:
 
   * [base](base/README.md)  Clixon base image 
-  * [main](main/README.md)  Main example and test application 
+  * [main](main/README.md)  Main example and test application used in upstream CI
 

docker/base/Dockerfile

@@ -38,15 +38,18 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison curl-dev
 
+# evhtp dependencies
+RUN apk add --update libevent libevent-dev
+
+# nghttp2 dependencies
+RUN apk add --update nghttp2
+
 # Create a directory to hold source-code, dependencies etc
 RUN mkdir /clixon
 
-# libevht
-# dependencies
-RUN apk add --update libevent cmake libevent-dev
-
 # clone libevhtp
 WORKDIR /clixon
+
 RUN git clone https://github.com/clicon/clixon-libevhtp.git
 WORKDIR /clixon/clixon-libevhtp
 RUN ./configure
@@ -67,19 +70,16 @@ RUN ./configure --prefix=/clixon/build
 RUN make
 RUN make install
 
+# Need to add www user manually
+RUN adduser -D -H -G www-data www-data
+
 # Copy Clixon from local dir
 RUN mkdir /clixon/clixon
 WORKDIR /clixon/clixon
 COPY clixon .
 
-# Need to add www user manually
-RUN adduser -D -H www-data
-
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --with-wwwuser=www-data --with-restconf=native
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --with-restconf=native --enable-nghttp2 --enable-evhtp
-
-RUN make
-RUN make install
 
 RUN make
 RUN make install
@@ -94,25 +94,19 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 RUN apk add --update flex bison
 
 # need to add www user manually
-RUN adduser -D -H www-data
+RUN adduser -D -H -G www-data www-data
 # for libevtp
 RUN apk add --update openssl libevent
 
-# Test-specific (for test scripts) XXX Maybe remove those?
+# nghttp2 dependencies
-RUN apk add --update sudo curl procps grep make bash
+RUN apk add --update nghttp2
 
 # Expose https port for restconf
-EXPOSE 80
+EXPOSE 80/tcp
+EXPOSE 443/tcp
 
 # Create clicon user and group
 RUN adduser -D -H clicon
-RUN adduser www-data clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
-COPY --from=0 /www-data /www-data
+COPY --from=0 /usr/local/lib/libevhtp.so* /usr/local/lib/
-
-# Manually created
-RUN chown www-data /www-data
-RUN chgrp www-data /www-data
-
-

docker/base/Makefile.in

@@ -44,7 +44,7 @@ IMG   = clixon/clixon # base image
 
 SHELL	= /bin/sh
 
-.PHONY: all clean distclean docker push depend install-include install uninstall test
+.PHONY: all clean distclean docker push depend install-include install uninstall
 
 all:	
 	echo "Run make docker to build docker image"
@@ -59,8 +59,6 @@ clean:
 distclean: clean
 	rm -f Makefile *~ .depend
 
-test:
-
 docker: clixon Dockerfile
 	sudo docker build -t $(IMG) .  # --no-cache
 

docker/base/README.md

@@ -1,8 +1,6 @@
 # Clixon base docker image
 
-This directory contains code for building and pushing the clixon base docker
+This directory contains code for building a clixon base docker container.
-container. By default it is pushed to docker hub clixon/clixon, but you can change
-the IMAGE in Makefile.in and push it to another name.
 
 This clixon base container uses native http.
 
@@ -10,12 +8,12 @@ The clixon docker base image can be used to build clixon
 applications. It has the whole code for a clixon release which it
 downloads from git.
 
-See [clixon-system](../main/README.md) for a more complete clixon image.
+## Build
-
-## Build and push
 
 Perform the build by `make docker`. This copies the latest _committed_ clixon code into the container.
 
+## Push
+
 You may also do `make push` if you want to push the image, but you may then consider changing the image name (in the makefile:s).
 
 (You may have to login for push with sudo docker login -u <username>)

docker/base/start.sh

@@ -1,11 +1,42 @@
 #!/bin/bash
+# ***** BEGIN LICENSE BLOCK *****
+# 
+# Copyright (C) 2017-2019 Olof Hagsand
+# Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)
+#
+# This file is part of CLIXON
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 3 or later (the "GPL"),
+# in which case the provisions of the GPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of the GPL, and not to allow others to
+# use your version of this file under the terms of Apache License version 2, 
+# indicate your decision by deleting the provisions above and replace them with
+# the notice and other provisions required by the GPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the Apache License version 2 or the GPL.
+#
+# ***** END LICENSE BLOCK *****
+
 # Usage: ./startup.sh
 # Debug: DBG=1 ./startup.sh
 # See also cleanup.sh
 
 >&2 echo "Running script: $0"
 
-
 # Start clixon-example backend
 sudo docker run --name clixon --rm -td clixon/clixon || err "Error starting clixon"
 

docker/main/Dockerfile.native

@@ -153,10 +153,5 @@ COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
 COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/experimental/
 COPY --from=0 /usr/local/share/openconfig/* /usr/local/share/openconfig/
 
-# Manually created
-RUN mkdir /www-data
-RUN chown clicon /www-data
-RUN chgrp clicon /www-data
-
 # Start the backend and restconf deamons
 CMD /usr/local/bin/startsystem.sh

lib/src/clixon_plugin.c

@@ -557,8 +557,10 @@ plugin_context_get(void)
  * 1) Make a check of resources
  * 2) Make a new check and compare with the old check, return 1 on success, 0 on fail
  * Log if there is a difference at loglevel WARNING.
- * You can modify the code to also fail with assert if you want early fail.
+ * Controlled by CLICON_PLUGIN_CALLBACK_CHECK:
- * Controlled by option 
+ * 0 : No checks
+ * 1 : warning logs on failure
+ * 2 : log and abort on failure
  *
  * @param[in]     h      Clixon handle
  * @param[in,out] wh     Either: NULL for init, will be assigned, OR previous handle (will be freed)
@@ -583,13 +585,15 @@ plugin_context_check(clicon_handle h,
     int                    i;
     struct plugin_context *oldpc;
     struct plugin_context *newpc = NULL;
+    int                    option;
 
     if (h == NULL){
 	errno = EINVAL;
 	return -1;
     }
+    option = clicon_option_int(h, "CLICON_PLUGIN_CALLBACK_CHECK");
     /* Check if plugion checks are enabled */
-    if (!clicon_option_bool(h, "CLICON_PLUGIN_CALLBACK_CHECK"))
+    if (option == 0)
 	return 1;
     if (wh == NULL){
 	errno = EINVAL;
@@ -631,11 +635,10 @@ plugin_context_check(clicon_handle h,
 	failed++;
     }
     /* XXX pc_termios.cc_t  c_cc[NCCS] not checked */
-#if 0
+    /* Abort if option is 2 or above on failure
-	/* In case you want early detection and crash. But otherwise it is recommended that
+     */
-	 * the caller looks for retval == 0 */
+    if (option > 1 && failed)
-	assert(failed == 0);
+	abort();
-#endif
     for (i=1; i<32; i++){
 	if (sigismember(&oldpc->pc_sigset, i) != sigismember(&newpc->pc_sigset, i)){
 	    clicon_log(LOG_WARNING, "%s Plugin context %s %s: Changed blocking of signal %s(%d) from %d to %d", __FUNCTION__,
@@ -659,11 +662,10 @@ plugin_context_check(clicon_handle h,
 		       newpc->pc_sigaction_vec[i].sa_sigaction);
 	    failed++;
 	}
-#if 0
+	/* Abort if option is 2 or above on failure
-	/* In case you want early detection and crash. But otherwise it is recommended that
+	 */
-	 * the caller looks for retval == 0 */
+	if (option > 1 && failed)
-	assert(failed == 0);
+	    abort();
-#endif
     }
     if (failed)
 	goto fail;

lib/src/clixon_yang_type.c

@@ -1400,8 +1400,10 @@ yang_type_resolve(yang_stmt   *yorig,
     retval = 0;
   done:
 #if 1
-    if (retval == 0 && yrestype != NULL) /* Assert that on success, yrestype is set */
+    if (retval == 0 && yrestype != NULL && *yrestype == NULL){
-	assert(*yrestype);
+	clicon_err(OE_YANG, 0, "No such type: \"%s\"", type);
+	retval = -1;
+    }
 #endif
     if (prefix)
 	free(prefix);

yang/clixon/clixon-config@2021-11-11.yang

@@ -934,17 +934,16 @@ module clixon-config {
                  but disable it in production, until this has been resolved.";
 	}
 	leaf CLICON_PLUGIN_CALLBACK_CHECK {
-	    type boolean;
+	    type int32;
-	    default false;
+	    default 0;
 	    description
 		"Debug option.
-                 If enabled, make a check of resources before and after each plugin callback code
+                 If >0, make a check of resources before and after each plugin callback code
                  to check if the plugin violated resources.
                  This is primarily intended for development and debugging but may also be enabled 
                  in a running system.
-                 If enabled, errors will be logged to syslog as WARNINGs.
+                 If 1, errors will be logged to syslog as WARNINGs.
-                 In case you want early detection and crash, you can uncomment assert statements and
+                 If 2, the program will abort using assert() on first error
-                 recompile.
                  The checks are currently made by plugin_context_check() and include:
                  - termios settings
                  - signal vectors