From 51278d5901b4bd44f6df8edb0fec04b179dae3d7 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 27 Sep 2021 12:23:34 +0200 Subject: [PATCH] Fixed static linking for coverage and fuzzing Preparations for Clixon 5.3 release --- CHANGELOG.md | 69 +++++++++++------------ README.md | 4 +- apps/cli/Makefile.in | 13 ++++- apps/netconf/Makefile.in | 1 + apps/restconf/Makefile.in | 1 + example/main/Makefile.in | 5 +- fuzz/cli/README.md | 4 +- lib/src/Makefile.in | 1 + test/cicd/coverage.sh | 31 ++++++---- test/test_cli_leafref.sh | 3 +- test/test_install.sh | 4 +- yang/clixon/clixon-config@2021-07-11.yang | 15 +---- 12 files changed, 83 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5f941d0..513f435b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Clixon Changelog -* [5.3.0](#530) Expected: September 2021 +* [5.3.0](#530) 27 September 2021 * [5.2.0](#520) 1 July 2021 * [5.1.0](#510) 15 April 2021 * [5.0.0](#500) 27 February 2021 @@ -31,28 +31,34 @@ * [3.3.1](#331) June 7 2017 ## 5.3.0 -Expected: September, 2021 +27 September, 2021 + +The 5.3 release has pagination support, YANG/auto-clilinkref changes and lots of bug fixes. ### New features * List pagination for Netconf and Restconf - * Experimental, work-in-progress - * Enable with LIST_PAGINATION compile-time option - * According to: + * Loosely based on: * draft-wwlh-netconf-list-pagination-00.txt * draft-wwlh-netconf-list-pagination-rc-01 + * Note: not a standardized feature * Added yangs: * ietf-restconf-list-pagination@2015-01-30.yang * clixon-netconf-list-pagination@2021-08-27.yang * ietf-yang-metadata@2016-08-05.yang - * New http media: application/yang-collection+xml/json + * Restconf change: + * New http media: application/yang-collection+xml/json * Updated state callback signature containing parameters for pagination * See API changes below + * See [User manual pagination](https://clixon-docs.readthedocs.io/en/latest/misc.html#pagination) + * Work-in-progress + * Enable remaining attriute with LIST_PAGINATION_REMAINING compile-time option + * sort/direction/where etc not supported * YANG Leafref feature update * Closer adherence to RFC 7950. Some of this is changed behavior, some is new feature. - * Essentially instead of looking at the referring leaf, context is referred(target) node * Validation uses referred node * Validation changed to use type of referred node, instead of just "string" + * Essentially instead of looking at the referring leaf, context is referred(target) node * Auto-cli * Changed to use type of referred node for typecheck * Completion uses referred node @@ -62,14 +68,6 @@ Expected: September, 2021 * Restconf YANG PATCH according to RFC 8072 (Work in progress) * Experimental: enable by setting YANG_PATCH in include/clixon_custom.h * Thanks to Alan Yaniger for providing this patch -* List pagination - * This is prototype work for ietf netconf work - - -* New state callback signature (ca_statedata2) - * The new callback contains parameters for pagination - * Goal is to replace ca_statedata callback - ### API changes on existing protocol/config features @@ -79,13 +77,13 @@ Users may have to change how they access the system * Leafref required-instance must be set to make strict data-node check * See changes under new feature "YANG leafref feature update" * Native Restconf - * Native restocnf is now default, not fcgi/nginx - * That is, to configure with fcgi, you need to explicitly configure: `--with-restconf=fcgi` + * Native restconf is now default, not fcgi/nginx + * To configure with fcgi, you need to explicitly configure: `--with-restconf=fcgi` * SSL client certs failures are returned as http 405 errors, not fail during SSL negotiation * New clixon-config@2021-07-11.yang revision * Added: `CLICON_RESTCONF_HTTP2_PLAIN` * Removed default of `CLICON_RESTCONF_INSTALLDIR` - * The default behaviour is changed to use the config $(sbindir) to locate `clixon_restconf` when starting restconf internally + * The default behaviour is changed to use the config `$(sbindir)` to locate `clixon_restconf` when starting restconf internally ### C/CLI-API changes on existing features @@ -93,7 +91,6 @@ Developers may need to change their code * You need to change all statedata plugin callback for the new pagination feature * If you dont use pagination you can ignore the values of the new parameters - * See [User manual pagination](https://clixon-docs.readthedocs.io/en/latest/misc.html#pagination) * The updated callback signature is as follows: ``` int statedata(clicon_handle h, @@ -108,24 +105,24 @@ Developers may need to change their code ### Minor features -* Added -H option to clixon_netconf: Do not require hello before request -* CLIXON_STATIC_PLUGIN to support statically linked plugins +* Fuzzing: + * Added netconf fuzzing + * Added `CLIXON_STATIC_PLUGINS` and description how to link CLI plugins statically + * See `fuzz/cli`, `fuzz/netconf` +* Added `-H` option to clixon_netconf: Do not require hello before request * JSON errors are now labelled with JSON and not XML * Restconf native HTTP/2: * Added option `CLICON_RESTCONF_HTTP2_PLAIN` for non-TLS http * Default disabled, set to true to enable HTTP/2 direct and switch/upgrade HTTP/1->HTTP/2 + * Recommendation is to used only TLS HTTP/2 * JSON encoding of YANG metadata according to RFC 7952 * XML -> JSON translation + * Note: JSON -> XML metadata is not implemented * Restconf internal start: fail early if clixon_restconf binary is not found - * If CLICON_BACKEND_RESTCONF_PROCESS is true + * If `CLICON_BACKEND_RESTCONF_PROCESS` is true * Added linenumbers to all YANG symbols for better debug and errors * Improved error messages for YANG identityref:s and leafref:s by adding original line numbers -### Minor features - -* ietf-yang-metadata RFC 7952 support, placeholder parsing and extension - * No actual json/xml semantics - ### Corrected Bugs * Partly Fixed: [String concatenation in YANG model leads to syntax error](https://github.com/clicon/clixon/issues/265) @@ -134,8 +131,8 @@ Developers may need to change their code * Fixed: [Duplicate lines emitted by cli_show_config (cli output style) when yang list element has composite key](https://github.com/clicon/clixon/issues/258) * Fixed: Typing 'q' in CLI more scrolling did not properly quit output * Output continued but was not shown, for a very large file this could cause considerable delay -* Fixed: Lock was broken in first get get access - * if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access. +* Fixed: Lock was broken in first get access + * If the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access. * Fixed: [JSON leaf-list output single element leaf-list does not use array](https://github.com/clicon/clixon/issues/261) * Fixed: Netconf diff callback did not work with choice and same value replace * Eg if YANG is `choice c { leaf x; leaf y }` and XML changed from `42` to `42` the datastrore changed, but was not detected by diff algorithms and provided to validate callbacks. @@ -143,22 +140,22 @@ Developers may need to change their code * Fixed: [Autocli does not offer completions for leafref to identityref #254](https://github.com/clicon/clixon/issues/254) * This is a part of YANG Leafref feature update * Fixed: [clixon_netconf errors on client XML Declaration with valid encoding spec](https://github.com/clicon/clixon/issues/250) -* Fixed: Yang patterns: \n and other non-printable characters were broken - * Example: Clixon interpereted them two characters: `\\ n` instead of ascii 10 +* Fixed: Yang patterns: `\n` and other non-printable characters were broken + * Example: Clixon interpereted them as the two characters: `\\` and `n` instead of ascii 10 * Fixed: The auto-cli identityref did not expand identities in grouping/usecases properly. * Fixed: [OpenConfig BGP afi-safi and when condition issues #249](https://github.com/clicon/clixon/issues/249) - * YANG when was not properly implemented for default values + * YANG "when" was not properly implemented for default values * Fixed: SEGV in clixon_netconf_lib functions from internal errors including validation. - * Check xerr argument both before and after call on netconf lib functions + * Check `xerr` argument both before and after call on netconf lib functions * Fixed: Leafs added as augments on NETCONF RPC input/output lacked cv:s causing error in default handling * Fixed: RFC 8040 yang-data extension allows non-key lists - * Added YANG_FLAG_NOKEY as exception to mandatory key lists + * Added `YANG_FLAG_NOKEY` as exception to mandatory key lists * Fixed: mandatory leaf in a uses statement caused abort - * Occurence was in ietf-yang-patch.yang + * Occurence was in `ietf-yang-patch.yang` * Native RESTCONF fixes for http/1 or http/2 only modes * Memleak in http/1-only * Exit if http/1 request sent to http/2-only (bad client magic) - * Hang if http/1 TLS request sent to http/2 only (alpn accepted http/1.1) + * Hang if http/1 TLS request sent to http/2 only (ALPN accepted http/1.1) * Fixed: [RESTConf GET for a specific list instance retrieves data from other submodules that have same list name and key value #244](https://github.com/clicon/clixon/issues/244) ## 5.2.0 diff --git a/README.md b/README.md index 0cde4fe0..00604026 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ See [documentation](https://clixon-docs.readthedocs.io), [project page](https:// Clixon is open-source and dual licensed. Either Apache License, Version 2.0 or GNU General Public License Version 2; you choose, see [LICENSE.md](LICENSE.md). -Latest release is 5.2.0 released on July 1st 2021. See [CHANGELOG.md](CHANGELOG.md) release history. +Latest release is 5.3.0 released in September 2021. See [CHANGELOG.md](CHANGELOG.md) release history. Clixon interaction is best done posting issues, pull requests, or joining the [slack channel](https://clixondev.slack.com). -[Slack invite](https://join.slack.com/t/clixondev/shared_invite/zt-seopvltv-hs~BS7UrMjRdKoDRlCK97w) (updated 30/6 2021) +[Slack invite](https://join.slack.com/t/clixondev/shared_invite/zt-wegsemtw-u~VRdvYPtSRKXqULDLlWJQ) (updated 26/9 2021) Clixon is sponsored by [Rubicon Communications LLC(Netgate)](https://www.netgate.com/) diff --git a/apps/cli/Makefile.in b/apps/cli/Makefile.in index 8152e4b2..a8280f56 100644 --- a/apps/cli/Makefile.in +++ b/apps/cli/Makefile.in @@ -70,8 +70,17 @@ ifeq ($(LINKAGE),dynamic) CLIXON_LIB = libclixon$(SH_SUFFIX).$(CLIXON_MAJOR).$(CLIXON_MINOR) else CLIXON_LIB = libclixon.a + +# Somewhat ad-hoc mechanism to find libcligen: +# First find in DESTDIR, if not, try root +ifneq ("$(wildcard $(DESTDIR)$(libdir)/libcligen.a)","") +CLIGEN_LIB = $(DESTDIR)$(libdir)/libcligen.a +else +CLIGEN_LIB = $(libdir)/libcligen.a endif +endif # LINKAGE + # For dependency. A little strange that we rely on it being built in the src dir # even though it may exist in $(libdir). But the new version may not have been installed yet. LIBDEPS = $(top_srcdir)/lib/src/$(CLIXON_LIB) @@ -150,6 +159,7 @@ install-include: clixon_cli.h clixon_cli_api.h cli_generate.h uninstall: rm -f $(DESTDIR)$(bindir)/$(APPL) rm -f $(DESTDIR)$(libdir)/$(MYLIBLINK)* + rm -f $(DESTDIR)$(libdir)/$(MYLIB) rm -f $(DESTDIR)$(includedir)/clixon/* .SUFFIXES: @@ -165,13 +175,14 @@ test.c : test: test.c $(LIBOBJ) $(MYLIB) $(CC) $(INCLUDES) $(LDFLAGS) $< $(LIBOBJ) -L. $(MYLIB) $(LIBS) -o $@ + # Note LIBDEPS is in lib/src and will always be remade du to a date dependency $(APPL): $(APPOBJ) $(MYLIB) $(LIBDEPS) ifeq ($(LINKAGE),dynamic) $(CC) $(LDFLAGS) $(APPOBJ) -L. $(MYLIB) $(LIBS) -o $@ else # Force static libcligen.a linking - $(CC) $(LDFLAGS) $(APPOBJ) $(EXTRAS) -L. $(LIBOBJ) $(DESTDIR)$(libdir)/libcligen.a $(LIBS) -o $@ + $(CC) $(LDFLAGS) $(APPOBJ) $(EXTRAS) -L. $(LIBOBJ) $(CLIGEN_LIB) $(LIBS) -o $@ endif $(MYLIBDYNAMIC) : $(LIBOBJ) $(LIBDEPS) diff --git a/apps/netconf/Makefile.in b/apps/netconf/Makefile.in index afe7b396..ef21b8af 100644 --- a/apps/netconf/Makefile.in +++ b/apps/netconf/Makefile.in @@ -139,6 +139,7 @@ install-include: clixon_netconf.h uninstall: rm -f $(DESTDIR)$(bindir)/$(APPL) rm -f $(DESTDIR)$(libdir)/$(MYLIBLINK)* + rm -f $(DESTDIR)$(libdir)/$(MYLIB) rm -f $(DESTDIR)$(includedir)/clixon/* .SUFFIXES: diff --git a/apps/restconf/Makefile.in b/apps/restconf/Makefile.in index c5565952..db67ad22 100644 --- a/apps/restconf/Makefile.in +++ b/apps/restconf/Makefile.in @@ -178,6 +178,7 @@ install-include: clixon_restconf.h uninstall: rm -f $(DESTDIR)$(sbindir)/$(APPL) rm -f $(DESTDIR)$(libdir)/$(MYLIBLINK)* + rm -f $(DESTDIR)$(libdir)/$(MYLIB) .SUFFIXES: .SUFFIXES: .c .o diff --git a/example/main/Makefile.in b/example/main/Makefile.in index 7a611303..514b00db 100644 --- a/example/main/Makefile.in +++ b/example/main/Makefile.in @@ -145,7 +145,10 @@ $(RESTCONF_PLUGIN): $(RESTCONF_OBJ) ifeq ($(LINKAGE),dynamic) $(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -lclixon_restconf else - $(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -lclixon_restconf +# XXX need to add -L ../../apps/restconf otherwise get undefined: +# plugin_load_one file:/usr/local/lib/example/restconf/example_restconf.so function:clixon_plugin_init +# plugin_load_one: 338: Plugins: dlopen(/usr/local/lib/example/restconf/example_restconf.so): /usr/local/lib/libclixon_restconf.so.5: undefined symbol: api_return_err + $(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/restconf/ -lclixon_restconf endif SRC = $(BE_SRC) $(BE2_SRC) $(CLI_SRC) $(NETCONF_SRC) diff --git a/fuzz/cli/README.md b/fuzz/cli/README.md index c991f71c..151c8281 100644 --- a/fuzz/cli/README.md +++ b/fuzz/cli/README.md @@ -30,9 +30,9 @@ To link an example plugin properly it gets a little more complex:: Below is an example of how to do this for the main example. You can replace the main example plugins with another application: ``` - CC=/usr/bin/afl-clang-fast CFLAGS="-O2 -Wall -DCLIXON_STATIC_PLUGINS" LINKAGE=static ./configure - make clean + CC=/usr/bin/afl-clang-fast CFLAGS="-O2 -Wall -DCLIXON_STATIC_PLUGINS" LINKAGE=static INSTALLFLAGS="" ./configure + make clean make sudo make install diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index ecb17f17..eea467fd 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -267,6 +267,7 @@ endif uninstall: rm -f $(DESTDIR)$(libdir)/$(MYLIBLINK)* + rm -f $(DESTDIR)$(libdir)/$(MYLIB) TAGS: find . -name '*.[chyl]' -print | etags - diff --git a/test/cicd/coverage.sh b/test/cicd/coverage.sh index c8929d69..1afa9b9c 100755 --- a/test/cicd/coverage.sh +++ b/test/cicd/coverage.sh @@ -8,25 +8,36 @@ if [ $# -ne 1 ]; then echo "usage: $0 " exit -1 fi + TOKEN=$1 # LINKAGE=static # Configure (clixon) -LDFLAGS=-coverage CFLAGS="-O2 -Wall -coverage" ./configure --with-restconf=native +CFLAGS="-g -Wall" INSTALLFLAGS="" ./configure +sudo ldconfig +LDFLAGS=-coverage LINKAGE=static CFLAGS="-g -Wall -coverage" INSTALLFLAGS="" ./configure # Build -sh ./test/cicd/clixon-mk.sh +make clean +make -j10 +sudo make install +sudo make install-include +(cd example; make) +(cd util; make) +(cd example; sudo make install) +(cd util; sudo make install) -# Kludge to run restconf as root, and touch all gcda files, cant do as wwwuser -(cd test; clixon_restconf="/www-data/clixon_restconf -r" ./test_api.sh) +# Kludge for netconf to add as non-root +(cd test; ./test_netconf_hello.sh) find . -name "*.gcda" | xargs sudo chmod 777 -# Run all tests -(cd test; ./sum.sh) -#GITHUB_SHA= -# Push upstream -# The -f dont seem to work -bash <(curl -s https://codecov.io/bash) -t ${TOKEN} +(cd test; clixon_restconf="clixon_restconf -r" ./sum.sh) + +# Push coverage +# PUSH $TOKEN + +# remove all coverage files (after gcov push) +find . -name "*.gcda" | xargs rm sleep 1 # ensure OK is last echo OK diff --git a/test/test_cli_leafref.sh b/test/test_cli_leafref.sh index 5fd11eac..83416ade 100755 --- a/test/test_cli_leafref.sh +++ b/test/test_cli_leafref.sh @@ -229,8 +229,9 @@ expectpart "$($clixon_cli -1 -f $cfg -l o discard)" 0 "" new "expand identityref 2nd level" expectpart "$(echo "set identityrefs2 identityref ?" | $clixon_cli -f $cfg 2> /dev/null)" 0 "ex:des" "ex:des2" "ex:des3" +# Note CI may have random number as host which may match "92" new "expand leafref 2nd level" -expectpart "$(echo "set leafrefs2 leafref ?" | $clixon_cli -f $cfg 2> /dev/null)" 0 "91" "93" --not-- "92" +expectpart "$(echo "set leafrefs2 leafref ?" | $clixon_cli -f $cfg 2> /dev/null)" 0 " 91" " 93" --not-- " 92" new "set identityref2 des" expectpart "$($clixon_cli -1 -f $cfg set identityrefs2 identityref ex:des)" 0 "^$" diff --git a/test/test_install.sh b/test/test_install.sh index 00090f39..12c43679 100755 --- a/test/test_install.sh +++ b/test/test_install.sh @@ -7,7 +7,6 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi # Eg on FreeBSD use gmake : ${make:=make} - # Check for soft links for .so files in case of dynamic linkage, but .a files f static linking if [ ${LINKAGE} = static ]; then LIBOPT=-f @@ -18,10 +17,13 @@ fi new "Set up installdir $dir" new "Make DESTDIR install ($dir)" + +# Not for static linkage, libcligen.a may be taken from elsewhere (cd ..; $make DESTDIR=$dir install) if [ $? -ne 0 ]; then err fi + new "Check installed files /usr" if [ ! -d $dir/usr ]; then err $dir/usr diff --git a/yang/clixon/clixon-config@2021-07-11.yang b/yang/clixon/clixon-config@2021-07-11.yang index a5b71076..50bad2cc 100644 --- a/yang/clixon/clixon-config@2021-07-11.yang +++ b/yang/clixon/clixon-config@2021-07-11.yang @@ -46,13 +46,12 @@ module clixon-config { revision 2021-07-11 { description "Added option: - CLICON_SYSTEM_CAPABILITIES CLICON_RESTCONF_HTTP2_PLAIN Removed default value: CLICON_RESTCONF_INSTALLDIR Marked as obsolete: CLICON_YANG_LIST_CHECK - (Will be) Released in Clixon 5.3"; + Released in Clixon 5.3"; } revision 2021-05-20 { description @@ -454,18 +453,6 @@ module clixon-config { only loading from startup but may occur in other circumstances as well. This means that sanity checks of erroneous XML/JSON may not be properly signalled."; } - leaf CLICON_SYSTEM_CAPABILITIES { - type boolean; - default false; - description - "Enable module ietf-system-capabilities and ietf-notification-capabilities - Note: There are several dependencies: - - ietf-yang-library revision 2019-01-04 is REQUIRED - - nacm - - ietf-yang-structure-ext.yang, - - ietf-yang-instance-data - see draft-ietf-netconf-notification-capabilities-17"; - } leaf CLICON_BACKEND_DIR { type string; description