- Modified linkage constant in makefile to test for dynamic only

- CLIXON_STATIC_PLUGIN to support statically linked plugins
- Added -H option to clixon_netconf: Do not require hello before request

CHANGELOG.md

@@ -45,6 +45,7 @@ Expected: September, 2021
     * ietf-restconf-list-pagination@2015-01-30.yang
     * clixon-netconf-list-pagination@2021-08-27.yang
     * ietf-yang-metadata@2016-08-05.yang
+  * New http media: application/yang-collection+xml/json
   * Updated state callback signature containing parameters for pagination
     * See API changes below
 * YANG Leafref feature update
@@ -63,8 +64,7 @@ Expected: September, 2021
   * Thanks to Alan Yaniger for providing this patch
 * List pagination
   * This is prototype work for ietf netconf work
-  * See draft-wwlh-netconf-list-pagination-00.txt
+
-  * New http media: application/yang-collection+xml/json
 
 * New state callback signature (ca_statedata2)
    * The new callback contains parameters for pagination

apps/backend/Makefile.in

@@ -47,7 +47,7 @@ endif
 SH_SUFFIX	= @SH_SUFFIX@
 INSTALLFLAGS  	= @INSTALLFLAGS@ 
 LDFLAGS 	= @LDFLAGS@
-ifeq ($(LINKAGE),static)
+ifneq ($(LINKAGE),dynamic)
 LDFLAGS 	+=  -rdynamic -L.
 endif
 prefix 		= @prefix@
@@ -65,10 +65,10 @@ CLIXON_MAJOR    = @CLIXON_VERSION_MAJOR@
 CLIXON_MINOR    = @CLIXON_VERSION_MINOR@
 
 # Use this clixon lib for linking
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	CLIXON_LIB	= libclixon.a
-else
 	CLIXON_LIB	= libclixon$(SH_SUFFIX).$(CLIXON_MAJOR).$(CLIXON_MINOR)
+else
+	CLIXON_LIB	= libclixon.a
 endif
 
 # For dependency. A little strange that we rely on it being built in the src dir
@@ -104,10 +104,10 @@ MYLIBSO         = lib$(MYNAME)$(SH_SUFFIX).$(CLIXON_MAJOR)
 MYLIBLINK       = lib$(MYNAME)$(SH_SUFFIX)
 MYLIBSTATIC     = lib$(MYNAME).a
 
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	MYLIB        = $(MYLIBSTATIC)
-else
 	MYLIB        = $(MYLIBDYNAMIC)
+else
+	MYLIB        = $(MYLIBSTATIC)
 endif
 
 all:	  $(MYLIB) $(APPL) test

apps/backend/backend_get.c

@@ -607,6 +607,7 @@ get_list_pagination(clicon_handle        h,
 					       offset, limit, &remaining, &xret)) < 0)
 	    goto done;
     }
+    /* Help function to filter out anything that is outside of xpath */
     if (filter_xpath_again(h, yspec, xret, xpath, nsc, &x1) < 0)
 	goto done;
 #ifdef LIST_PAGINATION_REMAINING

apps/cli/Makefile.in

@@ -46,7 +46,7 @@ endif
 SH_SUFFIX	= @SH_SUFFIX@
 INSTALLFLAGS  	= @INSTALLFLAGS@ 
 LDFLAGS 	= @LDFLAGS@	
-ifeq ($(LINKAGE),static)
+ifneq ($(LINKAGE),dynamic)
 LDFLAGS 	+=  -rdynamic -L.
 endif
 
@@ -66,10 +66,10 @@ CLIXON_MAJOR    = @CLIXON_VERSION_MAJOR@
 CLIXON_MINOR    = @CLIXON_VERSION_MINOR@
 
 # Use this clixon lib for linking
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	CLIXON_LIB	= libclixon.a
-else
 	CLIXON_LIB	= libclixon$(SH_SUFFIX).$(CLIXON_MAJOR).$(CLIXON_MINOR)
+else
+	CLIXON_LIB	= libclixon.a
 endif
 
 # For dependency. A little strange that we rely on it being built in the src dir
@@ -87,6 +87,10 @@ APPL		= clixon_cli
 APPSRC		= cli_main.c
 APPOBJ		= $(APPSRC:.c=.o)
 
+# HACK
+#EXTRAS         += $(top_srcdir)/example/main/example_cli.o
+#EXTRAS         += $(top_srcdir)/example/main/test_cli.o
+
 # Accessible from plugin
 LIBSRC	        = cli_common.c
 LIBSRC	       += cli_show.c
@@ -165,7 +169,7 @@ $(APPL): $(APPOBJ) $(MYLIB) $(LIBDEPS)
 ifeq ($(LINKAGE),dynamic)
 	$(CC) $(LDFLAGS) $(APPOBJ) -L. $(MYLIB) $(LIBS) -o $@
 else
-	$(CC) $(LDFLAGS) $(APPOBJ) -L. $(LIBOBJ) $(LIBS) -o $@
+	$(CC) $(LDFLAGS) $(APPOBJ) $(EXTRAS) -L. $(LIBOBJ) $(LIBS) -o $@
 endif
 
 $(MYLIBDYNAMIC) : $(LIBOBJ) $(LIBDEPS)

apps/cli/cli_main.c

@@ -412,7 +412,6 @@ main(int    argc,
     struct passwd *pw;
     char          *str;
     int            tabmode;
-    char          *dir;
     cvec          *nsctx_global = NULL; /* Global namespace context */
     size_t         cligen_buflen;
     size_t         cligen_bufthreshold;
@@ -623,10 +622,16 @@ main(int    argc,
     if (clixon_plugin_module_init(h) < 0)
 	goto done;
 
-    /* Load cli plugins before yangs are loaded (eg extension callbacks) */
+#ifndef CLIXON_STATIC_PLUGINS
+    {
+	char *dir;
+	/* Load cli .so plugins before yangs are loaded (eg extension callbacks) and 
+	 * before CLI is loaded by cli_syntax_load below */
 	if ((dir = clicon_cli_dir(h)) != NULL &&
 	    clixon_plugins_load(h, CLIXON_PLUGIN_INIT, dir, NULL) < 0)
 	    goto done;
+    }
+#endif
     
     /* Add (hardcoded) netconf features in case ietf-netconf loaded here
      * Otherwise it is loaded in netconf_module_load below
@@ -685,7 +690,8 @@ main(int    argc,
     if (autocli_start(h, printgen) < 0)
 	goto done;
 
-    /* Initialize cli syntax */
+    /* Initialize cli syntax. 
+     * Plugins have already been loaded by clixon_plugins_load above */
     if (cli_syntax_load(h) < 0)
 	goto done;
 

apps/cli/cli_plugin.c

@@ -249,10 +249,16 @@ clixon_str2fn(char  *name,
 }
 
 /*! Load a file containing syntax and append to specified modes, also load C plugin
+ *
+ * First load CLIgen file, 
+ * Then find which .so to load by looking in the "CLICON_PLUGIN" variable in that file.
+ * Make a lookup of plugins already loaded and resolve callbacks from cligen trees to
+ * dl symbols in the plugin.
  * @param[in]  h        Clixon handle
  * @param[in]  filename	Name of file where syntax is specified (in syntax-group dir)
  * @param[in]  dir	Name of dir, or NULL
  * @param[out] ptall    Universal CLIgen parse tree: apply to all modes
+ * @see clixon_plugins_load  Where .so plugin code has been loaded prior to this
  */
 static int
 cli_load_syntax_file(clicon_handle h,
@@ -271,7 +277,9 @@ cli_load_syntax_file(clicon_handle h,
     char         **vec = NULL;
     int            i, nvec;
     char          *plgnam;
+#ifndef CLIXON_STATIC_PLUGINS
     clixon_plugin_t *cp;
+#endif
 
     if ((pt = pt_new()) == NULL){
 	clicon_err(OE_UNIX, errno, "pt_new");
@@ -311,6 +319,7 @@ cli_load_syntax_file(clicon_handle h,
     prompt = cvec_find_str(cvv, "CLICON_PROMPT");
     plgnam = cvec_find_str(cvv, "CLICON_PLUGIN");
 
+#ifndef CLIXON_STATIC_PLUGINS
     if (plgnam != NULL) { /* Find plugin for callback resolving */
 	if ((cp = clixon_plugin_find(h, plgnam)) != NULL)
 	    handle = clixon_plugin_handle_get(cp);
@@ -321,6 +330,7 @@ cli_load_syntax_file(clicon_handle h,
 	    goto done;
 	}
     }
+#endif
     /* Resolve callback names to function pointers. */
     if (cligen_callbackv_str2fn(pt, (cgv_str2fn_t*)clixon_str2fn, handle) < 0){     
 	clicon_err(OE_PLUGIN, 0, "Mismatch between CLIgen file '%s' and CLI plugin file '%s'. Some possible errors:\n\t1. A function given in the CLIgen file does not exist in the plugin (ie link error)\n\t2. The CLIgen spec does not point to the correct plugin .so file (CLICON_PLUGIN=\"%s\" is wrong)", 
@@ -377,7 +387,10 @@ done:
     return retval;
 }
 
-/*! Load a syntax group. Includes both CLI plugin and CLIgen spec syntax files.
+/*! CLIgen spec syntax files and create CLIgen trees to drive the CLI syntax generator
+ *
+ * CLI .so plugins have been loaded: syntax table in place.
+ * Now load cligen syntax files and create cligen pt trees.
  * @param[in]     h       Clicon handle
  */
 int

apps/netconf/netconf_main.c

@@ -71,7 +71,7 @@
 #include "netconf_rpc.h"
 
 /* Command line options to be passed to getopt(3) */
-#define NETCONF_OPTS "hD:f:E:l:qa:u:d:p:y:U:t:eo:"
+#define NETCONF_OPTS "hD:f:E:l:qa:u:d:p:y:U:t:eHo:"
 
 #define NETCONF_LOGFILE "/tmp/clixon_netconf.log"
 
@@ -636,6 +636,7 @@ usage(clicon_handle h,
 	    "\t-U <user>\tOver-ride unix user with a pseudo user for NACM.\n"
 	    "\t-t <sec>\tTimeout in seconds. Quit after this time.\n"
 	    "\t-e \t\tDont ignore errors on packet input.\n"
+	    "\t-H \t\tHello messages are optional.\n"
 	    "\t-o \"<option>=<value>\"\tGive configuration option overriding config file (see clixon-config.yang)\n",
 	    argv0,
 	    clicon_netconf_dir(h)
@@ -764,6 +765,9 @@ main(int    argc,
 	case 'e': /* dont ignore packet errors */
 	    ignore_packet_errors = 0;
 	    break;
+	case 'H': /* Hello messages are optional */
+	    clicon_option_bool_set(h, "CLICON_NETCONF_HELLO_OPTIONAL", 1);
+	    break;
 	case 'o':{ /* Configuration option */
 	    char          *val;
 	    if ((val = index(optarg, '=')) == NULL)

example/main/Makefile.in

@@ -100,41 +100,47 @@ YANGSPECS	= clixon-example@2020-12-01.yang
 BE_SRC   = $(APPNAME)_backend.c
 BE_OBJ   = $(BE_SRC:%.c=%.o)
 $(BE_PLUGIN): $(BE_OBJ)
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $< -lclixon -lclixon_backend
+else
 	# can include -L in LDFLAGS?
 	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $< -lclixon -L ../../apps/backend/ -lclixon_backend 
-else
-	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $< -lclixon -lclixon_backend
 endif
 
 # Secondary NACM backend plugin
 BE2_SRC  = $(APPNAME)_backend_nacm.c
 BE2_OBJ  = $(BE2_SRC:%.c=%.o)
 $(BE2_PLUGIN): $(BE2_OBJ)
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $< -lclixon -L ../../apps/backend/ -lclixon_backend 
-else
 	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $< -lclixon -lclixon_backend
+else
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $< -lclixon -L ../../apps/backend/ -lclixon_backend 
 endif
 
 # CLI frontend plugin
 CLI_SRC  = $(APPNAME)_cli.c
 CLI_OBJ  = $(CLI_SRC:%.c=%.o)
 $(CLI_PLUGIN): $(CLI_OBJ)
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/cli/ -lclixon_cli
-else
 	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -lclixon_cli
+else
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/cli/ -lclixon_cli
+endif
+
+ifeq ($(LINKAGE),dynamic)
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -lclixon_cli
+else
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/cli/ -lclixon_cli
 endif
 
 # NETCONF frontend plugin
 NETCONF_SRC  = $(APPNAME)_netconf.c
 NETCONF_OBJ  = $(NETCONF_SRC:%.c=%.o)
 $(NETCONF_PLUGIN): $(NETCONF_OBJ)
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/netconf/ -lclixon_netconf
-else
 	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -lclixon_netconf
+else
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/netconf/ -lclixon_netconf
 endif
 
 # See configure.ac
@@ -142,10 +148,10 @@ endif
 RESTCONF_SRC  = $(APPNAME)_restconf.c
 RESTCONF_OBJ  = $(RESTCONF_SRC:%.c=%.o)
 $(RESTCONF_PLUGIN): $(RESTCONF_OBJ)
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/restconf/ -lclixon_restconf 
-else
 	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -lclixon_restconf
+else
+	$(CC) -Wall -shared $(LDFLAGS) -o $@ -lc $^ -lclixon -L ../../apps/restconf/ -lclixon_restconf 
 endif
 
 SRC     = $(BE_SRC) $(BE2_SRC) $(CLI_SRC) $(NETCONF_SRC)

example/main/example_backend.c

@@ -515,6 +515,8 @@ example_statefile(clicon_handle   h,
 	xt = _state_xml_cache;
     if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0) 
 	goto done;	
+    lower = 0;
+    upper = xlen;
     switch (pagmode){
     case PAGINATION_NONE:
 	lower = 0;

example/main/example_cli.c

@@ -132,6 +132,7 @@ example_client_rpc(clicon_handle h,
     return retval;
 }
 
+#ifndef CLIXON_STATIC_PLUGINS
 static clixon_plugin_api api = {
     "example",          /* name */
     clixon_plugin_init, /* init */
@@ -157,6 +158,7 @@ clixon_plugin_init(clicon_handle h)
 
     return &api;
 }
+#endif /* CLIXON_STATIC_PLUGINS */
 
 /*! Translate function from an original value to a new.
  * In this case, assume string and increment characters, eg HAL->IBM

fuzz/cli/README.md

@@ -13,6 +13,7 @@ Build and install a clixon system (in particular the backend, the CLI will be re
 ## Build
 
 Build clixon cli statically with the afl-clang compiler:
+
 ```
   CC=/usr/bin/afl-clang-fast LINKAGE=static ./configure # Dont care about restconf
   make clean
@@ -21,6 +22,30 @@ Build clixon cli statically with the afl-clang compiler:
   sudo make install
 ```
 
+To link an example plugin properly it gets a little more complex::
+
+- First, you need to identify which example plugins you want to link. Add these to `EXTRAS` variable
+- Configure and compile those plugins, where the `clixon_plugin_init()` function is removed.
+- Configure and compile the cli WITH the `EXTRAS` variable set.
+
+Below is an example of how to do this for the main example. You can replace the main example plugins with another application:
+```
+  CC=/usr/bin/afl-clang-fast  CFLAGS="-O2 -Wall -DCLIXON_STATIC_PLUGINS" LINKAGE=static ./configure
+  make clean
+  cd example/main # Compile and install application plugins (here main example)
+  make clean
+  make
+  sudo make install 
+  cd ../..
+  cd lib # Compile and install clixon lib
+  make
+  sudo make install
+  cd ..
+  cd apps/cli # Compile and install clixon_cli with pre-compiled plugins
+  EXTRAS="../../example/main/example_cli.o ../../example/main/test_cli.o" make clixon_cli
+  sudo make install
+```
+
 ## Run tests
 
 Run the script `runfuzz.sh` to run one test with a cli spec and an input string, eg:
@@ -29,4 +54,3 @@ Run the script `runfuzz.sh` to run one test with a cli spec and an input string,
 ```
 
 After (or during) the test, investigate results in the output dir.
-

fuzz/cli/input/1.cli

@@ -1 +1,3 @@
 set table parameter a value 42
+table
+example 42

lib/src/Makefile.in

@@ -107,10 +107,10 @@ MYLIBSO      = lib$(MYNAME)$(SH_SUFFIX).$(CLIXON_MAJOR)
 MYLIBLINK    = lib$(MYNAME)$(SH_SUFFIX)
 MYLIBSTATIC  = lib$(MYNAME).a
 
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-	MYLIB        = $(MYLIBSTATIC)
-else
 	MYLIB        = $(MYLIBDYNAMIC)
+else
+	MYLIB        = $(MYLIBSTATIC)
 endif
 
 all:	 $(MYLIB) $(MYLIBLINK)

util/Makefile.in

@@ -67,12 +67,12 @@ LINKAGE         = @LINKAGE@
 
 INCLUDES        = -I. @INCLUDES@ -I$(top_srcdir)/lib -I$(top_srcdir)/include
 
-ifeq ($(LINKAGE),static)
+ifeq ($(LINKAGE),dynamic)
-   CLIXON_LIB 		= libclixon.a
-   CLIXON_BACKEND_LIB	= libclixon_backend.a # for util_validate
-else
    CLIXON_LIB           = libclixon$(SH_SUFFIX).$(CLIXON_MAJOR).$(CLIXON_MINOR)
    CLIXON_BACKEND_LIB	= libclixon_backend$(SH_SUFFIX).$(CLIXON_MAJOR).$(CLIXON_MINOR)
+else
+   CLIXON_LIB 		= libclixon.a
+   CLIXON_BACKEND_LIB	= libclixon_backend.a # for util_validate
 endif
 
 # For dependency. A little strange that we rely on it being built in the src dir
@@ -146,13 +146,9 @@ clixon_util_regexp: clixon_util_regexp.c $(LIBDEPS)
 clixon_util_socket: clixon_util_socket.c $(LIBDEPS)
 	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@
 
-ifeq ($(LINKAGE),static)
 clixon_util_validate: clixon_util_validate.c $(LIBDEPS) 
-	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) clixon_util_validate.c $(LIBS) $(LIBDEPS) -o $@
+	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ -l clixon_backend -o $@ $(LIBS)
-else
+
-clixon_util_validate: clixon_util_validate.c $(LIBDEPS) 
-	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -l clixon_backend -o $@
-endif
 ifdef with_restconf
 clixon_util_stream: clixon_util_stream.c $(LIBDEPS)
 	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -lcurl -o $@

yang/clixon/clixon-config@2021-07-11.yang

@@ -615,7 +615,7 @@ module clixon-config {
 	leaf CLICON_CLISPEC_FILE {
 	    type string;
 	    description
-		"Specific frontend cligen spec file as aletrnative or complement
+		"Specific frontend cligen spec file as alternative or complement
                  to CLICON_CLISPEC_DIR. Also available as -c in clixon_cli.";
 	}
 	leaf CLICON_CLI_MODE {