* Unit test ifdef for restconf http1 parser

* test: fix cli-union test

apps/restconf/restconf_main_native.c

@@ -1073,8 +1073,21 @@ openssl_init_socket(clicon_handle h,
 
     /* ss is a server socket that the clients connect to. The callback
        therefore accepts clients on ss */
+#ifdef RESTCONF_HTTP1_UNITTEST
+    {
+	restconf_conn *rc;
+	if ((rc = restconf_conn_new(h, 0)) == NULL)
+	    goto done;
+	rc->rc_s = 0;
+	if (restconf_stream_data_new(rc, 0) == NULL)
+	    goto done;
+	if (clixon_event_reg_fd(0, restconf_connection, rc, "restconf socket") < 0) 
+	    goto done;
+    }
+#else
     if (clixon_event_reg_fd(ss, restconf_accept_client, rsock, "restconf socket") < 0) 
 	goto done;
+#endif
     retval = 0;
  done:
     return retval;
@@ -1404,11 +1417,6 @@ usage(clicon_handle h,
     exit(0);
 }
 
-/* Enable for normal use
- * Disable for unit testing, fuzzing, etc
- */
-#if 1
-
 int
 main(int    argc,
      char **argv)
@@ -1611,42 +1619,3 @@ main(int    argc,
     restconf_terminate(h);
     return retval;
 }
-
-#else /* Unit test */
-
-int
-main(int    argc,
-     char **argv)
-{
-    int            retval = -1;
-    clicon_handle  h = NULL;
-    restconf_conn *rc;
-    int            dbg = 0;
-    int            c;
-
-    while ((c = getopt(argc, argv, "hD:")) != -1)
-	switch (c) {
-	case 'h':
-	    fprintf(stderr, "%s [-h] [-D <dbg>]\n", argv[0]);
-	    break;
-	case 'D' : /* debug. Note this overrides any setting in the config */
-	    if (sscanf(optarg, "%d", &dbg) != 1)
-	    fprintf(stderr, "%s [-h] [-D <dbg>]\n", argv[0]);
-	    break;
-	}
-    if ((h = restconf_handle_init()) == NULL)
-	goto done;
-    clicon_log_init(__FILE__, LOG_DEBUG, CLICON_LOG_STDERR);
-    clicon_debug_init(dbg, NULL);
-    if ((rc = restconf_conn_new(h, 0)) == NULL)
-	goto done;
-    if (restconf_stream_data_new(rc, 0) == NULL)
-	goto done;
-    if (clixon_http1_parse_file(h, rc, stdin, "stdin") < 0)
-	goto done;
-    retval = 0;
- done:
-    return retval;
-}
-
-#endif

apps/restconf/restconf_native.c

@@ -471,8 +471,10 @@ native_clear_input(clicon_handle         h,
     int retval = -1;
 
     cbuf_reset(sd->sd_indata);
-    if (sd->sd_qvec)
+    if (sd->sd_qvec){
 	cvec_free(sd->sd_qvec);
+	sd->sd_qvec = NULL;
+    }
     if (restconf_param_del_all(h) < 0)
 	goto done;
     retval = 0;
@@ -679,10 +681,12 @@ restconf_http1(restconf_conn        *rc,
      */
     if ((ret = http1_check_content_length(h, sd, &status)) < 0)
 	goto done;
+#ifndef RESTCONF_HTTP1_UNITTEST /* Ignore content-length */
     if (status == 1){
 	(*readmore)++;
 	goto ok;
     }
+#endif
     /* nginx compatible, set HTTPS parameter if SSL */
     if (rc->rc_ssl)
 	if (restconf_param_set(h, "HTTPS", "https") < 0)
@@ -690,9 +694,15 @@ restconf_http1(restconf_conn        *rc,
     /* main restconf processing */
     if (restconf_http1_path_root(h, rc) < 0)
 	goto done;
+#ifdef RESTCONF_HTTP1_UNITTEST
+    if (native_buf_write(cbuf_get(sd->sd_outp_buf), cbuf_len(sd->sd_outp_buf),
+			 1, rc->rc_ssl) < 0)
+	goto done;
+#else
     if (native_buf_write(cbuf_get(sd->sd_outp_buf), cbuf_len(sd->sd_outp_buf),
 			 rc->rc_s, rc->rc_ssl) < 0)
 	goto done;
+#endif
     cvec_reset(sd->sd_outp_hdrs); /* Can be done in native_send_reply */
     cbuf_reset(sd->sd_outp_buf);
     cbuf_reset(sd->sd_inbuf);
@@ -910,5 +920,16 @@ restconf_connection(int   s,
     retval = 0;
  done:
     clicon_debug(1, "%s retval %d", __FUNCTION__, retval);
+#ifdef RESTCONF_HTTP1_UNITTEST /* unit test */
+    if (rc){
+	if (close(rc->rc_s) < 0){
+	    clicon_err(OE_UNIX, errno, "close");
+	    goto done;
+	}
+	clixon_event_unreg_fd(rc->rc_s, restconf_connection);
+	restconf_conn_free(rc);
+    }
+    exit(0);
+#endif
     return retval;
 } /* restconf_connection */

doc/DEVELOP.md

@@ -144,8 +144,18 @@ Note: remember to run autoheader sometimes (when?)
 And when you do note (https://github.com/clicon/cligen/issues/17) which states that cligen_custom.h should be in quote. 
 
 Get config.sub and config.guess:
+```
 $ wget -O config.guess 'https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD'
 $ wget -O config.sub 'https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD'
+```
+
+Or generate:
+```
+libtoolize --force
+aclocal
+autoheader
+automake --force-missing --add-missing
+```
 
 ## Debug
 How to debug

include/clixon_custom.h

@@ -144,3 +144,8 @@
  * The cache uses two yang stmt flag bits. One to say it is active, the second its value
  */
 #define USE_CONFIG_FLAG_CACHE
+
+/*! Restconf native unit test for fuzzing of http/1 parser
+ * See test/fuzz/http1
+ */
+#undef RESTCONF_HTTP1_UNITTEST

test/fuzz/http1/README.md

@@ -5,25 +5,21 @@ native restconf app and need some special compiling to run stand-alone.
 
 Install AFL, see [..](..)
 
-Edit `apps/restconf/restconf_main_native.c` by disabling the regular
+Enable `RESTCONF_HTTP1_UNITTEST` in `include/clixon_custom.h`.
-main function and replacing it with the unit testing `main`:
+
-```
---- a/apps/restconf/restconf_main_native.c
-+++ b/apps/restconf/restconf_main_native.c
-@@ -1403,7 +1403,7 @@ usage(clicon_handle h,
- /* Enable for normal use
-  * Disable for unit testing, fuzzing, etc
-  */
--#if 1
-+#if 0
-```
 
 Build and install clixon libraries and restconf statically
 ```
-  ./configure LINKAGE=static INSTALLFLAGS="" CC=/usr/bin/afl-clang-fast
+  ./configure --disable-nghttp2 LINKAGE=static INSTALLFLAGS="" CC=/usr/bin/afl-clang-fast CFLAGS="-g"
   make clean
   make
   sudo make install
   ./runfuzz.sh
 ```
 
+
+To view crashes
+```
+sudo chmod o+x output/crashes
+sudo chmod -R o+r output/crashes
+```

test/fuzz/http1/runfuzz.sh

@@ -8,8 +8,52 @@ if [ $# -ne 0 ]; then
     exit 255
 fi
 
+APPNAME=example
+cfg=$(pwd)/conf.xml
+
+cat <<EOF > $cfg
+<clixon-config xmlns="http://clicon.org/config">
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
+  <CLICON_FEATURE>clixon-restconf:allow-auth-none</CLICON_FEATURE>
+  <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>clixon-hello</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLI_MODE>hello</CLICON_CLI_MODE>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/hello/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_SOCK>/usr/local/var/hello.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/hello.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_XMLDB_DIR>/usr/local/var/hello</CLICON_XMLDB_DIR>
+  <CLICON_STARTUP_MODE>init</CLICON_STARTUP_MODE>
+  <CLICON_MODULE_LIBRARY_RFC7895>false</CLICON_MODULE_LIBRARY_RFC7895>
+  <CLICON_SOCK_GROUP>clicon</CLICON_SOCK_GROUP>
+  <CLICON_NETCONF_HELLO_OPTIONAL>true</CLICON_NETCONF_HELLO_OPTIONAL>
+  <CLICON_RESTCONF_USER>www-data</CLICON_RESTCONF_USER>
+  <CLICON_RESTCONF_PRIVILEGES>drop_perm</CLICON_RESTCONF_PRIVILEGES>
+  <restconf>
+      <enable>true</enable>
+      <auth-type>none</auth-type>
+      <pretty>false</pretty>
+      <debug>0</debug>
+      <log-destination>file</log-destination>
+      <socket>
+         <namespace>default</namespace>
+	 <address>0.0.0.0</address>
+	 <port>8088</port>
+	 <ssl>false</ssl>
+      </socket>
+   </restconf>
+</clixon-config>
+EOF
+
 MEGS=500 # memory limit for child process (50 MB)
 
+# Kill previous
+echo "cfg: $cfg"
+sudo clixon_backend -z -f $cfg -s init 
+
+# Start backend
+sudo clixon_backend -f $cfg -s init
+
 # remove input and input dirs
 #test ! -d input || rm -rf input
 test ! -d output || sudo rm -rf output
@@ -18,8 +62,21 @@ test ! -d output || sudo rm -rf output
 #test -d input || mkdir input
 test -d output || mkdir output
 
+if false; then
+    # Dryrun without afl (comment this if you run for real)
+    sudo /usr/local/sbin/clixon_restconf -rf $cfg < input/1.http || true
+    sudo /usr/local/sbin/clixon_restconf -rf $cfg < input/2.http || true
+    sudo /usr/local/sbin/clixon_restconf -rf $cfg < input/3.http || true
+    sudo /usr/local/sbin/clixon_restconf -rf $cfg < input/4.http || true
+    exit
+fi
+
 # Run script 
 #  CC=/usr/bin/afl-clang 
-sudo afl-fuzz -i input -o output -d -m $MEGS -- /usr/local/sbin/clixon_restconf
+sudo afl-fuzz -i input -o output -d -m $MEGS -- /usr/local/sbin/clixon_restconf -rf $cfg
+
+# To continue existing
+#sudo afl-fuzz -i - -o output -d -m $MEGS -- /usr/local/sbin/clixon_restconf -rf $cfg
+
 
 

test/test_cli_union.sh

@@ -112,9 +112,6 @@ wait_backend
 new "cli set leaf union"
 expectpart "$($clixon_cli -1 -f $cfg set tableleaf parleaf a value u1)" 0 "^$"
 
-new "cli set leaf union"
-expectpart "$($clixon_cli -1 -f $cfg set tableleaf parleaf a value u1)" 0 "^$"
-exit
 new "cli query leaf union - basic"
 expectpart "$(echo "set tableleaf parleaf a value ?" | $clixon_cli -f $cfg 2>/dev/null)" 0 u1 u2
 
@@ -128,6 +125,9 @@ fi
 new "cli set key union"
 expectpart "$($clixon_cli -1 -f $cfg set tablekey parkey u1 value 42)" 0 "^$"
 
+new "cli set key union again"
+expectpart "$($clixon_cli -1 -f $cfg set tablekey parkey u1 value 42)" 0 "^$"
+
 new "cli query key union - basic"
 expectpart "$(echo "set tablekey parkey ?" | $clixon_cli -f $cfg 2>/dev/null)" 0 u1 u2