Restconf and http/1 modifications for http/1-only

apps/restconf/clixon_http1_parse.y

@@ -63,6 +63,7 @@
 
 %type <string> body
 %type <string> absolute_paths
+%type <string> absolute_paths1
 %type <string> absolute_path
 %type <string> field_vchars
 %type <string> field_values
@@ -230,7 +231,8 @@ body          : body BODY
               |	    	{ _PARSE_DEBUG("body -> "); $$ = NULL; } 
 ;
 
-/* request-line = method SP request-target SP HTTP-version CRLF */request_line  : method SP request_target SP HTTP_version CRLF
+/* request-line = method SP request-target SP HTTP-version CRLF */
+request_line  : method SP request_target SP HTTP_version CRLF
                { 
    		   _PARSE_DEBUG("request-line -> method request-target HTTP_version CRLF");
 	       }
@@ -255,14 +257,14 @@ method        : TOKEN
  * query = <query, see [RFC3986], Section 3.4>
  * query       = *( pchar / "/" / "?" )
  */
-request_target : absolute_paths
+request_target : absolute_paths1
                  {
 		      if (restconf_param_set(_HY->hy_h, "REQUEST_URI", $1) < 0)
 			  YYABORT;
 		      free($1);
-		     _PARSE_DEBUG("request-target -> absolute-paths");
+		     _PARSE_DEBUG("request-target -> absolute-paths1");
 		 }
-	        | absolute_paths QMARK QUERY
+	        | absolute_paths1 QMARK QUERY
 		  {
 		      if (restconf_param_set(_HY->hy_h, "REQUEST_URI", $1) < 0)
 			  YYABORT;
@@ -270,10 +272,19 @@ request_target : absolute_paths
 		      if (http1_parse_query(_HY, $3) < 0)
 			  YYABORT;
 		      free($3);
-		      _PARSE_DEBUG("request-target -> absolute-paths ? query");
+		      _PARSE_DEBUG("request-target -> absolute-paths1 ? query");
 		  }
 ;
 
+/* absolute-paths1 = absolute-paths ["/"] 
+ * Not according to standards: trailing /
+ */
+absolute_paths1 : absolute_paths
+                      { $$ = $1;_PARSE_DEBUG("absolute-paths1 -> absolute-paths "); }
+                | absolute_paths SLASH
+		      { $$ = $1;_PARSE_DEBUG("absolute-paths1 -> absolute-paths / "); }
+;
+
 /* absolute-path = 1*( "/" segment ) */
 absolute_paths : absolute_paths absolute_path
                  {

apps/restconf/restconf_http1.c

@@ -403,6 +403,8 @@ restconf_http1_path_root(clicon_handle  h,
     /* Check sanity of session, eg ssl client cert validation, may set rc_exit */
     if (restconf_connection_sanity(h, rc, sd) < 0)
 	goto done;
+    if (rc->rc_exit)
+	goto fail;
 #ifdef HAVE_LIBNGHTTP2
     if ((ret = http1_upgrade_http2(h, sd)) < 0)
 	goto done;

apps/restconf/restconf_native.c

@@ -596,6 +596,7 @@ restconf_http1(restconf_conn        *rc,
     clicon_handle         h;
     int                   ret;
     int                   status;
+    cbuf                 *cberr = NULL;
     
     h = rc->rc_h;
     if ((sd = restconf_stream_find(rc, 0)) == NULL){
@@ -643,13 +644,18 @@ restconf_http1(restconf_conn        *rc,
 		(*readmore)++;
 		goto ok;
 	    }
-	    /* Return error. Honsetly, the sender could just e slow, it should really be a 
+	    /* Return error. Honestly, the sender could just e slow, it should really be a 
 	     * timeout here.
 	     */
-	    if (native_send_badrequest(h, rc->rc_s, rc->rc_ssl, "application/yang-data+xml",
+	    if ((cberr = cbuf_new()) == NULL){
-				       "<errors xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf\"><error><error-type>protocol</error-type><error-tag>malformed-message</error-tag><error-message>The requested URL or a header is in some way badly formed</error-message></error></errors>") < 0)
+		clicon_err(OE_UNIX, errno, "cbuf_new");
 		goto done;
 	    }
+	    cprintf(cberr, "<errors xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf\"><error><error-type>protocol</error-type><error-tag>malformed-message</error-tag><error-message>%s</error-message></error></errors>", clicon_err_reason);
+	    if (native_send_badrequest(h, rc->rc_s, rc->rc_ssl, "application/yang-data+xml", cbuf_get(cberr)) < 0)
+		goto done;
+	    goto ok;
+	}
 	/* Check for Continue and if so reply with 100 Continue 
 	 * ret == 1: send reply
 	 */
@@ -677,6 +683,10 @@ restconf_http1(restconf_conn        *rc,
 	(*readmore)++;
 	goto ok;
     }
+    /* nginx compatible, set HTTPS parameter if SSL */
+    if (rc->rc_ssl)
+	if (restconf_param_set(h, "HTTPS", "https") < 0)
+	    goto done;
     /* main restconf processing */
     if (restconf_http1_path_root(h, rc) < 0)
 	goto done;
@@ -685,6 +695,7 @@ restconf_http1(restconf_conn        *rc,
 	goto done;
     cvec_reset(sd->sd_outp_hdrs); /* Can be done in native_send_reply */
     cbuf_reset(sd->sd_outp_buf);
+    cbuf_reset(sd->sd_inbuf);
     if (rc->rc_exit){  /* Server-initiated exit */
 	SSL_free(rc->rc_ssl);
 	rc->rc_ssl = NULL;
@@ -700,11 +711,14 @@ restconf_http1(restconf_conn        *rc,
  ok:
     retval = 1;
  done:
+    if (cberr)
+	cbuf_free(cberr);
     return retval;
 }
 #endif
 
 #ifdef HAVE_LIBNGHTTP2
+#ifdef HAVE_HTTP1
 static int
 restconf_http2_upgrade(restconf_conn *rc)
 {
@@ -763,6 +777,7 @@ restconf_http2_upgrade(restconf_conn *rc)
  done:
     return retval;    
 }
+#endif /* HAVE_LIBHTTP1 */
 
 /*!
  * @param[in]  buf      Input buffer

test/config.sh.in

@@ -44,7 +44,7 @@ WITH_RESTCONF=@with_restconf@ # native, fcgi or ""
 # If set, curl options are set to use --http2 which may not be what you want, ie
 # you may want to force it to http/1 for example
 # If so, override before test
-: ${HAVE_LIBNGHTTP2:=@HAVE_LIBNGHTTP2@}o
+: ${HAVE_LIBNGHTTP2:=@HAVE_LIBNGHTTP2@}
 HAVE_HTTP1=@HAVE_HTTP1@
 
 # This is for libxml2 XSD regex engine

test/lib.sh

@@ -449,6 +449,7 @@ function wait_restconf(){
 	    err1 "restconf timeout $DEMWAIT seconds"
 	fi
 	sleep $DEMSLEEP
+#	echo "curl $CURLOPTS -X GET $myproto://localhost/restconf"
 	hdr=$(curl $CURLOPTS -X GET $myproto://localhost/restconf 2> /dev/null)
 #	echo "hdr:\"$hdr\""
 	let i++;

test/test_perf_restconf.sh

@@ -2,14 +2,21 @@
 # Scaling/ performance tests for non-ssl RESTCONF
 # Lists (and leaf-lists)
 # Add, get and delete entries
+# If both HTTP/1 and /2, force to /1 to test native http/1 implementation
 
 # Override default to use http/1.1, comment to use https/2
-HAVE_LIBNGHTTP2=false
 RCPROTO=http
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
+# Pin to http/1
+if [ ${HAVE_LIBNGHTTP2} = true -a ${HAVE_HTTP1} = true ]; then
+    HAVE_LIBNGHTTP2=false
+    CURLOPTS="${CURLOPTS} --http1.1"
+    HVER=1.1
+fi
+
 # Which format to use as datastore format internally
 : ${format:=xml}
 
@@ -79,6 +86,7 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
   <CLICON_LOG_STRING_LIMIT>128</CLICON_LOG_STRING_LIMIT>
+  <CLICON_RESTCONF_HTTP2_PLAIN>true</CLICON_RESTCONF_HTTP2_PLAIN>
   $RESTCONFIG
 </clixon-config>
 EOF
@@ -136,36 +144,11 @@ if [ $r -ne 0 ]; then
     err1 "retval 0" $r
 fi
 
-# Remove Content-Length line (depends on size)
+# Only compare relevant data line
-# Note: do not use sed -i since it is not portable between gnu and bsd
-sed '/Content-Length:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/content-length:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-# Remove (nginx) web-server specific lines
-sed '/Server:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/Date:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/Transfer-Encoding:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/Connection:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-
-# Create a file to compare with
-if ${HAVE_LIBNGHTTP2}; then
-    if [ ${HAVE_HTTP1} -a ${RCPROTO} = http ]; then
-	# Add 101 switch protocols for http 1->2 upgrade
-	echo "HTTP/1.1 101 Switching Protocols
" > $ftest
-        echo "Upgrade: h2c
" >> $ftest
-	echo "
" >> $ftest
-	echo "HTTP/$HVER 200 
" >> $ftest
-    else
-	echo "HTTP/$HVER 200 
" > $ftest
-    fi
-else
-    echo "HTTP/$HVER 200 OK
" > $ftest
-fi
-echo "Content-Type: application/yang-data+xml
" >> $ftest
-echo "Cache-Control: no-cache
" >> $ftest
-echo "
">> $ftest
 echo -n "<data>">> $ftest
 cat $fdataxml >> $ftest
 echo "</data>
" >> $ftest
+sed -i '/<data>/!d' $foutput
 
 ret=$(diff -i $ftest $foutput)
 if [ $? -ne 0 ]; then

test/test_perf_restconf_ssl.sh

@@ -3,10 +3,6 @@
 # Lists (and leaf-lists)
 # Add, get and delete entries
 
-# Override default to use http/1.1, comment to use https/2
-HAVE_LIBNGHTTP2=false
-RCPROTO=http
-
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
@@ -106,6 +102,7 @@ cat <<EOF > $cfg
   <CLICON_CLISPEC_DIR>/usr/local/lib/example/clispec</CLICON_CLISPEC_DIR>
   <CLICON_CLI_LINESCROLLING>0</CLICON_CLI_LINESCROLLING>
   <CLICON_LOG_STRING_LIMIT>128</CLICON_LOG_STRING_LIMIT>
+  <CLICON_RESTCONF_HTTP2_PLAIN>true</CLICON_RESTCONF_HTTP2_PLAIN>
   $RESTCONFIG
 </clixon-config>
 EOF
@@ -163,36 +160,11 @@ if [ $r -ne 0 ]; then
     err1 "retval 0" $r
 fi
 
-# Remove Content-Length line (depends on size)
+# Only compare relevant data line
-# Note: do not use sed -i since it is not portable between gnu and bsd
-sed '/Content-Length:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/content-length:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-# Remove (nginx) web-server specific lines
-sed '/Server:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/Date:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/Transfer-Encoding:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-sed '/Connection:/d' $foutput > $foutput2 && mv $foutput2 $foutput
-
-# Create a file to compare with
-if ${HAVE_LIBNGHTTP2}; then
-    if [ ${HAVE_HTTP1} -a ${RCPROTO} = http ]; then
-	# Add 101 switch protocols for http 1->2 upgrade
-	echo "HTTP/1.1 101 Switching Protocols
" > $ftest
-        echo "Upgrade: h2c
" >> $ftest
-	echo "
" >> $ftest
-	echo "HTTP/$HVER 200 
" >> $ftest
-    else
-	echo "HTTP/$HVER 200 
" > $ftest
-    fi
-else
-    echo "HTTP/$HVER 200 OK
" > $ftest
-fi
-echo "Content-Type: application/yang-data+xml
" >> $ftest
-echo "Cache-Control: no-cache
" >> $ftest
-echo "
">> $ftest
 echo -n "<data>">> $ftest
 cat $fdataxml >> $ftest
 echo "</data>
" >> $ftest
+sed -i '/<data>/!d' $foutput
 
 ret=$(diff -i $ftest $foutput)
 if [ $? -ne 0 ]; then

test/test_restconf_continue.sh

@@ -1,16 +1,25 @@
 #!/usr/bin/env bash
 # Restconf HTTP/1.1 Expect/Continue functionality
 # Trigger Expect by curl -H. Some curls seem to trigger one on large PUTs but not all
-
+# If both HTTP/1 and /2, force to /1 to test native http/1 implementation
-# Override default to use http/1.1
-# In http/2 there is no explicit continue
-HAVE_LIBNGHTTP2=false
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
+if ! ${HAVE_HTTP1}; then
+    echo "...skipped: Must run with http/1"
+    if [ "$s" = $0 ]; then exit 0; else return 0; fi
+fi
+
 APPNAME=example
 
+if [ ${HAVE_LIBNGHTTP2} = true ]; then
+    # Pin to http/1
+    HAVE_LIBNGHTTP2=false
+    CURLOPTS="${CURLOPTS} --http1.1"
+    HVER=1.1
+fi
+
 cfg=$dir/conf.xml
 fyang=$dir/restconf.yang
 fjson=$dir/large.json

test/test_restconf_err.sh

@@ -21,19 +21,27 @@
 # XXX does not test rpc-error from backend in api_return_err?
 
 # Override default to use http/1.1
-# Force to HTTP 1.1 no SSL due to netcat
-RCPROTO=http
-HAVE_LIBNGHTTP2=false
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
 # Does not work with native http/2-only
 if [ "${WITH_RESTCONF}" = "native" -a ${HAVE_HTTP1} = false ]; then
+#if ! ${HAVE_HTTP1}; then
     echo "...skipped: must run with http/1"
     if [ "$s" = $0 ]; then exit 0; else return 0; fi
 fi
 
+# Pin to http/1
+if [ ${HAVE_LIBNGHTTP2} = true -a ${HAVE_HTTP1} = true ]; then
+    HAVE_LIBNGHTTP2=false
+    CURLOPTS="${CURLOPTS} --http1.1"
+    HVER=1.1
+fi
+
+# Force to HTTP 1.1 no SSL due to netcat
+RCPROTO=http
+
 APPNAME=example
 
 cfg=$dir/conf.xml

test/test_restconf_op.sh

@@ -205,6 +205,7 @@ new "restconf DELETE whole datastore"
 expectpart "$(curl $CURLOPTS -X DELETE $RCPROTO://localhost/restconf/data)" 0 "HTTP/$HVER 204"
 
 #--------------- Multiple request in single TCP tests
+new "Multiple requests: GET + POST using --next"
 expectpart "$(curl $CURLOPTS -H "Accept: application/yang-data+xml" -X GET $RCPROTO://localhost/restconf/data?content=config --next $CURLOPTS -H "Content-Type: application/yang-data+json" -X POST $RCPROTO://localhost/restconf/data -d '{"example:cont1":{"interface":{"name":"local0","type":"regular"}}}')" 0 "HTTP/$HVER 200" '<data/>' "HTTP/$HVER 201"
 
 #--------------- json type tests