Added http support for evhtp (not only https)

apps/restconf/restconf_api_evhtp.c

@@ -131,23 +131,26 @@ restconf_reply_send(void  *req0,
 {
     evhtp_request_t    *req = (evhtp_request_t *)req0;
     int                 retval = -1;
-    evhtp_connection_t *conn;
     struct evbuffer    *eb = NULL;
-    const char *reason_phrase;
+    const char         *reason_phrase;
     
     req->status = code;
     if ((reason_phrase = restconf_code2reason(code)) == NULL)
 	reason_phrase="";
-#if 1 /* XXX  remove status header för evhtp? */
+#if 0 /* XXX  remove status header för evhtp? */
     if (restconf_reply_header(req, "Status", "%d %s", code, reason_phrase) < 0)
 	goto done;
 #endif
-#if 1    /* Optional? */
+#if 0    /* Optional? */
-    if ((conn = evhtp_request_get_connection(req)) == NULL){
+    {
-	clicon_err(OE_DAEMON, EFAULT, "evhtp_request_get_connection");
+	evhtp_connection_t *conn;
-	goto done;
+
+	if ((conn = evhtp_request_get_connection(req)) == NULL){
+	    clicon_err(OE_DAEMON, EFAULT, "evhtp_request_get_connection");
+	    goto done;
+	}
+	htp_sslutil_add_xheaders(req->headers_out, conn->ssl, HTP_SSLUTILS_XHDR_ALL);
     }
-    htp_sslutil_add_xheaders(req->headers_out, conn->ssl, HTP_SSLUTILS_XHDR_ALL);
 #endif
 
     /* If body, add a content-length header */
@@ -190,10 +193,20 @@ restconf_get_indata(void *req0)
 {
     evhtp_request_t *req = (evhtp_request_t *)req0;    
     cbuf            *cb = NULL;
-    
+    size_t           len;
+    unsigned char   *buf;
+
     if ((cb = cbuf_new()) == NULL)
 	return NULL;
-    if (evbuffer_get_length(req->buffer_in))
+    len = evbuffer_get_length(req->buffer_in);
-	cprintf(cb, "%s", evbuffer_pullup(req->buffer_in, -1));
+    if (len > 0){
+	if ((buf = evbuffer_pullup(req->buffer_in, len)) == NULL){
+	    clicon_err(OE_CFG, errno, "evbuffer_pullup");
+	    return NULL;
+	}
+	/* Note the pullup may not be null-terminated */
+	cbuf_append_buf(cb, buf, len);
+    }
+
     return cb;
 }

apps/restconf/restconf_main_evhtp.c

@@ -78,10 +78,11 @@
 #include "restconf_lib.h"       /* generic shared with plugins */
 #include "restconf_handle.h"
 #include "restconf_api.h"       /* generic not shared with plugins */
+#include "restconf_err.h"
 #include "restconf_root.h"
 
 /* Command line options to be passed to getopt(3) */
-#define RESTCONF_OPTS "hD:f:l:p:d:y:a:u:o:P:c:k:"
+#define RESTCONF_OPTS "hD:f:l:p:d:y:a:u:o:P:s"
 
 /* Need global variable to for signal handler XXX */
 static clicon_handle _CLICON_HANDLE = NULL;
@@ -248,7 +249,8 @@ convert_fcgi(evhtp_header_t *hdr,
  * @param[in]  h    Clicon handle
  * @param[in]  req  Evhtp request struct
  * @param[out] qvec Query parameters, ie the ?<id>=<val>&<id>=<val> stuff
- * @retval     0    OK
+ * @retval     1    OK continue
+ * @retval     0    Fail, dont continue
  * @retval    -1    Error
  * The following parameters are set:
  * QUERY_STRING
@@ -293,16 +295,30 @@ evhtp_params_set(clicon_handle    h,
 	goto done;
     if (restconf_param_set(h, "REQUEST_URI", path->full) < 0)
 	goto done;
-    if (restconf_param_set(h, "HTTPS", "https") < 0) /* some string or NULL */
+    clicon_debug(1, "%s proto:%d", __FUNCTION__, req->proto);
-	goto done;
+    if (req->proto != EVHTP_PROTO_10 &&
+	req->proto != EVHTP_PROTO_11){
+	if (restconf_badrequest(h, req)	< 0)
+	    goto done;
+	goto fail;
+    }
+    clicon_debug(1, "%s conn->ssl:%d", __FUNCTION__, req->conn->ssl?1:0);
+    if (req->conn->ssl != NULL){
+	if (restconf_param_set(h, "HTTPS", "https") < 0) /* some string or NULL */
+	    goto done;
+    }
+
     /* Translate all http headers by capitalizing, prepend w HTTP_ and - -> _
      * Example: Host -> HTTP_HOST 
      */
     if (evhtp_headers_for_each(req->headers_in, convert_fcgi, h) < 0)
 	goto done;
-    retval = 0;
+    retval = 1;
  done:
     return retval;
+ fail:
+    retval = 0;
+    goto done;
 }
 
 static int
@@ -368,7 +384,8 @@ static void
 cx_path_wellknown(evhtp_request_t *req,
 		  void            *arg)
 {
-    clicon_handle       h = arg;
+    clicon_handle h = arg;
+    int           ret;
 
     clicon_debug(1, "------------");
     /* input debug */
@@ -377,12 +394,13 @@ cx_path_wellknown(evhtp_request_t *req,
     /* get accepted connection */
 
     /* set fcgi-like paramaters (ignore query vector) */
-    if (evhtp_params_set(h, req, NULL) < 0)
+    if ((ret = evhtp_params_set(h, req, NULL)) < 0)
 	goto done;
-    /* call generic function */
+    if (ret == 1){
-    if (api_well_known(h, req) < 0)
+	/* call generic function */
-	goto done;
+	if (api_well_known(h, req) < 0)
-
+	    goto done;
+    }
     /* Clear (fcgi) paramaters from this request */
     if (restconf_param_del_all(h) < 0)
 	goto done;
@@ -398,6 +416,7 @@ cx_path_restconf(evhtp_request_t *req,
 		 void            *arg)
 {
     clicon_handle h = arg;
+    int           ret;
     cvec         *qvec = NULL;
 
     clicon_debug(1, "------------");
@@ -411,11 +430,13 @@ cx_path_restconf(evhtp_request_t *req,
 	goto done;
     }
     /* set fcgi-like paramaters (ignore query vector) */
-    if (evhtp_params_set(h, req, qvec) < 0)
+    if ((ret = evhtp_params_set(h, req, qvec)) < 0)
-	goto done;
-    /* call generic function */
-    if (api_root_restconf(h, req, qvec) < 0)
 	goto done;
+    if (ret == 1){
+	/* call generic function */
+	if (api_root_restconf(h, req, qvec) < 0)
+	    goto done;
+    }
 
     /* Clear (fcgi) paramaters from this request */
     if (restconf_param_del_all(h) < 0)
@@ -424,6 +445,52 @@ cx_path_restconf(evhtp_request_t *req,
     return; /* void */
 }
 
+/*! Get Server cert info
+ * @param[out] ssl_config
+ */
+static int
+get_servercerts(evhtp_ssl_cfg_t *ssl_config,
+		const char      *pki_dir,
+		const char      *ssl_server_cert)
+{
+    int         retval = -1;
+    cbuf       *cb = NULL;
+    struct stat f_stat;
+
+    if (ssl_config == NULL){
+	clicon_err(OE_CFG, EINVAL, "ssl_config is NULL");
+	goto done;
+    }
+    if ((cb = cbuf_new()) == NULL){
+	clicon_err(OE_UNIX, errno, "cbuf_new");
+	goto done;
+    }
+    cprintf(cb, "%s/%s-crt.pem", pki_dir, ssl_server_cert);
+    if ((ssl_config->pemfile = strdup(cbuf_get(cb))) == NULL){
+	clicon_err(OE_UNIX, errno, "strdup");
+	goto done;
+    }
+    if (stat(ssl_config->pemfile, &f_stat) != 0) {
+	clicon_err(OE_FATAL, errno, "Cannot load SSL cert '%s'", ssl_config->pemfile);
+	goto done;
+    }
+    cbuf_reset(cb);
+    cprintf(cb, "%s/%s-key.pem", pki_dir, ssl_server_cert);
+    if ((ssl_config->privfile = strdup(cbuf_get(cb))) == NULL){
+	clicon_err(OE_UNIX, errno, "strdup");
+	goto done;
+    }
+    if (stat(ssl_config->privfile, &f_stat) != 0) {
+	clicon_err(OE_FATAL, errno, "Cannot load SSL key '%s'", ssl_config->privfile);
+	goto done;
+    }
+    retval = 0;
+ done:
+    if (cb)
+	cbuf_free(cb);
+    return retval;
+}
+
 /*! Usage help routine
  * @param[in]  argv0  command line
  * @param[in]  h      Clicon handle
@@ -445,9 +512,8 @@ usage(clicon_handle h,
     	    "\t-a UNIX|IPv4|IPv6 Internal backend socket family\n"
     	    "\t-u <path|addr>\t  Internal socket domain path or IP addr (see -a)\n"
 	    "\t-o \"<option>=<value>\" Give configuration option overriding config file (see clixon-config.yang)\n"
-    	    "\t-P <port>\t  HTTPS port (default 443)\n"
+	    "\t-s\t  SSL server, https\n"
-	    "\t-c <cert>\t  SSL server certificate - pemfile (mandatory)\n"
+    	    "\t-P <port>\t  HTTP port (default 80, or 443 if -s is given)\n"
-	    "\t-k <key>\t  SSL private key - privfile (mandatory)\n"
 	    ,
 	    argv0,
 	    clicon_restconf_dir(h)
@@ -473,15 +539,17 @@ main(int    argc,
     cvec              *nsctx_global = NULL; /* Global namespace context */
     size_t             cligen_buflen;
     size_t             cligen_bufthreshold;
-    uint16_t           port = 443;
+    uint16_t           defaultport = 80;
+    uint16_t           port = 0;
 #ifdef _EVHTP_NYI 
     char              *stream_path;
 #endif
     evhtp_t           *htp = NULL;
     struct event_base *evbase = NULL;
     evhtp_ssl_cfg_t   *ssl_config = NULL;
-    struct stat        f_stat;
     int                dbg = 0;
+    char              *ssl_server = NULL; /* SSL server name, default "server", base for srv certs */
+    char              *pki_dir = CLIXON_PKI_DIR;
 
     /* In the startup, logs to stderr & debug flag set later */
     clicon_log_init(__PROGRAM__, LOG_INFO, logdst); 
@@ -542,13 +610,6 @@ main(int    argc,
 #ifdef _EVHTP_NYI 
     stream_path = clicon_option_str(h, "CLICON_STREAM_PATH");
 #endif
-    /* Init evhtp ssl config struct */
-    if ((ssl_config = malloc(sizeof(evhtp_ssl_cfg_t))) == NULL){
-	clicon_err(OE_UNIX, errno, "malloc");
-	goto done;
-    }
-    memset(ssl_config, 0, sizeof(evhtp_ssl_cfg_t));
-    ssl_config->ssl_opts = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1;
     
     /* Now rest of options, some overwrite option file */
     optind = 1;
@@ -589,39 +650,38 @@ main(int    argc,
 		goto done;
 	    break;
 	}
+	case 's': /* ssl: use https */
+	    ssl_server = CLIXON_SERVER_CERT;
+	    defaultport = 443; /* unless explicit -P ? */
+	    break;
 	case 'P': /* http port */
 	    if (!strlen(optarg))
 		usage(h, argv0);
 	    port=atoi(optarg);
 	    break;
-	case 'c': /* SSL Server Certificate */
-	    ssl_config->pemfile = optarg;
-	    break;
-	case 'k': /* SSL private key */
-	    ssl_config->privfile = optarg;
-	    break;
         default:
             usage(h, argv0);
             break;
 	}
     argc -= optind;
     argv += optind;
-    /* Check ssl mandatory options */
+    /* port = defaultport unless explicitly set -P */
-    if (ssl_config->pemfile == NULL || ssl_config->privfile == NULL)
+    if (port == 0)
-	usage(h, argv0);
+	port = defaultport;
-    /* Verify SSL files */
+    /* Check server ssl certs */
-    if (ssl_config->pemfile == NULL)
+    if (ssl_server){
-	usage(h, argv0);
+	/* Init evhtp ssl config struct */
-    if (stat(ssl_config->pemfile, &f_stat) != 0) {
+	if ((ssl_config = malloc(sizeof(evhtp_ssl_cfg_t))) == NULL){
-	clicon_err(OE_FATAL, errno, "Cannot load SSL cert '%s'", ssl_config->pemfile);
+	    clicon_err(OE_UNIX, errno, "malloc");
-	goto done;
+	    goto done;
-    }
+	}
-    if (ssl_config->privfile == NULL) 
+	memset(ssl_config, 0, sizeof(evhtp_ssl_cfg_t));
-	usage(h, argv0);
+	ssl_config->ssl_opts = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1;
-    if (stat(ssl_config->privfile, &f_stat) != 0) {
+
-	clicon_err(OE_FATAL, errno, "Cannot load SSL key '%s'", ssl_config->privfile);
+	if (get_servercerts(ssl_config, pki_dir, ssl_server) < 0)
-	goto done;
+	    goto done;
     }
+
     //    ssl_verify_mode             = htp_sslutil_verify2opts(optarg);
     assert(SSL_VERIFY_NONE == 0);
     /* Access the remaining argv/argc options (after --) w clicon-argv_get() */
@@ -637,9 +697,11 @@ main(int    argc,
 	clicon_err(OE_UNIX, errno, "evhtp_new");
 	goto done;
     }
-    if (evhtp_ssl_init(htp, ssl_config) < 0){
+    if (ssl_server){
-	clicon_err(OE_UNIX, errno, "evhtp_new");
+	if (evhtp_ssl_init(htp, ssl_config) < 0){
-	goto done;
+	    clicon_err(OE_UNIX, errno, "evhtp_new");
+	    goto done;
+	}
     }
 #ifndef EVHTP_DISABLE_EVTHR
     evhtp_use_threads_wexit(htp, NULL, NULL, 4, NULL);

example/main/example_backend.c

@@ -657,11 +657,12 @@ upgrade_2014_to_2016(clicon_handle h,
     char      *name;
 
     clicon_debug(1, "%s from:%d to:%d", __FUNCTION__, from, to);
+    if (op != XML_FLAG_CHANGE) /* Only treat fully present modules */
+    	goto ok;
     /* Get Yang module for this namespace. Note it may not exist (if obsolete) */
     yspec = clicon_dbspec_yang(h);	
     if ((ym = yang_find_module_by_namespace(yspec, ns)) == NULL)
 	goto ok; /* shouldnt happen */
-    clicon_debug(1, "%s module %s", __FUNCTION__, ym?yang_argument_get(ym):"none");
     /* Get all XML nodes with that namespace */
     if (xml_namespace_vec(h, xt, ns, &vec, &vlen) < 0)
 	goto done;
@@ -758,6 +759,8 @@ upgrade_2016_to_2018(clicon_handle h,
     int        i;
 
     clicon_debug(1, "%s from:%d to:%d", __FUNCTION__, from, to);
+    if (op != XML_FLAG_CHANGE) /* Only treat fully present modules */
+    	goto ok;
     /* Get Yang module for this namespace. Note it may not exist (if obsolete) */
     yspec = clicon_dbspec_yang(h);	
     if ((ym = yang_find_module_by_namespace(yspec, ns)) == NULL)

include/clixon_custom.h

@@ -101,3 +101,7 @@
  */
 #define STATE_ORDERED_BY_SYSTEM
 
+/*! Dir for Public Key Infrastructure (PKI) X.509 certificates
+ */
+#define CLIXON_SERVER_CERT "server"
+#define CLIXON_PKI_DIR "/etc/pki/clixon"

lib/src/clixon_datastore_read.c

@@ -293,7 +293,7 @@ text_read_modstate(clicon_handle       h,
 		continue;
 	    if ((srev = xml_find_body(xs, "revision")) == NULL)
 		continue;
-	    if (strcmp(frev, srev)!=0){
+	    if (strcmp(frev, srev) != 0){
 		/* 3c) File module-state does not match system */
 		if ((xf2 = xml_dup(xf)) == NULL)
 		    goto done;

lib/src/clixon_yang.c

@@ -224,6 +224,15 @@ yang_argument_get(yang_stmt *ys)
     return ys->ys_argument;
 }
 
+/*
+ * Note on cvec on XML nodes:
+ * 1. It is always created in xml_new. It could be lazily created on use to save a little memory
+ * 2. Only some yang statements use the cvec, as follows:
+ *  2a. ranges and lengths: [min, max]
+ *  2b. list: keys
+ *  2c. identity types: derived instances: identityrefs, save <module>:<idref>
+ *  2d. type: leafref types: derived instances.
+ */
 /*! Set yang argument, not not copied
  * @param[in] ys   Yang statement node
  * @param[in] arg  Argument
@@ -335,6 +344,7 @@ yang_stmt *
 ys_new(enum rfc_6020 keyw)
 {
     yang_stmt *ys;
+    cvec      *cvv;
 
     if ((ys = malloc(sizeof(*ys))) == NULL){
 	clicon_err(OE_YANG, errno, "malloc");
@@ -344,10 +354,11 @@ ys_new(enum rfc_6020 keyw)
     ys->ys_keyword    = keyw;
     /* The cvec contains stmt-specific variables. Only few stmts need variables so the
        cvec could be lazily created to save some heap and cycles. */
-    if ((ys->ys_cvec = cvec_new(0)) == NULL){ 
+    if ((cvv = cvec_new(0)) == NULL){ 
 	clicon_err(OE_YANG, errno, "cvec_new");
 	return NULL;
     }
+    yang_cvec_set(ys, cvv);
     return ys;
 }
 
@@ -1712,7 +1723,9 @@ ys_populate_identity(clicon_handle h,
 	    goto done;
 	}
 	//	    continue; /* root identity */
-	/* Check if derived id is already in base identifier */
+	/* Check if derived id is already in base identifier 
+	 * note that cvec is always created in ys_new()
+	 */
 	idrefvec = yang_cvec_get(ybaseid);
 	if (cvec_find(idrefvec, idref) != NULL)
 	    continue;

test/README.md

@@ -93,6 +93,7 @@ You may add your site-specific modifications in a `site.sh` file. Example:
 
 For example, in FreeBSD, add:
 ```
-wwwuser=www
+  wwwuser=www
-make=gmake
+  make=gmake
 ```
+

test/lib.sh

@@ -90,9 +90,6 @@ fi
 # RESTCONF protocol, eg http or https
 : ${RCPROTO:=http}
 
-# RESTCONF error message (if not up)
-: ${RCERROR:="HTTP/1.1 502 Bad Gateway"}
-
 # www user (on linux typically www-data, freebsd www)
 # @see wwwstartuser which can be dropped to this
 : ${wwwuser:=www-data}
@@ -234,8 +231,13 @@ wait_backend(){
 # @see wait_restconf
 start_restconf(){
     # Start in background 
-    echo "sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $*"
+    if [ $RCPROTO = https ]; then
-    sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $* &
+	EXTRA="-s"
+    else
+	EXTRA=
+    fi
+    echo "sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $EXTRA $*"
+    sudo -u $wwwstartuser -s $clixon_restconf $RCLOG -D $DBG $EXTRA $* &
     if [ $? -ne 0 ]; then
 	err
     fi

test/test_upgrade_failsafe.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
-# Starting clixon with outdated (or not) modules
+# Starting clixon with outdated (or not) modules.
+# It is a test of handling modstate, identifying invalid startups
+# and entering failsafe
+# No active upgrading of an outdated db is made
 # This relies on storing RFC7895 YANG Module Library modules-state info
 # in the datastore (or XML files?)
 # The test is made with three Yang models A, B and C as follows:
@@ -32,7 +35,6 @@ fyangB=$dir/B@2019-01-01.yang
 # Yang module A revision "0814-01-28"
 # Note that this Yang model will exist in the DIR but will not be loaded
 # by the system. Just here for reference
-# XXX: Maybe it should be loaded and used in draft-wu?
 cat <<EOF > $fyangA0
 module A{
   prefix a;
@@ -268,7 +270,7 @@ runtest(){
 	wait_backend
     else
 	new "Restart backend as eg follows: -Ff $cfg -s $mode -o \"CLICON_XMLDB_MODSTATE=$modstate\" ($BETIMEOUT s)"
-	sleep $BETIMEOUT
+#	sleep $BETIMEOUT
     fi
 
     new "Check running db content"
@@ -279,7 +281,7 @@ runtest(){
 	new "Check startup db content"
 	expecteof "$clixon_netconf -qf $cfg" 0 "<rpc><get-config><source><startup/></source></get-config></rpc>]]>]]>" "^<rpc-reply>$expstart</rpc-reply>]]>]]>$"
     fi
-    
+
     if [ $BE -ne 0 ]; then
 	new "Kill backend"
 	# Check if premature kill
@@ -294,42 +296,47 @@ runtest(){
 
 # Compatible == all yang modules match
 # runtest <mode> <expected running> <expected startup>
+# This is really just that modstate is stripped from candidate and running if modstate is off
 new "1. Run without CLICON_XMLDB_MODSTATE ensure no modstate in datastore"
 (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
 (cd $dir; cp compat-valid.xml startup_db)
 runtest false startup '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>'
 
 new "Verify no modstate in running"
-expect="module"
+expect="modules-state"
 ret=$(sudo grep $expect $dir/running_db)
 if [ -n "$ret" ]; then
     err "did not expect $expect" "$ret"
 fi
 
 new "2. Load compatible valid startup (all OK)"
+# This is really just that modstate is used in candidate and running if modstate is on
 (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
 (cd $dir; cp compat-valid.xml startup_db)
 runtest true startup '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>'
 
 new "Verify modstate in running"
-expect="module"
+expect="modules-state"
 ret=$(sudo grep $expect $dir/running_db)
 if [ -z "$ret" ]; then
     err "Expected $expect" "$ret"
 fi
 
 new "3. Load compatible running valid running (rest of tests are startup)"
+# Just test that a valid db survives start from running
 (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
 (cd $dir; cp compat-valid.xml running_db)
 runtest true running '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' ''
 #'<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>'
 
 new "4. Load non-compat valid startup"
+# Just test that a valid db survives start from running
 (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
 (cd $dir; cp non-compat-valid.xml startup_db)
 runtest true startup '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>' '<data><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b></data>'
 
 new "5. Load non-compat invalid startup. Enter failsafe, startup invalid."
+# A test that if a non-valid startup is encountered, validation fails and failsafe is entered
 (cd $dir; rm -f tmp_db candidate_db running_db startup_db) # remove databases
 (cd $dir; cp non-compat-invalid.xml startup_db)
 runtest true startup '<data><a1 xmlns="urn:example:a">always work</a1></data>' '<data><a0 xmlns="urn:example:a">old version</a0><a1 xmlns="urn:example:a">always work</a1><b xmlns="urn:example:b">other text</b><c xmlns="urn:example:c">bla bla</c></data>' # sorted