From a5f32fbedf469bbc8b018f235c6dba1c6be8dcf1 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Tue, 26 Jan 2021 17:33:24 +0100 Subject: [PATCH] * Restconf evhtp using network namespaces implemented --- CHANGELOG.md | 3 +- apps/restconf/restconf_main_evhtp.c | 60 ++----- lib/clixon/clixon.h.in | 1 + lib/clixon/clixon_netns.h | 14 ++ lib/src/Makefile.in | 2 +- lib/src/clixon_netns.c | 247 ++++++++++++++++++++++++++++ test/test_restconf_netns.sh | 195 ++++++++++++++++++++++ test/test_restconf_rpc.sh | 55 +------ 8 files changed, 479 insertions(+), 98 deletions(-) create mode 100644 lib/clixon/clixon_netns.h create mode 100644 lib/src/clixon_netns.c create mode 100755 test/test_restconf_netns.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 512158c9..65086bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ Users may have to change how they access the system ### Minor changes +* Restconf evhtp using network namespaces implemented * Added validation of clixon-restconf.yang: server-key-path and server-cert-path must be present if ssl enabled. * Only if `CLICON_BACKEND_RESTCONF_PROCESS` is true * Experimental IPC API, `clixon_client`, to support a loose integration model @@ -63,7 +64,7 @@ Users may have to change how they access the system * This is work-in-progress and is expected to change * Use [https://github.com/clicon/libevhtp](https://github.com/clicon/libevhtp) instead of [https://github.com/criticalstack/libevhtp](https://github.com/criticalstack/libevhtp) as a source of the evhtp source * Added callback to process-control RPC feature in clixon-lib.yang to manage processes - * WHen an RPC comes in, be able to look at configuration + * When an RPC comes in, be able to look at configuration * Changed behavior of starting restconf internally using `CLICON_BACKEND_RESTCONF_PROCESS` monitoring changes in enable flag, not only the RPC. The semantics is as follows: * on RPC start, if enable is true, start the service, if false, error or ignore it * on RPC stop, stop the service diff --git a/apps/restconf/restconf_main_evhtp.c b/apps/restconf/restconf_main_evhtp.c index f7f9f723..395ab1af 100644 --- a/apps/restconf/restconf_main_evhtp.c +++ b/apps/restconf/restconf_main_evhtp.c @@ -623,33 +623,39 @@ cx_verify_certs(int pre_verify, return pre_verify; } -/*! +/*! Create and bind restconf socket * - * @param[out] addr Address as string, eg "0.0.0.0", "::" + * @param[in] netns0 Network namespace, special value "default" is same as NULL + * @param[in] addr Address as string, eg "0.0.0.0", "::" * @param[in] addrtype One of inet:ipv4-address or inet:ipv6-address + * @param[in] port TCP port * @param[out] ss Server socket (bound for accept) */ static int -restconf_socket_init(clicon_handle h, +restconf_socket_init(const char *netns0, const char *addr, const char *addrtype, uint16_t port, int *ss) { int retval = -1; - int s = -1; struct sockaddr * sa; struct sockaddr_in6 sin6 = { 0 }; struct sockaddr_in sin = { 0 }; size_t sin_len; - int on = 1; + const char *netns; + /* netns default -> NULL */ + if (netns0 != NULL && strcmp(netns0, "default")==0) + netns = NULL; + else + netns = netns0; if (strcmp(addrtype, "inet:ipv6-address") == 0) { sin_len = sizeof(struct sockaddr_in6); sin6.sin6_port = htons(port); sin6.sin6_family = AF_INET6; - evutil_inet_pton(AF_INET6, addr, &sin6.sin6_addr); + inet_pton(AF_INET6, addr, &sin6.sin6_addr); sa = (struct sockaddr *)&sin6; } else if (strcmp(addrtype, "inet:ipv4-address") == 0) { @@ -664,43 +670,11 @@ restconf_socket_init(clicon_handle h, clicon_err(OE_XML, EINVAL, "Unexpected addrtype: %s", addrtype); return -1; } - /* create inet socket */ - if ((s = socket(sa->sa_family, SOCK_STREAM, 0)) < 0) { - clicon_err(OE_UNIX, errno, "socket"); + if (clixon_netns_socket(netns, sa, sin_len, SOCKET_LISTEN_BACKLOG, ss) < 0) goto done; - } - evutil_make_socket_closeonexec(s); - evutil_make_socket_nonblocking(s); - if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, (void *)&on, sizeof(on)) == -1) { - clicon_err(OE_UNIX, errno, "setsockopt SO_KEEPALIVE"); - goto done; - } - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void *)&on, sizeof(on)) == -1) { - clicon_err(OE_UNIX, errno, "setsockopt SO_REUSEADDR"); - goto done; - } - /* only bind ipv6, otherwise it may bind to ipv4 as well which is strange but seems default */ - if (sa->sa_family == AF_INET6 && - setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) == -1) { - clicon_err(OE_UNIX, errno, "setsockopt IPPROTO_IPV6"); - goto done; - } - if (bind(s, sa, sin_len) == -1) { - clicon_err(OE_UNIX, errno, "bind port %u", port); - goto done; - } - if (listen(s, SOCKET_LISTEN_BACKLOG) < 0){ - clicon_err(OE_UNIX, errno, "listen"); - goto done; - } - if (ss) - *ss = s; retval = 0; done: - if (retval != 0 && s != -1) - evutil_closesocket(s); return retval; - // return evhtp_bind_sockaddr(htp, sa, sin_len, SOCKET_LISTEN_BACKLOG); } /*! Usage help routine @@ -865,12 +839,12 @@ cx_evhtp_socket(clicon_handle h, int auth_type_client_certificate) { int retval = -1; - char *namespace = NULL; + char *netns = NULL; char *address = NULL; char *addrtype = NULL; uint16_t ssl = 0; uint16_t port = 0; - int ss; + int ss = -1; evhtp_t *htp = NULL; /* This is socket create a new evhtp_t instance */ @@ -900,7 +874,7 @@ cx_evhtp_socket(clicon_handle h, evhtp_set_gencb(htp, cx_gencb, h); /* Extract socket parameters from single socket config: ns, addr, port, ssl */ - if (cx_evhtp_socket_extract(h, xs, nsc, &namespace, &address, &addrtype, &port, &ssl) < 0) + if (cx_evhtp_socket_extract(h, xs, nsc, &netns, &address, &addrtype, &port, &ssl) < 0) goto done; /* Sanity checks of socket parameters */ if (ssl){ @@ -915,7 +889,7 @@ cx_evhtp_socket(clicon_handle h, } } /* Open restconf socket and bind */ - if (restconf_socket_init(h, address, addrtype, port, &ss) < 0) + if (restconf_socket_init(netns, address, addrtype, port, &ss) < 0) goto done; /* ss is a server socket that the clients connect to. The callback therefore accepts clients on ss */ diff --git a/lib/clixon/clixon.h.in b/lib/clixon/clixon.h.in index 92c86d10..24fadf6a 100644 --- a/lib/clixon/clixon.h.in +++ b/lib/clixon/clixon.h.in @@ -73,6 +73,7 @@ extern "C" { #include #include #include +#include #include #include #include diff --git a/lib/clixon/clixon_netns.h b/lib/clixon/clixon_netns.h new file mode 100644 index 00000000..98f34ce8 --- /dev/null +++ b/lib/clixon/clixon_netns.h @@ -0,0 +1,14 @@ +/* + * Network namespace code + * @thanks Anders Franzén, especially get_sock() and send_sock() functions +*/ + +#ifndef _CLIXON_NETNS_H_ +#define _CLIXON_NETNS_H_ + +/* + * Prototypes + */ +int clixon_netns_socket(const char *netns, struct sockaddr *sa, size_t sin_len, int backlog, int *sock); + +#endif /* _CLIXON_NETNS_H_ */ diff --git a/lib/src/Makefile.in b/lib/src/Makefile.in index 68ac51a6..dd7c537d 100644 --- a/lib/src/Makefile.in +++ b/lib/src/Makefile.in @@ -83,7 +83,7 @@ SRC = clixon_sig.c clixon_uid.c clixon_log.c clixon_err.c clixon_event.c \ clixon_proto.c clixon_proto_client.c \ clixon_xpath.c clixon_xpath_ctx.c clixon_xpath_eval.c clixon_xpath_function.c clixon_xpath_optimize.c \ clixon_sha1.c clixon_datastore.c clixon_datastore_write.c clixon_datastore_read.c \ - clixon_netconf_lib.c clixon_stream.c clixon_nacm.c clixon_client.c + clixon_netconf_lib.c clixon_stream.c clixon_nacm.c clixon_client.c clixon_netns.c YACCOBJS = lex.clixon_xml_parse.o clixon_xml_parse.tab.o \ lex.clixon_yang_parse.o clixon_yang_parse.tab.o \ diff --git a/lib/src/clixon_netns.c b/lib/src/clixon_netns.c new file mode 100644 index 00000000..8743b6e6 --- /dev/null +++ b/lib/src/clixon_netns.c @@ -0,0 +1,247 @@ +/* + * + * Network namespace code + * @thanks Anders Franzén, especially get_sock() and send_sock() functions + * + * fork, + * child: + * switch to ns, + * create sock, + * bind to address, + * sendmsg sock back to parent + * parent: + * readmsg sock from child + * kill child? + * return sock + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "clixon_err.h" +#include "clixon_log.h" +#include "clixon_netns.h" + +/* + * @thanks Anders Franzén + */ +static int +send_sock(int usock, + int fd) +{ + int retval = -1; + int *fdptr; + struct msghdr msg={0}; + struct cmsghdr *cmsg; + char buf[CMSG_SPACE(sizeof(fd))]; + + memset(buf,0,sizeof(buf)); + msg.msg_control=buf; + msg.msg_controllen=sizeof(buf); + cmsg=CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level=SOL_SOCKET; + cmsg->cmsg_type=SCM_RIGHTS; + cmsg->cmsg_len=CMSG_LEN(sizeof(fd)); + fdptr=(int *)CMSG_DATA(cmsg); + memcpy(fdptr,&fd,sizeof(fd)); + msg.msg_controllen=CMSG_SPACE(sizeof(fd)); + if (sendmsg(usock, &msg, 0) < 0){ + clicon_err(OE_UNIX, errno, "sendmsg"); + goto done; + } + retval = 0; + done: + return retval; +} + +/* + * @thanks Anders Franzén + */ +static int +get_sock(int usock, + int *fd) +{ + int retval = -1; + struct msghdr msg={0}; + struct cmsghdr *cmsg; + char buf[128]; + + msg.msg_iov=0; + msg.msg_iovlen=0; + msg.msg_control=buf; + msg.msg_controllen=sizeof(buf); + /* Block here */ + if (recvmsg(usock, &msg, 0) < 0){ + clicon_err(OE_UNIX, errno, "recvmsg"); + goto done; + } + cmsg=CMSG_FIRSTHDR(&msg); + memcpy(fd, CMSG_DATA(cmsg), sizeof(*fd)); + retval = 0; + done: + return retval; +} + +/*! Create and bind stream socket + * @param[in] sa Socketaddress + * @param[in] sa_len Length of sa. Tecynicaliyu to be independent of sockaddr sa_len + * @param[in] backlog Listen backlog, queie of pending connections + * @param[out] sock Server socket (bound for accept) + */ +int +create_socket(struct sockaddr *sa, + size_t sin_len, + int backlog, + int *sock) +{ + int retval = -1; + int s = -1; + int on = 1; + + clicon_debug(1, "%s", __FUNCTION__); + if (sock == NULL){ + clicon_err(OE_PROTO, EINVAL, "Requires socket output parameter"); + goto done; + } + /* create inet socket */ + if ((s = socket(sa->sa_family, + SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, + 0)) < 0) { + clicon_err(OE_UNIX, errno, "socket"); + goto done; + } + if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, (void *)&on, sizeof(on)) == -1) { + clicon_err(OE_UNIX, errno, "setsockopt SO_KEEPALIVE"); + goto done; + } + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void *)&on, sizeof(on)) == -1) { + clicon_err(OE_UNIX, errno, "setsockopt SO_REUSEADDR"); + goto done; + } + /* only bind ipv6, otherwise it may bind to ipv4 as well which is strange but seems default */ + if (sa->sa_family == AF_INET6 && + setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) == -1) { + clicon_err(OE_UNIX, errno, "setsockopt IPPROTO_IPV6"); + goto done; + } + if (bind(s, sa, sin_len) == -1) { + clicon_err(OE_UNIX, errno, "bind"); + goto done; + } + if (listen(s, backlog) < 0){ + clicon_err(OE_UNIX, errno, "listen"); + goto done; + } + if (sock) + *sock = s; + retval = 0; + done: + if (retval != 0 && s != -1) + close(s); + return retval; +} + +int +fork_netns_socket(const char *netns, + struct sockaddr *sa, + size_t sin_len, + int backlog, + int *sock) +{ + int retval = -1; + int sp[2] = {0,}; + pid_t child; + + if (socketpair(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0, sp) < 0){ + clicon_err(OE_UNIX, errno, "socketpair"); + goto done; + } + if ((child = fork()) < 0) { + clicon_err(OE_UNIX, errno, "fork"); + goto done; + } + if (child == 0) { /* Child */ + char path[MAXPATHLEN]; + int fd; + int s = -1; + + close(sp[0]); + /* Switch to namespace */ + sprintf(path,"/var/run/netns/%s", netns); + if ((fd=open(path, O_RDONLY)) < 0) { + clicon_err(OE_UNIX, errno, "open"); + return -1; + } + if (setns(fd, CLONE_NEWNET) < 0){ + clicon_err(OE_UNIX, errno, "setns"); + return -1; + } + close(fd); + /* Create socket in this namespace */ + if (create_socket(sa, sin_len, backlog, &s) < 0) + return -1; + /* Send socket to parent */ + if (send_sock(sp[1], s) < 0) + return -1; + close(s); + close(sp[1]); + exit(0); + } + /* Parent */ + close(sp[1]); + if (get_sock(sp[0], sock) < 0) + goto done; + close(sp[0]); + retval = 0; + done: + return retval; +} + +/*! Create and bind stream socket in network namespace + * @param[in] netns Network namespace + * @param[in] sa Socketaddress + * @param[in] sa_len Length of sa. Tecynicaliyu to be independent of sockaddr sa_len + * @param[in] backlog Listen backlog, queie of pending connections + * @param[out] sock Server socket (bound for accept) + */ +int +clixon_netns_socket(const char *netns, + struct sockaddr *sa, + size_t sin_len, + int backlog, + int *sock) +{ + int retval = -1; + + clicon_debug(1, "%s", __FUNCTION__); + if (netns == NULL){ + if (create_socket(sa, sin_len, backlog, sock) < 0) + goto done; + goto ok; + } + else { + if (fork_netns_socket(netns, sa, sin_len, backlog, sock) < 0) + goto done; + } + ok: + retval = 0; + done: + return retval; +} diff --git a/test/test_restconf_netns.sh b/test/test_restconf_netns.sh new file mode 100755 index 00000000..eb0ec0ef --- /dev/null +++ b/test/test_restconf_netns.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# Restconf evhtp using socket network namespace (netns) support +# Listen to a default and a separate netns +# Init running with a=42 +# Get the config from default and netns namespace with/without SSL +# Write b=99 in netns and read from default + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +# Skip it other than evhtp +if [ "${WITH_RESTCONF}" != "evhtp" ]; then + if [ "$s" = $0 ]; then exit 0; else return 0; fi # skip +fi + +APPNAME=example + +cfg=$dir/conf.xml +startupdb=$dir/startup_db + +netns=clixonnetns +veth=veth0 +vethpeer=veth1 +vaddr=10.23.1.1 # address in netns + +# Create server certs +certdir=$dir/certs +srvkey=$certdir/srv_key.pem +srvcert=$certdir/srv_cert.pem +cakey=$certdir/ca_key.pem # needed? +cacert=$certdir/ca_cert.pem +test -d $certdir || mkdir $certdir +. ./certs.sh + +# XXX Note default port need to be 80 for wait_restconf to work +RESTCONFIG=$(cat < + true + password + $srvcert + $srvkey + $cakey + + default +
0.0.0.0
+ 80 + false + + + + $netns +
0.0.0.0
+ 80 + false + + + + $netns +
0.0.0.0
+ 443 + true + +" +EOF +) + +cat < $cfg + + $cfg + ietf-netconf:startup + /usr/local/share/clixon + $IETFRFC + clixon-example + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/backend + example_backend.so$ + /usr/local/lib/$APPNAME/restconf + false + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + true + $RESTCONFIG + +EOF + +new "Create netns: $netns" +sudo ip netns delete $netns +# Create netns +sudo ip netns add $netns +if [ -z "$(ip netns list | grep $netns)" ]; then + err "$netns" "$netns does not exist" +fi + +new "Create veth pair: $veth and $vethpeer" +sudo ip link delete $veth 2> /dev/null +sudo ip link delete $vethpeer 2> /dev/null +sudo ip link add $veth type veth peer name $vethpeer +if [ -z "$(ip netns show $veth)" ]; then + err "$veth" "$veth does not exist" +fi +if [ -z "$(ip netns show $vethpeer)" ]; then + err "$veth" "$vethpeer does not exist" +fi + +new "Move $vethpeer to netns $netns" +sudo ip link set $vethpeer netns $netns +if [ -z "$( sudo ip netns exec $netns ip link show $vethpeer)" ]; then + err "$veth" "$vethpeer does not exist" +fi + +new "Assign address $vaddr on $veth in netns $netns" +sudo ip netns exec $netns ip addr add $vaddr/24 dev $vethpeer +sudo ip netns exec $netns ip link set dev $vethpeer up +sudo ip netns exec $netns ip link set dev lo up +#sudo ip netns exec $netns ping $vaddr + +#----------------- + +new "test params: -f $cfg" +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -z -f $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg + + new "waiting" + wait_backend +fi + +if [ $RC -ne 0 ]; then + new "kill old restconf daemon" + stop_restconf_pre + + new "start restconf daemon" + start_restconf -f $cfg + + new "waiting" + wait_restconf # need to use port 80/443 +fi + +new "add sample config w netconf" +expecteof "$clixon_netconf -qf $cfg" 0 "a42
]]>]]>" "^]]>]]>$" + +new "netconf commit" +expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^]]>]]>$" + +new "restconf http get config on default netns" +expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' http://127.0.0.1/restconf/data/clixon-example:table)" 0 "HTTP/1.1 200 OK" 'a42
' + +new "restconf http get config on addr:$vaddr in netns:$netns" +expectpart "$(sudo ip netns exec $netns curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' https://$vaddr/restconf/data/clixon-example:table)" 0 "HTTP/1.1 200 OK" 'a42
' + +new "restconf https/SSL get config on addr:$vaddr in netns:$netns" +expectpart "$(sudo ip netns exec $netns curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' https://$vaddr/restconf/data/clixon-example:table)" 0 "HTTP/1.1 200 OK" 'a42
' + +new "restconf https/SSL put table b" +expectpart "$(sudo ip netns exec $netns curl $CURLOPTS -X POST -H 'Content-Type: application/yang-data+xml' -d 'b99' https://$vaddr/restconf/data/clixon-example:table)" 0 "HTTP/1.1 201 Created" + +new "restconf http get table b on default ns" +expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' http://127.0.0.1/restconf/data/clixon-example:table/parameter=b)" 0 "HTTP/1.1 200 OK" 'b99' + +# Negative +new "restconf get config on wrong port in netns:$netns" +expectpart "$(sudo ip netns exec $netns curl $CURLOPTS -X GET -H 'Accept: application/yang-data+xml' $RCPROTO://$vaddr:8888/restconf/data/clixon-example:table)" 7 + +if [ $RC -ne 0 ]; then + new "Kill restconf daemon" + stop_restconf +fi + +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg +fi + +sudo ip link delete $veth +sudo ip netns delete $netns + +new "endtest" +endtest + +rm -rf $dir + diff --git a/test/test_restconf_rpc.sh b/test/test_restconf_rpc.sh index 23bfd27f..467211d4 100755 --- a/test/test_restconf_rpc.sh +++ b/test/test_restconf_rpc.sh @@ -6,7 +6,7 @@ # - on backend start make the state as configured # - on enable change, make the state as configured # - No restconf config means enable: false (extra rule) -# Also work-in-progress network namespaces, ip netns +# See test_restconf_netns for network namespaces # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi @@ -270,61 +270,10 @@ expecteof "$clixon_netconf -qf $cfg" 0 "]]>]]>" "^applicationoperation-failederrorSSL enabled but server-cert-path not set]]>]]>$" -if false; then # Work in progress - namespace -#------------------------------- -# Now in a separate network namespace -new "restconf rpc in network namespace" -netns=xxx -sudo ip netns delete $netns -#sudo ip netns add $netns - -new "test params: -f $cfg" -if [ $BE -ne 0 ]; then - new "kill old backend" - sudo clixon_backend -z -f $cfg - if [ $? -ne 0 ]; then - err - fi - new "start backend -s init -f $cfg -- -n $netns" - start_backend -s init -f $cfg -- -n $netns - - new "waiting" - wait_backend -fi - -new "kill old restconf" -stop_restconf_pre - -new "netconf start restconf" -expecteof "$clixon_netconf -qf $cfg" 0 "restconfstart]]>]]>" "]]>]]>" - -new "10)check status on" -expecteof "$clixon_netconf -qf $cfg" 0 "restconfstatus]]>]]>" "true]]>]]>" - -new "stop restconf" -expecteof "$clixon_netconf -qf $cfg" 0 "restconfstop]]>]]>" "]]>]]>" - -if [ $BE -ne 0 ]; then - new "Kill backend" - # Check if premature kill - pid=$(pgrep -u root -f clixon_backend) - if [ -z "$pid" ]; then - err "backend already dead" - fi - # kill backend - stop_backend -f $cfg - - new "11)check no restconf" - ps=$(ps aux|grep "$WWWDIR/clixon_restconf" | grep -v grep) -fi - -sudo ip netns delete $netns - -fi # namespaces - unset pid sleep $DEMWAIT # Lots of processes need to die before next test +new "endtest" endtest rm -rf $dir