From 2b2a2ec1addd82a7dc0347cca31ebec3a0853903 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 22 Apr 2024 14:17:29 +0200 Subject: [PATCH] Restconf native stream support --- apps/backend/backend_client.c | 15 +- apps/backend/backend_get.c | 4 +- apps/restconf/Makefile.in | 6 +- apps/restconf/README.md | 8 + apps/restconf/restconf_lib.c | 16 +- apps/restconf/restconf_lib.h | 8 +- apps/restconf/restconf_nghttp2.c | 36 ++- apps/restconf/restconf_root.c | 27 ++- apps/restconf/restconf_stream.c | 202 +++++++++++++++++ apps/restconf/restconf_stream.h | 1 + apps/restconf/restconf_stream_fcgi.c | 117 +--------- apps/restconf/restconf_stream_native.c | 296 +++++++++++++++++++++++++ include/clixon_custom.h | 5 + test/test_restconf_notifications.sh | 90 ++++---- 14 files changed, 626 insertions(+), 205 deletions(-) create mode 100644 apps/restconf/restconf_stream.c create mode 100644 apps/restconf/restconf_stream_native.c diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 4cd62d05..6fc07428 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -1153,9 +1153,14 @@ from_client_create_subscription(clixon_handle h, /* XXX should use prefix cf edit_config */ if ((nsc = xml_nsctx_init(NULL, EVENT_RFC5277_NAMESPACE)) == NULL) goto done; - if ((x = xpath_first(xe, nsc, "//stream")) != NULL) - stream = xml_find_value(x, "body"); - if ((x = xpath_first(xe, nsc, "//stopTime")) != NULL){ + if ((x = xpath_first(xe, nsc, "stream")) != NULL){ + if ((stream = xml_find_value(x, "body")) == NULL){ + if (netconf_bad_element(cbret, "application", "stream", "Expected stream name") < 0) + goto done; + goto ok; + } + } + if ((x = xpath_first(xe, nsc, "stopTime")) != NULL){ if ((stoptime = xml_find_value(x, "body")) != NULL && str2time(stoptime, &stop) < 0){ if (netconf_bad_element(cbret, "application", "stopTime", "Expected timestamp") < 0) @@ -1163,7 +1168,7 @@ from_client_create_subscription(clixon_handle h, goto ok; } } - if ((x = xpath_first(xe, nsc, "//startTime")) != NULL){ + if ((x = xpath_first(xe, nsc, "startTime")) != NULL){ if ((starttime = xml_find_value(x, "body")) != NULL && str2time(starttime, &start) < 0){ if (netconf_bad_element(cbret, "application", "startTime", "Expected timestamp") < 0) @@ -1171,7 +1176,7 @@ from_client_create_subscription(clixon_handle h, goto ok; } } - if ((xfilter = xpath_first(xe, nsc, "//filter")) != NULL){ + if ((xfilter = xpath_first(xe, nsc, "filter")) != NULL){ if ((ftype = xml_find_value(xfilter, "type")) != NULL){ /* Only accept xpath as filter type */ if (strcmp(ftype, "xpath") != 0){ diff --git a/apps/backend/backend_get.c b/apps/backend/backend_get.c index 89a7323a..36a0b65e 100644 --- a/apps/backend/backend_get.c +++ b/apps/backend/backend_get.c @@ -72,7 +72,7 @@ #include "backend_handle.h" #include "backend_get.h" -/*! restrconf get capabilities +/*! Restconf get capabilities * * Maybe should be in the restconf client instead of backend? * @param[in] h Clixon handle @@ -154,7 +154,7 @@ client_get_streams(clixon_handle h, /* Second argument is a hack to have the same function for the * RFC5277 and 8040 stream cases */ - if (stream_get_xml(h, strcmp(top,"restconf-state")==0, cb) < 0) + if (stream_get_xml(h, strcmp(top, "restconf-state")==0, cb) < 0) goto done; cprintf(cb,"", top); diff --git a/apps/restconf/Makefile.in b/apps/restconf/Makefile.in index 8b580e45..c5992829 100644 --- a/apps/restconf/Makefile.in +++ b/apps/restconf/Makefile.in @@ -107,6 +107,7 @@ APPSRC += restconf_methods_post.c APPSRC += restconf_methods_get.c APPSRC += restconf_methods_patch.c APPSRC += restconf_root.c +APPSRC += restconf_stream.c APPSRC += clixon_http_data.c APPSRC += restconf_main_$(with_restconf).c ifeq ($(with_restconf),native) @@ -115,11 +116,8 @@ APPSRC += restconf_native.c APPSRC += restconf_nghttp2.c # HTTP/2 endif -# Fcgi-specific source including main -ifeq ($(with_restconf),fcgi) -# Streams notifications have some fcgi specific handling +# Streams notifications have some fcgi/nghttp2 specific handling APPSRC += restconf_stream_$(with_restconf).c -endif # internal http/1 parser YACCOBJS = diff --git a/apps/restconf/README.md b/apps/restconf/README.md index 462c6d24..17d662e4 100644 --- a/apps/restconf/README.md +++ b/apps/restconf/README.md @@ -20,6 +20,14 @@ Ensure www-data is member of the CLICON_SOCK_GROUP (default clicon). If not, add sudo usermod -a -G clicon www-data ``` +### nghttp2 + +For details on the C API see https://nghttp2.org + +### openssl + +For details on the C-API see https://www.openssl.org/ docs/manual pages + ## Nginx Installation instruction for Nginx. Other reverse proxies should work but are not verified. diff --git a/apps/restconf/restconf_lib.c b/apps/restconf/restconf_lib.c index f19582bb..8cff2f99 100644 --- a/apps/restconf/restconf_lib.c +++ b/apps/restconf/restconf_lib.c @@ -207,8 +207,8 @@ static const map_str2int http_media_map[] = { {"application/yang-data+json", YANG_DATA_JSON}, {"application/yang-patch+xml", YANG_PATCH_XML}, {"application/yang-patch+json", YANG_PATCH_JSON}, - {"application/yang-data+xml-list", YANG_PAGINATION_XML}, /* draft-wwlh-netconf-list-pagination-rc-02 */ - {NULL, -1} + {"application/yang-data+xml-list", YANG_PAGINATION_XML}, /* draft-wwlh-netconf-list-pagination-rc-02 */ + {NULL, -1} }; /* Mapping to http proto types */ @@ -749,12 +749,12 @@ int restconf_config_init(clixon_handle h, cxobj *xrestconf) { - int retval = -1; - char *enable; - cxobj *x; - char *bstr; - cvec *nsc = NULL; - int auth_type; + int retval = -1; + char *enable; + cxobj *x; + char *bstr; + cvec *nsc = NULL; + int auth_type; yang_stmt *yspec; yang_stmt *y; diff --git a/apps/restconf/restconf_lib.h b/apps/restconf/restconf_lib.h index 3644da0c..93fbdbd5 100644 --- a/apps/restconf/restconf_lib.h +++ b/apps/restconf/restconf_lib.h @@ -50,10 +50,10 @@ extern "C" { * @note DUPLICATED in clixon_restconf.h */ enum restconf_media{ - YANG_DATA_JSON, /* "application/yang-data+json" */ - YANG_DATA_XML, /* "application/yang-data+xml" */ - YANG_PATCH_JSON, /* "application/yang-patch+json" */ - YANG_PATCH_XML, /* "application/yang-patch+xml" */ + YANG_DATA_JSON, /* "application/yang-data+json" */ + YANG_DATA_XML, /* "application/yang-data+xml" */ + YANG_PATCH_JSON, /* "application/yang-patch+json" */ + YANG_PATCH_XML, /* "application/yang-patch+xml" */ YANG_PAGINATION_XML, /* draft-wwlh-netconf-list-pagination-rc-02.txt */ }; typedef enum restconf_media restconf_media; diff --git a/apps/restconf/restconf_nghttp2.c b/apps/restconf/restconf_nghttp2.c index b5701132..a04f89a9 100644 --- a/apps/restconf/restconf_nghttp2.c +++ b/apps/restconf/restconf_nghttp2.c @@ -85,6 +85,9 @@ #include "restconf_err.h" #include "restconf_root.h" #include "restconf_native.h" /* Restconf-openssl mode specific headers*/ +#ifdef RESTCONF_NATIVE_STREAM +#include "restconf_stream.h" +#endif #ifdef HAVE_LIBNGHTTP2 /* Ends at end-of-file */ #include "restconf_nghttp2.h" /* Restconf-openssl mode specific headers*/ #include "clixon_http_data.h" @@ -145,10 +148,10 @@ static void nghttp2_print_headers(nghttp2_nv *nva, size_t nvlen) { - size_t i; + size_t i; - for (i = 0; i < nvlen; ++i) - nghttp2_print_header(nva[i].name, nva[i].namelen, nva[i].value, nva[i].valuelen); + for (i = 0; i < nvlen; ++i) + nghttp2_print_header(nva[i].name, nva[i].namelen, nva[i].value, nva[i].valuelen); } #endif /* NOTUSED */ @@ -338,6 +341,12 @@ restconf_nghttp2_path(restconf_stream_data *sd) if (api_http_data(h, sd, sd->sd_qvec) < 0) goto done; } +#ifdef RESTCONF_NATIVE_STREAM + else if (api_path_is_stream(h)){ + if (api_stream(h, sd, sd->sd_qvec, NULL) < 0) + goto done; + } +#endif else if (api_root_restconf(h, sd, sd->sd_qvec) < 0) /* error handling */ goto done; } @@ -482,13 +491,21 @@ http2_exec(restconf_conn *rc, if ((sd->sd_path = restconf_uripath(rc->rc_h)) == NULL) goto done; sd->sd_proto = HTTP_2; /* XXX is this necessary? */ + clixon_debug(CLIXON_DBG_RESTCONF, "path:%s", sd->sd_path); + /* Early sanity check. Full dispatch in restconf_nghttp2_path */ if (strcmp(sd->sd_path, RESTCONF_WELL_KNOWN) == 0 || api_path_is_restconf(rc->rc_h) - || api_path_is_data(rc->rc_h)){ + || api_path_is_data(rc->rc_h) +#ifdef RESTCONF_NATIVE_STREAM + || api_path_is_stream(rc->rc_h) +#endif + ) { + clixon_debug(CLIXON_DBG_RESTCONF, "path found"); if (restconf_nghttp2_path(sd) < 0) goto done; } else{ + clixon_debug(CLIXON_DBG_RESTCONF, "path not found"); sd->sd_code = 404; /* not found */ } if (restconf_param_del_all(rc->rc_h) < 0) // XXX @@ -766,12 +783,12 @@ select_padding_callback(nghttp2_session *session, */ static ssize_t data_source_read_length_callback(nghttp2_session *session, - uint8_t frame_type, - int32_t stream_id, - int32_t session_remote_window_size, - int32_t stream_remote_window_size, + uint8_t frame_type, + int32_t stream_id, + int32_t session_remote_window_size, + int32_t stream_remote_window_size, uint32_t remote_max_frame_size, - void *user_data) + void *user_data) { // restconf_conn *rc = (restconf_conn *)user_data; clixon_debug(CLIXON_DBG_RESTCONF, ""); @@ -1001,6 +1018,7 @@ http2_session_init(restconf_conn *rc) nghttp2_session_callbacks_set_select_padding_callback(callbacks, select_padding_callback); nghttp2_session_callbacks_set_data_source_read_length_callback(callbacks, data_source_read_length_callback); #endif + nghttp2_session_callbacks_set_on_begin_frame_callback(callbacks, on_begin_frame_callback); nghttp2_session_callbacks_set_send_data_callback(callbacks, send_data_callback); diff --git a/apps/restconf/restconf_root.c b/apps/restconf/restconf_root.c index 02b0489d..13ca2e4e 100644 --- a/apps/restconf/restconf_root.c +++ b/apps/restconf/restconf_root.c @@ -82,14 +82,14 @@ api_path_is_restconf(clixon_handle h) char *path = NULL; char *restconf_api_path; - if ((path = restconf_uripath(h)) == NULL) - goto done; + if ((path = restconf_uripath(h)) == NULL) + goto done; if ((restconf_api_path = clicon_option_str(h, "CLICON_RESTCONF_API_ROOT")) == NULL) goto done; - if (strlen(path) < strlen(restconf_api_path)) /* "/" + restconf */ - goto done; - if (strncmp(path, restconf_api_path, strlen(restconf_api_path)) != 0) - goto done; + if (strlen(path) < strlen(restconf_api_path)) /* "/" + restconf */ + goto done; + if (strncmp(path, restconf_api_path, strlen(restconf_api_path)) != 0) + goto done; retval = 1; done: if (path) @@ -171,7 +171,6 @@ api_root_restconf_exact(clixon_handle h, char *request_method, int pretty, restconf_media media_out) - { int retval = -1; yang_stmt *yspec; @@ -269,9 +268,9 @@ api_yang_library_version(clixon_handle h, int pretty, restconf_media media_out) { - int retval = -1; - cxobj *xt = NULL; - cbuf *cb = NULL; + int retval = -1; + cxobj *xt = NULL; + cbuf *cb = NULL; yang_stmt *yspec; clixon_debug(CLIXON_DBG_RESTCONF, ""); @@ -356,9 +355,9 @@ api_data(clixon_handle h, /* https://tools.ietf.org/html/rfc8527#section-3.2 */ /* We assume that dynamic datastores are read only at this time 20201105 */ if (IETF_DS_DYNAMIC == ds) - dynamic = 1; + dynamic = 1; if ((IETF_DS_INTENDED == ds) || (IETF_DS_RUNNING == ds) - || (IETF_DS_DYNAMIC == ds) || (IETF_DS_OPERATIONAL == ds)) { + || (IETF_DS_DYNAMIC == ds) || (IETF_DS_OPERATIONAL == ds)) { read_only = 1; } @@ -545,7 +544,7 @@ api_root_restconf(clixon_handle h, } clixon_debug(CLIXON_DBG_RESTCONF, "api_resource=%s", api_resource); if (uri_str2cvec(path, '/', '=', 1, &pcvec) < 0) /* rest url eg /album=ricky/foo */ - goto done; + goto done; /* data */ if ((cb = restconf_get_indata(req)) == NULL) /* XXX NYI ACTUALLY not always needed, do this later? */ goto done; @@ -606,7 +605,7 @@ api_root_restconf(clixon_handle h, goto done; if (api_return_err0(h, req, xerr, pretty, media_out, 0) < 0) goto done; - goto ok; + goto ok; } /* ds is assigned at this point */ if (0 > api_data(h, req, path, pcvec, 3, qvec, indata, pretty, media_out, ds)) diff --git a/apps/restconf/restconf_stream.c b/apps/restconf/restconf_stream.c new file mode 100644 index 00000000..63ebf9b1 --- /dev/null +++ b/apps/restconf/restconf_stream.c @@ -0,0 +1,202 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2024 Olof Hagsand and Rubicon Communications, LLC(Netgate) + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + Restconf event stream implementation. + See RFC 8040 RESTCONF Protocol + Sections 3.8, 6, 9.3 + + RFC8040: + A RESTCONF server MAY send the "retry" field, and if it does, RESTCONF + clients SHOULD use it. A RESTCONF server SHOULD NOT send the "event" + or "id" fields, as there are no meaningful values. RESTCONF + servers that do not send the "id" field also do not need to support + the HTTP header field "Last-Event-ID" + + The RESTCONF client can then use this URL value to start monitoring + the event stream: + + GET /streams/NETCONF HTTP/1.1 + Host: example.com + Accept: text/event-stream + Cache-Control: no-cache + Connection: keep-alive + + The server MAY support the "start-time", "stop-time", and "filter" + query parameters, defined in Section 4.8. Refer to Appendix B.3.6 + for filter parameter examples. + + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* cligen */ +#include + +/* clixon */ +#include + +#include "restconf_lib.h" +#include "restconf_handle.h" +#include "restconf_api.h" +#include "restconf_err.h" +#include "restconf_stream.h" + +/*! Check if uri path denotes a stream/notification path + * + * @retval 1 Yes, a stream path + * @retval 0 No, not a stream path + */ +int +api_path_is_stream(clixon_handle h) +{ + int retval = 0; + char *path = NULL; + char *stream_path; + + if ((path = restconf_uripath(h)) == NULL) + goto done; + if ((stream_path = clicon_option_str(h, "CLICON_STREAM_PATH")) == NULL) + goto done; + if (strlen(path) < 1 + strlen(stream_path)) /* "/" + stream */ + goto done; + if (path[0] != '/') + goto done; + if (strncmp(path+1, stream_path, strlen(stream_path)) != 0) + goto done; + retval = 1; + done: + if (path) + free(path); + return retval; +} + +/*! Send subscription to backend + * + * @param[in] h Clixon handle + * @param[in] req Generic Www handle (can be part of clixon handle) + * @param[in] name Stream name + * @param[in] qvec + * @param[in] pretty Pretty-print json/xml reply + * @param[in] media_out Restconf output media + * @param[out] sp Socket -1 if not set + * @retval 0 OK + * @retval -1 Error + */ +int +restconf_subscription(clixon_handle h, + void *req, + char *name, + cvec *qvec, + int pretty, + restconf_media media_out, + int *sp) +{ + int retval = -1; + cxobj *xret = NULL; + cxobj *xe; + cbuf *cb = NULL; + int s; /* socket */ + int i; + cg_var *cv; + char *vname; + + clixon_debug(CLIXON_DBG_STREAM, ""); + *sp = -1; + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + cprintf(cb, "%s", + NETCONF_BASE_NAMESPACE, NETCONF_MESSAGE_ID_ATTR, EVENT_RFC5277_NAMESPACE, name); + /* Print all fields */ + for (i=0; i"); + cv2cbuf(cv, cb); + cprintf(cb, ""); + } + else if (strcmp(vname, "stop-time") == 0){ + cprintf(cb, ""); + cv2cbuf(cv, cb); + cprintf(cb, ""); + } + } + cprintf(cb, "]]>]]>"); + if (clicon_rpc_netconf(h, cbuf_get(cb), &xret, &s) < 0) + goto done; + if ((xe = xpath_first(xret, NULL, "rpc-reply/rpc-error")) != NULL){ + if (api_return_err(h, req, xe, pretty, media_out, 0) < 0) + goto done; + goto ok; + } + /* Setting up stream */ + if (restconf_reply_header(req, "Content-Type", "text/event-stream") < 0) + goto done; + if (restconf_reply_header(req, "Cache-Control", "no-cache") < 0) + goto done; + if (restconf_reply_header(req, "Connection", "keep-alive") < 0) + goto done; + if (restconf_reply_header(req, "X-Accel-Buffering", "no") < 0) + goto done; + if (restconf_reply_send(req, 201, NULL, 0) < 0) + goto done; + *sp = s; + ok: + retval = 0; + done: + clixon_debug(CLIXON_DBG_STREAM, "retval: %d", retval); + if (xret) + xml_free(xret); + if (cb) + cbuf_free(cb); + return retval; +} diff --git a/apps/restconf/restconf_stream.h b/apps/restconf/restconf_stream.h index e11cb856..2aa7107c 100644 --- a/apps/restconf/restconf_stream.h +++ b/apps/restconf/restconf_stream.h @@ -41,6 +41,7 @@ * Prototypes */ int api_path_is_stream(clixon_handle h); +int restconf_subscription(clixon_handle h, void *req, char *name, cvec *qvec, int pretty, restconf_media media_out, int *sp); int stream_child_free(clixon_handle h, int pid); int stream_child_freeall(clixon_handle h); int api_stream(clixon_handle h, void *req, cvec *qvec, int *finish); diff --git a/apps/restconf/restconf_stream_fcgi.c b/apps/restconf/restconf_stream_fcgi.c index 63ff7b78..c20ec1f6 100644 --- a/apps/restconf/restconf_stream_fcgi.c +++ b/apps/restconf/restconf_stream_fcgi.c @@ -118,36 +118,6 @@ struct stream_child{ */ static struct stream_child *STREAM_CHILD = NULL; -/*! Check if uri path denotes a stream/notification path - * - * @retval 1 Yes, a stream path - * @retval 0 No, not a stream path - */ -int -api_path_is_stream(clixon_handle h) -{ - int retval = 0; - char *path = NULL; - char *stream_path; - - if ((path = restconf_uripath(h)) == NULL) - goto done; - if ((stream_path = clicon_option_str(h, "CLICON_STREAM_PATH")) == NULL) - goto done; - if (strlen(path) < 1 + strlen(stream_path)) /* "/" + stream */ - goto done; - if (path[0] != '/') - goto done; - if (strncmp(path+1, stream_path, strlen(stream_path)) != 0) - goto done; - retval = 1; - done: - if (path) - free(path); - return retval; - -} - /*! Find restconf child using PID and cleanup FCGI Request data * * For forked, called on SIGCHILD @@ -272,91 +242,6 @@ restconf_stream_cb(int s, return retval; } -/*! Send subscription to backend - * - * @param[in] h Clixon handle - * @param[in] req Generic Www handle (can be part of clixon handle) - * @param[in] name Stream name - * @param[in] qvec - * @param[in] pretty Pretty-print json/xml reply - * @param[in] media_out Restconf output media - * @param[out] sp Socket -1 if not set - * @retval 0 OK - * @retval -1 Error - */ -static int -restconf_stream(clixon_handle h, - void *req, - char *name, - cvec *qvec, - int pretty, - restconf_media media_out, - int *sp) -{ - int retval = -1; - cxobj *xret = NULL; - cxobj *xe; - cbuf *cb = NULL; - int s; /* socket */ - int i; - cg_var *cv; - char *vname; - - clixon_debug(CLIXON_DBG_STREAM, ""); - *sp = -1; - if ((cb = cbuf_new()) == NULL){ - clixon_err(OE_XML, errno, "cbuf_new"); - goto done; - } - cprintf(cb, "%s", - NETCONF_BASE_NAMESPACE, NETCONF_MESSAGE_ID_ATTR, EVENT_RFC5277_NAMESPACE, name); - /* Print all fields */ - for (i=0; i"); - cv2cbuf(cv, cb); - cprintf(cb, ""); - } - else if (strcmp(vname, "stop-time") == 0){ - cprintf(cb, ""); - cv2cbuf(cv, cb); - cprintf(cb, ""); - } - } - cprintf(cb, "]]>]]>"); - if (clicon_rpc_netconf(h, cbuf_get(cb), &xret, &s) < 0) - goto done; - if ((xe = xpath_first(xret, NULL, "rpc-reply/rpc-error")) != NULL){ - if (api_return_err(h, req, xe, pretty, media_out, 0) < 0) - goto done; - goto ok; - } - - /* Setting up stream */ - if (restconf_reply_header(req, "Content-Type", "text/event-stream") < 0) - goto done; - if (restconf_reply_header(req, "Cache-Control", "no-cache") < 0) - goto done; - if (restconf_reply_header(req, "Connection", "keep-alive") < 0) - goto done; - if (restconf_reply_header(req, "X-Accel-Buffering", "no") < 0) - goto done; - if (restconf_reply_send(req, 201, NULL, 0) < 0) - goto done; - *sp = s; - ok: - retval = 0; - done: - clixon_debug(CLIXON_DBG_STREAM, "retval: %d", retval); - if (xret) - xml_free(xret); - if (cb) - cbuf_free(cb); - return retval; -} - /* restconf */ #include "restconf_lib.h" #include "restconf_stream.h" @@ -491,7 +376,7 @@ api_stream(clixon_handle h, goto done; if (ret == 0) goto ok; - if (restconf_stream(h, req, method, qvec, pretty, media_out, &s) < 0) + if (restconf_subscription(h, req, method, qvec, pretty, media_out, &s) < 0) goto done; if (s != -1){ #ifdef STREAM_FORK diff --git a/apps/restconf/restconf_stream_native.c b/apps/restconf/restconf_stream_native.c new file mode 100644 index 00000000..26fd2540 --- /dev/null +++ b/apps/restconf/restconf_stream_native.c @@ -0,0 +1,296 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2020-2024 Olof Hagsand and Rubicon Communications, LLC(Netgate) + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + + * nghttp2 callback mechanism + * + * nghttp2_session_mem_recv() + * on_begin_headers_callback() + * create sd + * on_header_callback() NGHTTP2_HEADERS + * translate all headers + * on_data_chunk_recv_callback + * get indata + * on_frame_recv_callback NGHTTP2_FLAG_END_STREAM + * get method and call handler + * create rr + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* cligen */ +#include + +/* clixon */ +#include + +#ifdef HAVE_LIBNGHTTP2 +#include +#endif + +/* restconf */ +#include "restconf_lib.h" /* generic shared with plugins */ +#include "restconf_handle.h" +#include "restconf_api.h" /* generic not shared with plugins */ +#include "restconf_err.h" +#include "restconf_root.h" +#include "restconf_native.h" /* Restconf-openssl mode specific headers*/ +#include "restconf_stream.h" + +#ifdef HAVE_LIBNGHTTP2 /* Ends at end-of-file */ +#include "restconf_nghttp2.h" /* Restconf-openssl mode specific headers*/ + +/*! Callback when stream notifications arrive from backend + * + * @param[in] s Socket + * @param[in] req Generic Www handle (can be part of clixon handle) + * @retval 0 OK + * @retval -1 Error + * @see netconf_notification_cb + */ +static int +restconf_native_stream_cb(int s, + void *arg) +{ + int retval = -1; + int eof; + cxobj *xtop = NULL; /* top xml */ + cxobj *xn; /* notification xml */ + cbuf *cb = NULL; + cbuf *cbmsg = NULL; + int pretty = 0; /* XXX should be via arg */ + int ret; + + clixon_debug(CLIXON_DBG_STREAM, ""); + if (clixon_msg_rcv11(s, NULL, 0, &cbmsg, &eof) < 0) + goto done; + clixon_debug(CLIXON_DBG_STREAM, "%s", cbuf_get(cbmsg)); + /* handle close from remote end: this will exit the client */ + if (eof){ + clixon_debug(CLIXON_DBG_STREAM, "eof"); + clixon_err(OE_PROTO, ESHUTDOWN, "Socket unexpected close"); + errno = ESHUTDOWN; + clixon_exit_set(1); + goto done; + } + if ((ret = clixon_xml_parse_string(cbuf_get(cbmsg), YB_NONE, NULL, &xtop, NULL)) < 0) + goto done; + if (ret == 0){ + clixon_err(OE_XML, EFAULT, "Invalid notification"); + goto done; + } + /* create event */ + if ((cb = cbuf_new()) == NULL){ + clixon_err(OE_PLUGIN, errno, "cbuf_new"); + goto done; + } + if ((xn = xpath_first(xtop, NULL, "notification")) == NULL) + goto ok; + if (clixon_xml2cbuf(cb, xn, 0, pretty, NULL, -1, 0) < 0) + goto done; + ok: + retval = 0; + done: + clixon_debug(CLIXON_DBG_STREAM, "retval: %d", retval); + if (xtop != NULL) + xml_free(xtop); + if (cbmsg) + cbuf_free(cbmsg); + if (cb) + cbuf_free(cb); + return retval; +} + +/*! Process a stream request, native variant + * + * @param[in] h Clixon handle + * @param[in] req Generic Www handle (can be part of clixon handle) + * @param[in] qvec Query parameters, ie the ?=&= stuff + * @param[out] finish Not used in native? + * @retval 0 OK + * @retval -1 Error + * @see api_stream fcgi implementation + * @note According to RFC8040 Sec 6 accept-stream is text/event-stream, but stream data + * is XML according to RFC5277. But what is error return? assume XML here + */ +int +api_stream(clixon_handle h, + void *req, + cvec *qvec, + int *finish) +{ + int retval = -1; + char *path = NULL; + char *request_method = NULL; /* GET,.. */ + char *streampath; + int pretty; + char **pvec = NULL; + int pn; + cvec *pcvec = NULL; /* for rest api */ + cxobj *xerr = NULL; + char *media_str = NULL; + char *stream_name; + restconf_media media_stream = TEXT_EVENT_STREAM; /* text/event-stream, see RFC8040 sec 6 */ + restconf_media media_out = YANG_DATA_XML; + int ret; + int backend_socket = -1; + + fprintf(stderr, "%s\n", __FUNCTION__); + clixon_debug(CLIXON_DBG_STREAM, ""); + if (req == NULL){ + errno = EINVAL; + goto done; + } + streampath = clicon_option_str(h, "CLICON_STREAM_PATH"); + if ((path = restconf_uripath(h)) == NULL) + goto done; + clixon_debug(CLIXON_DBG_STREAM, "path:%s", path); + request_method = restconf_param_get(h, "REQUEST_METHOD"); + clixon_debug(CLIXON_DBG_STREAM, "method:%s", request_method); + pretty = restconf_pretty_get(h); + clixon_debug(CLIXON_DBG_STREAM, "pretty:%d", pretty); + /* Get media for output (proactive negotiation) RFC7231 by using + * Accept:. This is for methods that have output, such as GET, + * operation POST, etc + * If accept is * default is yang-json + */ + media_str = restconf_param_get(h, "HTTP_ACCEPT"); + clixon_debug(CLIXON_DBG_STREAM, "accept(media):%s", media_str); + if (media_str == NULL){ + if (restconf_not_acceptable(h, req, pretty, media_out) < 0) + goto done; + goto ok; + } + media_stream = restconf_media_str2int(media_str); + clixon_debug(CLIXON_DBG_STREAM, "media_out:%s", restconf_media_int2str(media_stream)); + switch ((int)media_stream){ + case -1: + if (strcmp(media_str, "*/*") == 0){ /* catch-all */ + media_out = TEXT_EVENT_STREAM; + } + else{ + if (restconf_not_acceptable(h, req, pretty, media_out) < 0) + goto done; + goto ok; + } + break; + case TEXT_EVENT_STREAM: + break; + default: + if (restconf_not_acceptable(h, req, pretty, media_out) < 0) + goto done; + goto ok; + break; + } + if ((pvec = clicon_strsep(path, "/", &pn)) == NULL) + goto done; + if (strlen(pvec[0]) != 0){ + if (netconf_invalid_value_xml(&xerr, "protocol", "Invalid path, /stream/ expected") < 0) + goto done; + if (api_return_err0(h, req, xerr, pretty, media_out, 0) < 0) + goto done; + goto ok; + } + else if (strcmp(pvec[1], streampath)){ + if (netconf_invalid_value_xml(&xerr, "protocol", "Invalid path, /stream/ expected") < 0) + goto done; + if (api_return_err0(h, req, xerr, pretty, media_out, 0) < 0) + goto done; + goto ok; + } + else if ((stream_name = pvec[2]) == NULL || + strlen(stream_name) == 0){ + if (netconf_invalid_value_xml(&xerr, "protocol", "Invalid path, /stream/ expected") < 0) + goto done; + if (api_return_err0(h, req, xerr, pretty, media_out, 0) < 0) + goto done; + goto ok; + } + clixon_debug(CLIXON_DBG_STREAM, "stream-name: %s", stream_name); + if (uri_str2cvec(path, '/', '=', 1, &pcvec) < 0) /* rest url eg /album=ricky/foo */ + goto done; + /* If present, check credentials. See "plugin_credentials" in plugin + * See RFC 8040 section 2.5 + */ + if ((ret = restconf_authentication_cb(h, req, pretty, media_out)) < 0) + goto done; + if (ret == 0) + goto ok; + clixon_debug(CLIXON_DBG_STREAM, "passed auth"); + if (restconf_subscription(h, req, stream_name, qvec, pretty, media_out, &backend_socket) < 0) + goto done; + if (backend_socket != -1){ + // XXX Could add forking here eventurally + /* Listen to backend socket */ + if (clixon_event_reg_fd(backend_socket, + restconf_native_stream_cb, + req, + "stream socket") < 0) + goto done; + } + ok: + retval = 0; + done: + fprintf(stderr, "%s retval %d\n", __FUNCTION__, retval); + clixon_debug(CLIXON_DBG_STREAM, "retval:%d", retval); + if (xerr) + xml_free(xerr); + if (path) + free(path); + if (pvec) + free(pvec); + if (pcvec) + cvec_free(pcvec); + return retval; +} + +#endif /* HAVE_LIBNGHTTP2 */ diff --git a/include/clixon_custom.h b/include/clixon_custom.h index dcf970da..79998d47 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -211,3 +211,8 @@ * Digest use is not cryptographic use, so SHA1 is enough for now */ #undef USE_SHA256 + +/*! Restconf native stream support + */ +#define RESTCONF_NATIVE_STREAM + diff --git a/test/test_restconf_notifications.sh b/test/test_restconf_notifications.sh index 2556aa9e..09376f20 100755 --- a/test/test_restconf_notifications.sh +++ b/test/test_restconf_notifications.sh @@ -28,7 +28,8 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi # Skip it other than fcgi and http -if [ "${WITH_RESTCONF}" != "fcgi" -o "$RCPROTO" = https ]; then +#if [ "${WITH_RESTCONF}" != "fcgi" -o "$RCPROTO" = https ]; then +if false; then rm -rf $dir if [ "$s" = $0 ]; then exit 0; else return 0; fi # skip fi @@ -41,8 +42,8 @@ if [ $valgrindtest -ne 0 ]; then fi # Degraded does not work at all -rm -rf $dir -if [ "$s" = $0 ]; then exit 0; else return 0; fi # skip +#rm -rf $dir +#if [ "$s" = $0 ]; then exit 0; else return 0; fi # skip : ${SLEEP2:=1} SLEEP5=.5 @@ -77,7 +78,7 @@ cat < $cfg true true streams - https://localhost + $RCPROTO://localhost 60 $RESTCONFIG @@ -89,42 +90,42 @@ EOF # RFC5277 NETCONF Event Notifications # using reportingEntity (rfc5277) not reporting-entity (rfc8040) cat < $fyang - module example { - namespace "urn:example:clixon"; - prefix ex; - organization "Example, Inc."; - contact "support at example.com"; - description "Example Notification Data Model Module."; - revision "2016-07-07" { - description "Initial version."; - reference "example.com document 2-9976."; - } - notification event { - description "Example notification event."; - leaf event-class { - type string; - description "Event class identifier."; - } - container reportingEntity { - description "Event specific information."; - leaf card { - type string; - description "Line card identifier."; - } - } - leaf severity { - type string; - description "Event severity description."; - } - } - container state { - config false; - description "state data for the example application (must be here for example get operation)"; - leaf-list op { - type string; - } - } +module example { + namespace "urn:example:clixon"; + prefix ex; + organization "Example, Inc."; + contact "support at example.com"; + description "Example Notification Data Model Module."; + revision "2016-07-07" { + description "Initial version."; + reference "example.com document 2-9976."; } + notification event { + description "Example notification event."; + leaf event-class { + type string; + description "Event class identifier."; + } + container reportingEntity { + description "Event specific information."; + leaf card { + type string; + description "Line card identifier."; + } + } + leaf severity { + type string; + description "Event severity description."; + } + } + container state { + config false; + description "state data for the example application (must be here for example get operation)"; + leaf-list op { + type string; + } + } +} EOF # Temporary pause between tests to make state timeout @@ -134,7 +135,6 @@ function test-pause() sleep 5 # -m 1 means 1 sec timeout curl -Ssik --http1.1 -X GET -m 1 -H "Accept: text/event-stream" -H "Cache-Control: no-cache" -H "Connection: keep-alive" "http://localhost/streams/EXAMPLE" 2>&1 > /dev/null - } new "test params: -f $cfg" @@ -166,7 +166,6 @@ wait_restconf new "netconf event stream discovery RFC8040 Sec 6.2" expecteof_netconf "$clixon_netconf -D $DBG -qf $cfg" 0 "$DEFAULTHELLO" "" "" "EXAMPLEExample event streamtruexmlhttps://localhost/streams/EXAMPLE" -# # 1.2 Netconf stream subscription # 2. Restconf RFC8040 stream testing @@ -185,10 +184,15 @@ sleep $SLEEP2 new "restconf monitor event nonexist stream" # Note cant use -S or -i here, the former dont know, latter because expectwait cant take # partial returns like expectpart can -expectwait "curl -sk -X GET -H \"Accept: text/event-stream\" -H \"Cache-Control: no-cache\" -H \"Connection: keep-alive\" $RCPROTO://localhost/streams/NOTEXIST" 0 "" "" 2 'applicationinvalid-valueerrorNo such stream' +expectpart "$(curl $CURLOPTS -X GET -H "Accept: text/event-stream" -H "Cache-Control: no-cache" -H "Connection: keep-alive" $RCPROTO://localhost/streams/NOTEXIST)" 0 "HTTP/$HVER 400" 'applicationinvalid-valueerrorNo such stream' # 2a) start subscription 8s - expect 1-2 notifications + new "2a) start subscriptions 8s - expect 1-2 notifications" +echo "curl $CURLOPTS --no-buffer -X GET -H \"Accept: text/event-stream\" -H \"Cache-Control: no-cache\" -H \"Connection: keep-alive\" $RCPROTO://localhost/streams/EXAMPLE" +#curl $CURLOPTS --no-buffer -X GET -H "Accept: text/event-stream" -H "Cache-Control: no-cache" -H "Connection: keep-alive" $RCPROTO://localhost/streams/EXAMPLE + +exit ret=$($clixon_util_stream -u $RCPROTO://localhost/streams/EXAMPLE -t 8) expect="data: ${DATE}T[0-9:.]*ZfaultEthernet0major" @@ -201,7 +205,7 @@ nr=$(echo "$ret" | grep -c "data:") if [ $nr -lt 1 -o $nr -gt 2 ]; then err 2 "$nr" fi - +exit test-pause # 2b) start subscription 8s - stoptime after 5s - expect 1-2 notifications