experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Stream debug and tests

Browse source
This commit is contained in:
Olof hagsand 2018-10-30 22:28:24 +01:00
parent e4adec413a
commit fa9b9c7e2e
8 changed files with 295 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

35
apps/backend/backend_client.c
View file

@ -83,29 +83,32 @@ ce_find_bypid(struct client_entry *ce_list,
/*! Stream callback for netconf stream notification (RFC 5277) /*! Stream callback for netconf stream notification (RFC 5277)
* @param[in] h Clicon handle * @param[in] h Clicon handle
* @param[in] op 0:event, 1:rm
* @param[in] event Event as XML * @param[in] event Event as XML
* @param[in] arg Extra argument provided in stream_ss_add * @param[in] arg Extra argument provided in stream_ss_add
* @see stream_ss_add * @see stream_ss_add
*/ */
static int static int
ce_event_cb(clicon_handle h, ce_event_cb(clicon_handle h,
int op,
cxobj *event, cxobj *event,
void *arg) void *arg)
{ {
struct client_entry *ce = (struct client_entry *)arg; struct client_entry *ce = (struct client_entry *)arg;
if (send_msg_notify_xml(ce->ce_s, event) < 0){ clicon_debug(1, "%s op:%d", __FUNCTION__, op);
if (errno == ECONNRESET || errno == EPIPE){ switch (op){
clicon_log(LOG_WARNING, "client %d reset", ce->ce_nr); case 1:
#if 0 /* Risk of recursion here */
/* We should remove here but removal is not possible if (ce->ce_s)
from a client since backend_client is not linked.
Maybe we should add it to the plugin, but it feels
"safe" that you cant remove a client.
Instead, the client is (hopefully) removed elsewhere?
*/
backend_client_rm(h, ce); backend_client_rm(h, ce);
#endif break;
default:
if (send_msg_notify_xml(ce->ce_s, event) < 0){
if (errno == ECONNRESET || errno == EPIPE){
clicon_log(LOG_WARNING, "client %d reset", ce->ce_nr);
}
break;
} }
} }
return 0; return 0;
@ -126,6 +129,7 @@ backend_client_rm(clicon_handle h,
struct client_entry *c0; struct client_entry *c0;
struct client_entry **ce_prev; struct client_entry **ce_prev;
clicon_debug(1, "%s", __FUNCTION__);
c0 = backend_client_list(h); c0 = backend_client_list(h);
ce_prev = &c0; /* this points to stack and is not real backpointer */ ce_prev = &c0; /* this points to stack and is not real backpointer */
for (c = *ce_prev; c; c = c->ce_next){ for (c = *ce_prev; c; c = c->ce_next){
@ -842,20 +846,19 @@ from_client_create_subscription(clicon_handle h,
if ((x = xpath_first(xe, "//stream")) != NULL) if ((x = xpath_first(xe, "//stream")) != NULL)
stream = xml_find_value(x, "body"); stream = xml_find_value(x, "body");
if ((x = xpath_first(xe, "//stopTime")) != NULL){ if ((x = xpath_first(xe, "//stopTime")) != NULL){
stoptime = xml_find_value(x, "body"); if ((stoptime = xml_find_value(x, "body")) != NULL &&
if (str2time(stoptime, &stop) < 0){ str2time(stoptime, &stop) < 0){
if (netconf_bad_element(cbret, "application", "<bad-element>stopTime</bad-element>", "Expected timestamp") < 0) if (netconf_bad_element(cbret, "application", "<bad-element>stopTime</bad-element>", "Expected timestamp") < 0)
goto done; goto done;
goto ok; goto ok;
} }
} }
if ((x = xpath_first(xe, "//startTime")) != NULL){ if ((x = xpath_first(xe, "//startTime")) != NULL){
starttime = xml_find_value(x, "body"); if ((starttime = xml_find_value(x, "body")) != NULL &&
if (str2time(starttime, &start) < 0){ str2time(starttime, &start) < 0){
if (netconf_bad_element(cbret, "application", "<bad-element>startTime</bad-element>", "Expected timestamp") < 0) if (netconf_bad_element(cbret, "application", "<bad-element>startTime</bad-element>", "Expected timestamp") < 0)
goto done; goto done;
goto ok; goto ok;
goto done;
} }
} }
if ((xfilter = xpath_first(xe, "//filter")) != NULL){ if ((xfilter = xpath_first(xe, "//filter")) != NULL){

4
apps/backend/backend_main.c
View file

@ -133,7 +133,7 @@ usage(clicon_handle h,
char *confpid = clicon_backend_pidfile(h); char *confpid = clicon_backend_pidfile(h);
char *group = clicon_sock_group(h); char *group = clicon_sock_group(h);
fprintf(stderr, "usage:%s\n" fprintf(stderr, "usage:%s <options>*\n"
"where options are\n" "where options are\n"
"\t-h\t\tHelp\n" "\t-h\t\tHelp\n"
"\t-D <level>\tDebug level\n" "\t-D <level>\tDebug level\n"
@ -860,6 +860,8 @@ main(int argc,
if (debug) if (debug)
clicon_option_dump(h, debug); clicon_option_dump(h, debug);
if (stream_timer_setup(0, h) < 0)
goto done;
if (event_loop() < 0) if (event_loop() < 0)
goto done; goto done;
retval = 0; retval = 0;

4
apps/restconf/README.md
View file

@ -90,8 +90,6 @@ where
You access the streams using curl, but they differ slightly in behaviour as described in the following two sections. You access the streams using curl, but they differ slightly in behaviour as described in the following two sections.
### Native event streams
Add the following to extend the nginx configuration file with the following statements: Add the following to extend the nginx configuration file with the following statements:
``` ```
location /streams { location /streams {
@ -111,11 +109,13 @@ where the first command retrieves only new notifications, and the second receive
### Nginx Nchan streams ### Nginx Nchan streams
As an alternative, Nginx/Nchan can be used for streams.
Nginx uses pub/sub channels and can be configured in a variety of Nginx uses pub/sub channels and can be configured in a variety of
ways. The following uses a simple variant with one generic subscription ways. The following uses a simple variant with one generic subscription
channel (streams) and one publication channel (pub). channel (streams) and one publication channel (pub).
Configure clixon with `--enable-publish` which enables curl code for publishing streams to nchan. Configure clixon with `--enable-publish` which enables curl code for publishing streams to nchan.
Set configure option CLICON_STREAM_PUB to, for example, http://localhost/pub to enable pushing notifications to nchan.
Download and install nchan, see (https://nchan.io/#install). Download and install nchan, see (https://nchan.io/#install).

31
apps/restconf/restconf_stream.c
View file

@ -108,14 +108,12 @@ restconf_stream_cb(int s,
clicon_debug(1, "%s msg_rcv error", __FUNCTION__); clicon_debug(1, "%s msg_rcv error", __FUNCTION__);
goto done; goto done;
} }
clicon_debug(1, "%s msg: %s", __FUNCTION__, reply->op_body); clicon_debug(1, "%s msg: %s", __FUNCTION__, reply?reply->op_body:"null");
/* handle close from remote end: this will exit the client */ /* handle close from remote end: this will exit the client */
if (eof){ if (eof){
clicon_debug(1, "%s eof", __FUNCTION__); clicon_debug(1, "%s eof", __FUNCTION__);
clicon_err(OE_PROTO, ESHUTDOWN, "Socket unexpected close"); clicon_err(OE_PROTO, ESHUTDOWN, "Socket unexpected close");
close(s);
errno = ESHUTDOWN; errno = ESHUTDOWN;
event_unreg_fd(s, restconf_stream_cb);
FCGX_FPrintF(r->out, "SHUTDOWN\r\n"); FCGX_FPrintF(r->out, "SHUTDOWN\r\n");
FCGX_FPrintF(r->out, "\r\n"); FCGX_FPrintF(r->out, "\r\n");
FCGX_FFlush(r->out); FCGX_FFlush(r->out);
@ -148,7 +146,6 @@ restconf_stream_cb(int s,
FCGX_FPrintF(r->out, "data: %s\r\n", cbuf_get(cb)); FCGX_FPrintF(r->out, "data: %s\r\n", cbuf_get(cb));
FCGX_FPrintF(r->out, "\r\n"); FCGX_FPrintF(r->out, "\r\n");
FCGX_FFlush(r->out); FCGX_FFlush(r->out);
ok: ok:
retval = 0; retval = 0;
done: done:
@ -186,8 +183,8 @@ restconf_stream(clicon_handle h,
cg_var *cv; cg_var *cv;
char *vname; char *vname;
*sp = -1;
clicon_debug(1, "%s", __FUNCTION__); clicon_debug(1, "%s", __FUNCTION__);
*sp = -1;
if ((cb = cbuf_new()) == NULL){ if ((cb = cbuf_new()) == NULL){
clicon_err(OE_XML, errno, "cbuf_new"); clicon_err(OE_XML, errno, "cbuf_new");
goto done; goto done;
@ -228,6 +225,7 @@ restconf_stream(clicon_handle h,
ok: ok:
retval = 0; retval = 0;
done: done:
clicon_debug(1, "%s retval: %d", __FUNCTION__, retval);
if (xret) if (xret)
xml_free(xret); xml_free(xret);
if (cb) if (cb)
@ -239,6 +237,19 @@ restconf_stream(clicon_handle h,
#include "restconf_lib.h" #include "restconf_lib.h"
#include "restconf_stream.h" #include "restconf_stream.h"
static int
stream_checkuplink(int s,
void *arg)
{
FCGX_Request *r = (FCGX_Request *)arg;
clicon_debug(1, "%s", __FUNCTION__);
if (FCGX_GetError(r->out) != 0){ /* break loop */
clicon_debug(1, "%s FCGX_GetError upstream", __FUNCTION__);
clicon_exit_set();
}
return 0;
}
int int
stream_timeout(int s, stream_timeout(int s,
void *arg) void *arg)
@ -248,8 +259,10 @@ stream_timeout(int s,
FCGX_Request *r = (FCGX_Request *)arg; FCGX_Request *r = (FCGX_Request *)arg;
clicon_debug(1, "%s", __FUNCTION__); clicon_debug(1, "%s", __FUNCTION__);
if (FCGX_GetError(r->out) != 0) /* break loop */ if (FCGX_GetError(r->out) != 0){ /* break loop */
clicon_debug(1, "%s FCGX_GetError upstream", __FUNCTION__);
clicon_exit_set(); clicon_exit_set();
}
else{ else{
gettimeofday(&t, NULL); gettimeofday(&t, NULL);
t1.tv_sec = 1; t1.tv_usec = 0; t1.tv_sec = 1; t1.tv_usec = 0;
@ -355,10 +368,16 @@ api_stream(clicon_handle h,
(void*)r, (void*)r,
"stream socket") < 0) "stream socket") < 0)
goto done; goto done;
if (event_reg_fd(r->listen_sock,
stream_checkuplink,
(void*)r,
"stream socket") < 0)
goto done;
/* Poll upstream errors */ /* Poll upstream errors */
stream_timeout(0, (void*)r); stream_timeout(0, (void*)r);
/* Start loop */ /* Start loop */
event_loop(); event_loop();
close(s);
event_unreg_fd(s, restconf_stream_cb); event_unreg_fd(s, restconf_stream_cb);
clicon_exit_reset(); clicon_exit_reset();
} }

5
lib/clixon/clixon_stream.h
View file

@ -41,11 +41,12 @@
*/ */
/* Subscription callback /* Subscription callback
* @param[in] h Clicon handle * @param[in] h Clicon handle
* @param[in] op Operation: 0 OK, 1 Close
* @param[in] event Event as XML * @param[in] event Event as XML
* @param[in] arg Extra argument provided in stream_ss_add * @param[in] arg Extra argument provided in stream_ss_add
* @see stream_ss_add * @see stream_ss_add
*/ */
typedef int (*stream_fn_t)(clicon_handle h, cxobj *event, void *arg); typedef int (*stream_fn_t)(clicon_handle h, int op, cxobj *event, void *arg);
struct stream_subscription{ struct stream_subscription{
qelem_t ss_q; /* queue header */ qelem_t ss_q; /* queue header */
@ -90,7 +91,7 @@ int stream_timer_setup(int fd, void *arg);
struct stream_subscription *stream_ss_add(clicon_handle h, char *stream, struct stream_subscription *stream_ss_add(clicon_handle h, char *stream,
char *xpath, struct timeval *start, struct timeval *stop, char *xpath, struct timeval *start, struct timeval *stop,
stream_fn_t fn, void *arg); stream_fn_t fn, void *arg);
int stream_ss_rm(event_stream_t *es, struct stream_subscription *ss); int stream_ss_rm(clicon_handle h, event_stream_t *es, struct stream_subscription *ss);
struct stream_subscription *stream_ss_find(event_stream_t *es, struct stream_subscription *stream_ss_find(event_stream_t *es,
stream_fn_t fn, void *arg); stream_fn_t fn, void *arg);
int stream_ss_delete_all(clicon_handle h, stream_fn_t fn, void *arg); int stream_ss_delete_all(clicon_handle h, stream_fn_t fn, void *arg);

36
lib/src/clixon_stream.c
View file

@ -164,7 +164,7 @@ stream_delete_all(clicon_handle h)
if (es->es_description) if (es->es_description)
free(es->es_description); free(es->es_description);
while ((ss = es->es_subscription) != NULL) while ((ss = es->es_subscription) != NULL)
stream_ss_rm(es, ss); stream_ss_rm(h, es, ss);
while ((r = es->es_replay) != NULL){ while ((r = es->es_replay) != NULL){
DELQ(r, es->es_replay, struct stream_replay *); DELQ(r, es->es_replay, struct stream_replay *);
if (r->r_xml) if (r->r_xml)
@ -239,6 +239,7 @@ stream_timer_setup(int fd,
struct stream_replay *r; struct stream_replay *r;
struct stream_replay *r1; struct stream_replay *r1;
clicon_debug(2, "%s", __FUNCTION__);
/* Go thru callbacks and see if any have timed out, if so remove them /* Go thru callbacks and see if any have timed out, if so remove them
* Could also be done by a separate timer. * Could also be done by a separate timer.
*/ */
@ -255,7 +256,8 @@ stream_timer_setup(int fd,
do { do {
if (timerisset(&ss->ss_stoptime) && timercmp(&ss->ss_stoptime, &now, <)){ if (timerisset(&ss->ss_stoptime) && timercmp(&ss->ss_stoptime, &now, <)){
ss1 = NEXTQ(struct stream_subscription *, ss); ss1 = NEXTQ(struct stream_subscription *, ss);
if (stream_ss_rm(es, ss) < 0) /* Signal to remove stream for upper levels */
if (stream_ss_rm(h, es, ss) < 0)
goto done; goto done;
ss = ss1; ss = ss1;
} }
@ -369,15 +371,20 @@ stream_ss_add(clicon_handle h,
* @retval -1 Error * @retval -1 Error
*/ */
int int
stream_ss_rm(event_stream_t *es, stream_ss_rm(clicon_handle h,
event_stream_t *es,
struct stream_subscription *ss) struct stream_subscription *ss)
{ {
clicon_debug(1, "%s", __FUNCTION__);
DELQ(ss, es->es_subscription, struct stream_subscription *); DELQ(ss, es->es_subscription, struct stream_subscription *);
/* Remove from upper layers - close socket etc. */
(*ss->ss_fn)(h, 1, NULL, ss->ss_arg);
if (ss->ss_stream) if (ss->ss_stream)
free(ss->ss_stream); free(ss->ss_stream);
if (ss->ss_xpath) if (ss->ss_xpath)
free(ss->ss_xpath); free(ss->ss_xpath);
free(ss); free(ss);
return 0; return 0;
} }
@ -421,9 +428,10 @@ stream_ss_delete_all(clicon_handle h,
if ((es = clicon_stream(h)) != NULL){ if ((es = clicon_stream(h)) != NULL){
do { do {
if ((ss = stream_ss_find(es, fn, arg)) != NULL) if ((ss = stream_ss_find(es, fn, arg)) != NULL){
if (stream_ss_rm(es, ss) < 0) if (stream_ss_rm(h, es, ss) < 0)
goto done; goto done;
}
es = NEXTQ(struct event_stream *, es); es = NEXTQ(struct event_stream *, es);
} while (es && es != clicon_stream(h)); } while (es && es != clicon_stream(h));
} }
@ -451,18 +459,24 @@ stream_notify_xml(clicon_handle h,
int retval = -1; int retval = -1;
struct stream_subscription *ss; struct stream_subscription *ss;
clicon_debug(1, "%s", __FUNCTION__); clicon_debug(2, "%s", __FUNCTION__);
/* Go thru all subscriptions and find matches */ /* Go thru all subscriptions and find matches */
if ((ss = es->es_subscription) != NULL) if ((ss = es->es_subscription) != NULL)
do { do {
if (timerisset(&ss->ss_stoptime) && /* stoptime has passed */ if (timerisset(&ss->ss_stoptime) && /* stoptime has passed */
timercmp(&ss->ss_stoptime, tv, <)) timercmp(&ss->ss_stoptime, tv, <)){
; struct stream_subscription *ss1;
ss1 = NEXTQ(struct stream_subscription *, ss);
/* Signal to remove stream for upper levels */
if (stream_ss_rm(h, es, ss) < 0)
goto done;
ss = ss1;
}
else /* xpath match */ else /* xpath match */
if (ss->ss_xpath == NULL || if (ss->ss_xpath == NULL ||
strlen(ss->ss_xpath)==0 || strlen(ss->ss_xpath)==0 ||
xpath_first(xevent, "%s", ss->ss_xpath) != NULL) xpath_first(xevent, "%s", ss->ss_xpath) != NULL)
if ((*ss->ss_fn)(h, xevent, ss->ss_arg) < 0) if ((*ss->ss_fn)(h, 0, xevent, ss->ss_arg) < 0)
goto done; goto done;
ss = NEXTQ(struct stream_subscription *, ss); ss = NEXTQ(struct stream_subscription *, ss);
} while (ss && ss != es->es_subscription); } while (ss && ss != es->es_subscription);
@ -499,7 +513,7 @@ stream_notify(clicon_handle h,
struct timeval tv; struct timeval tv;
event_stream_t *es; event_stream_t *es;
clicon_debug(1, "%s", __FUNCTION__); clicon_debug(2, "%s", __FUNCTION__);
if ((es = stream_find(h, stream)) == NULL) if ((es = stream_find(h, stream)) == NULL)
goto ok; goto ok;
va_start(args, event); va_start(args, event);
@ -604,7 +618,7 @@ stream_replay_notify(clicon_handle h,
if (timerisset(&ss->ss_stoptime) && if (timerisset(&ss->ss_stoptime) &&
timercmp(&r->r_tv, &ss->ss_stoptime, >)) timercmp(&r->r_tv, &ss->ss_stoptime, >))
break; break;
if ((*ss->ss_fn)(h, r->r_xml, ss->ss_arg) < 0) if ((*ss->ss_fn)(h, 0, r->r_xml, ss->ss_arg) < 0)
goto done; goto done;
r = NEXTQ(struct stream_replay *, r); r = NEXTQ(struct stream_replay *, r);
} while (r && r!=es->es_replay); } while (r && r!=es->es_replay);

168
test/test_stream.sh
View file

@ -4,9 +4,24 @@
# 1. http server setup, such as nginx described in apps/restconf/README.md # 1. http server setup, such as nginx described in apps/restconf/README.md
# especially SSE - ngchan setup # especially SSE - ngchan setup
# 2. Example stream as Clixon example which needs registration, callback and # 2. Example stream as Clixon example which needs registration, callback and
# notification generating code # notification generating code every 5s
#
# Testing of streams is quite complicated.
# Here are some testing dimensions in restconf alone:
# - start/stop subscription
# - start-time/stop-time in subscription
# - stream retention time
# - native vs nchan implementation
# Focussing on 1-3
# 2a) start sub 8s - see 2 notifications
# 2b) start sub 8s - stoptime after 5s - see 1 notifications
# 2c) start sub 8s - replay from start -8s - see 4 notifications
# 2d) start sub 8s - replay from start -8s to stop +4s - see 3 notifications
# 2e) start sub 8s - replay from -90s w retention 60s - see 10 notifications
APPNAME=example APPNAME=example
UTIL=../util/clixon_util_stream
DATE=$(date +"%Y-%m-%d")
# include err() and new() functions and creates $dir # include err() and new() functions and creates $dir
. ./lib.sh . ./lib.sh
cfg=$dir/conf.xml cfg=$dir/conf.xml
@ -33,6 +48,7 @@ cat <<EOF > $cfg
<CLICON_STREAM_DISCOVERY_RFC8040>true</CLICON_STREAM_DISCOVERY_RFC8040> <CLICON_STREAM_DISCOVERY_RFC8040>true</CLICON_STREAM_DISCOVERY_RFC8040>
<CLICON_STREAM_PATH>streams</CLICON_STREAM_PATH> <CLICON_STREAM_PATH>streams</CLICON_STREAM_PATH>
<CLICON_STREAM_URL>https://localhost</CLICON_STREAM_URL> <CLICON_STREAM_URL>https://localhost</CLICON_STREAM_URL>
<CLICON_STREAM_RETENTION>60</CLICON_STREAM_RETENTION>
</config> </config>
EOF EOF
@ -84,34 +100,131 @@ if [ $? -ne 0 ]; then
err err
fi fi
new "start backend -s init -f $cfg -y $fyang" new "start backend -s init -f $cfg -y $fyang"
sudo $clixon_backend -s init -f $cfg -y $fyang # -D 1 sudo $clixon_backend -s init -f $cfg -y $fyang -D 1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
err err
fi fi
new "kill old restconf daemon" new "kill old restconf daemon"
sudo pkill -u www-data clixon_restconf #sudo pkill -u www-data clixon_restconf
new "start restconf daemon" new "start restconf daemon"
sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -f $cfg -y $fyang # -D 1 #sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -f $cfg -y $fyang -D 1
sleep 2
#
# 1. Netconf RFC5277 stream testing
new "1. Netconf RFC5277 stream testing"
# 1.1 Stream discovery
new "netconf event stream discovery RFC5277 Sec 3.2.5" new "netconf event stream discovery RFC5277 Sec 3.2.5"
expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 '<rpc><get><filter type="xpath" select="netconf/streams" xmlns="urn:ietf:params:xml:ns:netmod:notification"/></get></rpc>]]>]]>' '<rpc-reply><data><netconf><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>false</replay-support></stream></streams></netconf></data></rpc-reply>]]>]]>' expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 '<rpc><get><filter type="xpath" select="netconf/streams" xmlns="urn:ietf:params:xml:ns:netmod:notification"/></get></rpc>]]>]]>' '<rpc-reply><data><netconf><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>true</replay-support></stream></streams></netconf></data></rpc-reply>]]>]]>'
new "netconf event stream discovery RFC8040 Sec 6.2" new "netconf event stream discovery RFC8040 Sec 6.2"
expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 '<rpc><get><filter type="xpath" select="restconf-state/streams" xmlns="urn:ietf:params:xml:ns:netmod:notification"/></get></rpc>]]>]]>' '<rpc-reply><data><restconf-state><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>false</replay-support><access><encoding>xml</encoding><location>https://localhost/streams/EXAMPLE</location></access></stream></streams></restconf-state></data></rpc-reply>]]>]]>' expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 '<rpc><get><filter type="xpath" select="restconf-state/streams" xmlns="urn:ietf:params:xml:ns:netmod:notification"/></get></rpc>]]>]]>' '<rpc-reply><data><restconf-state><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>true</replay-support><access><encoding>xml</encoding><location>https://localhost/streams/EXAMPLE</location></access></stream></streams></restconf-state></data></rpc-reply>]]>]]>'
#
# 1.2 Netconf stream subscription
new "netconf EXAMPLE subscription"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>EXAMPLE</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf subscription with empty startTime"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>EXAMPLE</stream><startTime/></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf EXAMPLE subscription with simple filter"
expectwait "$clixon_netconf -qf $cfg -y $fyang" "<rpc><create-subscription><stream>EXAMPLE</stream><filter type=\"xpath\" select=\"event\"/></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf EXAMPLE subscription with filter classifier"
expectwait "$clixon_netconf -qf $cfg -y $fyang" "<rpc><create-subscription><stream>EXAMPLE</stream><filter type=\"xpath\" select=\"event[event-class='fault']\"/></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf NONEXIST subscription"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>NONEXIST</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-tag>invalid-value</error-tag><error-type>application</error-type><error-severity>error</error-severity><error-message>No such stream</error-message></rpc-error></rpc-reply>]]>]]>$' 5
new "netconf EXAMPLE subscription with wrong date"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>EXAMPLE</stream><startTime>kallekaka</startTime></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-tag>bad-element</error-tag><error-type>application</error-type><error-info><bad-element>startTime</bad-element></error-info><error-severity>error</error-severity><error-message>Expected timestamp</error-message></rpc-error></rpc-reply>]]>]]>$' 0
#new "netconf EXAMPLE subscription with replay"
#NOW=$(date +"%Y-%m-%dT%H:%M:%S")
#sleep 10
#expectwait "$clixon_netconf -qf $cfg -y $fyang" "<rpc><create-subscription><stream>EXAMPLE</stream><startTime>$NOW</startTime></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 10
#
# 2. Restconf RFC8040 stream testing
new "2. Restconf RFC8040 stream testing"
# 2.1 Stream discovery
new "restconf event stream discovery RFC8040 Sec 6.2" new "restconf event stream discovery RFC8040 Sec 6.2"
expectfn "curl -s -X GET http://localhost/restconf/data/ietf-restconf-monitoring:restconf-state/streams" 0 '{"streams": {"stream": \[{"name": "EXAMPLE","description": "Example event stream","replay-support": false,"access": \[{"encoding": "xml","location": "https://localhost/streams/EXAMPLE"}\]}\]}' expectfn "curl -s -X GET http://localhost/restconf/data/ietf-restconf-monitoring:restconf-state/streams" 0 '{"streams": {"stream": \[{"name": "EXAMPLE","description": "Example event stream","replay-support": true,"access": \[{"encoding": "xml","location": "https://localhost/streams/EXAMPLE"}\]}\]}'
new "restconf subscribe RFC8040 Sec 6.3, get location" new "restconf subscribe RFC8040 Sec 6.3, get location"
expectfn "curl -s -X GET http://localhost/restconf/data/ietf-restconf-monitoring:restconf-state/streams/stream=EXAMPLE/access=xml/location" 0 '{"location": "https://localhost/streams/EXAMPLE"}' expectfn "curl -s -X GET http://localhost/restconf/data/ietf-restconf-monitoring:restconf-state/streams/stream=EXAMPLE/access=xml/location" 0 '{"location": "https://localhost/streams/EXAMPLE"}'
# Restconf stream subscription RFC8040 Sec 6.3 - Native solution # Restconf stream subscription RFC8040 Sec 6.3
# Start Subscription w error
new "restconf monitor event nonexist stream" new "restconf monitor event nonexist stream"
expectwait 'curl -s -X GET -H "Accept: text/event-stream" -H "Cache-Control: no-cache" -H "Connection: keep-alive" http://localhost/streams/NOTEXIST' 0 '<errors xmlns="urn:ietf:params:xml:ns:yang:ietf-restconf"><error><error-tag>invalid-value</error-tag><error-type>application</error-type><error-severity>error</error-severity><error-message>No such stream</error-message></error></errors>' 2 expectwait 'curl -s -X GET -H "Accept: text/event-stream" -H "Cache-Control: no-cache" -H "Connection: keep-alive" http://localhost/streams/NOTEXIST' 0 '<errors xmlns="urn:ietf:params:xml:ns:yang:ietf-restconf"><error><error-tag>invalid-value</error-tag><error-type>application</error-type><error-severity>error</error-severity><error-message>No such stream</error-message></error></errors>' 2
# 2a) start subscription 8s - see 2 notifications
new "2a) start subscriptions 8s - see 2 notifications"
ret=$($UTIL -u http://localhost/streams/EXAMPLE -t 8)
expect="data: <notification xmlns=\"urn:ietf:params:xml:ns:netconf:notification:1.0\"><eventTime>${DATE}T[0-9:.]*</eventTime><event><event-class>fault</event-class><reportingEntity><card>Ethernet0</card></reportingEntity><severity>major</severity></event>"
match=$(echo "$ret" | grep -Eo "$expect")
if [ -z "$match" ]; then
err "$expect" "$ret"
fi
nr=$(echo "$ret" | grep -c "data:")
if [ $nr != 1 -a $nr != 2 ]; then
err 2 "$nr"
fi
# 2b) start subscription 8s - stoptime after 5s - see 1 notifications
new "2b) start subscriptions 8s - stoptime after 5s - see 1 notifications"
ret=$($UTIL -u http://localhost/streams/EXAMPLE -t 8 -e +10)
expect="data: <notification xmlns=\"urn:ietf:params:xml:ns:netconf:notification:1.0\"><eventTime>${DATE}T[0-9:.]*</eventTime><event><event-class>fault</event-class><reportingEntity><card>Ethernet0</card></reportingEntity><severity>major</severity></event>"
match=$(echo "$ret" | grep -Eo "$expect")
if [ -z "$match" ]; then
err "$expect" "$ret"
fi
nr=$(echo "$ret" | grep -c "data:")
if [ $nr != 1 -a $nr != 2 ]; then
err 1 "$nr"
fi
# 2c
new "2c) start sub 8s - replay from start -8s - see 4 notifications"
ret=$($UTIL -u http://localhost/streams/EXAMPLE -t 10 -s -8)
expect="data: <notification xmlns=\"urn:ietf:params:xml:ns:netconf:notification:1.0\"><eventTime>${DATE}T[0-9:.]*</eventTime><event><event-class>fault</event-class><reportingEntity><card>Ethernet0</card></reportingEntity><severity>major</severity></event>"
match=$(echo "$ret" | grep -Eo "$expect")
if [ -z "$match" ]; then
err "$expect" "$ret"
fi
nr=$(echo "$ret" | grep -c "data:")
if [ $nr != 4 ]; then
err 4 "$nr"
fi
exit
#-----------------
sudo pkill -u www-data clixon_restconf
new "Kill backend"
# kill backend
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err "kill backend"
fi
# Check if still alive
pid=`pgrep clixon_backend`
if [ -n "$pid" ]; then
sudo kill $pid
fi
rm -rf $dir
exit
#--------------------------------------------------------------------
# Need manual testing # Need manual testing
new "restconf monitor streams native NEEDS manual testing" new "restconf monitor streams native NEEDS manual testing"
if false; then if false; then
@ -138,45 +251,6 @@ if false; then
# Expect: # Expect:
echo foo echo foo
fi fi
# Netconf stream subscription
# Switch here since subscriptions takes time
if true; then
new "netconf EXAMPLE subscription"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>EXAMPLE</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf EXAMPLE subscription with simple filter"
expectwait "$clixon_netconf -qf $cfg -y $fyang" "<rpc><create-subscription><stream>EXAMPLE</stream><filter type=\"xpath\" select=\"event\"/></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf EXAMPLE subscription with filter classifier"
expectwait "$clixon_netconf -qf $cfg -y $fyang" "<rpc><create-subscription><stream>EXAMPLE</stream><filter type=\"xpath\" select=\"event[event-class='fault']\"/></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 5
new "netconf NONEXIST subscription"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>NONEXIST</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-tag>invalid-value</error-tag><error-type>application</error-type><error-severity>error</error-severity><error-message>No such stream</error-message></rpc-error></rpc-reply>]]>]]>$' 5
fi
new "netconf EXAMPLE subscription with wrong date"
expectwait "$clixon_netconf -qf $cfg -y $fyang" '<rpc><create-subscription><stream>EXAMPLE</stream><startTime>kallekaka</startTime></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-tag>bad-element</error-tag><error-type>application</error-type><error-info><bad-element>startTime</bad-element></error-info><error-severity>error</error-severity><error-message>Expected timestamp</error-message></rpc-error></rpc-reply>]]>]]>$' 0
new "netconf EXAMPLE subscription with replay"
NOW=$(date +"%Y-%m-%dT%H:%M:%S")
sleep 10
expectwait "$clixon_netconf -qf $cfg -y $fyang" "<rpc><create-subscription><stream>EXAMPLE</stream><startTime>$NOW</startTime></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 10
new "Kill restconf daemon"
sudo pkill -u www-data clixon_restconf
new "Kill backend"
# kill backend
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err "kill backend"
fi
# Check if still alive
pid=`pgrep clixon_backend`
if [ -n "$pid" ]; then
sudo kill $pid
fi
rm -rf $dir

133
util/clixon_util_stream.c
View file

@ -33,7 +33,7 @@
* Stream restconf support functions. * Stream restconf support functions.
* (Original in grideye) * (Original in grideye)
* Example: clixon_util_stream http://localhost/streams/EXAMPLE 10 * Example: clixon_util_stream -u http://localhost/streams/EXAMPLE -s 2018-10-21T19:22:16
*/ */
#ifdef HAVE_CONFIG_H #ifdef HAVE_CONFIG_H
@ -49,6 +49,7 @@
#include <fnmatch.h> #include <fnmatch.h>
#include <stdint.h> #include <stdint.h>
#include <assert.h> #include <assert.h>
#include <syslog.h>
#include <curl/curl.h> #include <curl/curl.h>
/* cligen */ /* cligen */
@ -93,8 +94,7 @@ curl_get_cb(void *ptr,
* *
* If getdata is set, return the (malloced) data (which should be freed). * If getdata is set, return the (malloced) data (which should be freed).
* *
* @param[in] query 'q' parameter that should be URL-encoded, ie ?q=<encoded> * @param[in] start 'start-time' parameter that will be URL-encoded
* XXX: dont add q=, there may be more parameters.
* @retval -1 fatal error * @retval -1 fatal error
* @retval 1 ok * @retval 1 ok
* *
@ -103,10 +103,11 @@ curl_get_cb(void *ptr,
* better TCP performance * better TCP performance
*/ */
int int
url_get(char *url, stream_url_get(char *url,
char *query, char *start,
int timeout, char *stop,
char **getdata) int timeout,
char **getdata)
{ {
int retval = -1; int retval = -1;
CURL *curl; CURL *curl;
@ -126,30 +127,36 @@ url_get(char *url,
if ((cbf = cbuf_new()) == NULL) if ((cbf = cbuf_new()) == NULL)
goto done; goto done;
if (query){
if ((encoded = curl_easy_escape(curl, query, 0)) == NULL){
clicon_debug(1, "curl_easy_escape");
goto done;
}
}
if ((err = malloc(CURL_ERROR_SIZE)) == NULL) { if ((err = malloc(CURL_ERROR_SIZE)) == NULL) {
clicon_debug(1, "%s: malloc", __FUNCTION__); clicon_debug(1, "%s: malloc", __FUNCTION__);
goto done; goto done;
} }
/* specify URL to get */
if (query)
cprintf(cbf, "%s?q=%s", url, encoded);
else
cprintf(cbf, "%s", url);
curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
/* HEADERS */
list = curl_slist_append(list, "Accept: text/event-stream"); list = curl_slist_append(list, "Accept: text/event-stream");
list = curl_slist_append(list, "Cache-Control: no-cache"); list = curl_slist_append(list, "Cache-Control: no-cache");
list = curl_slist_append(list, "Connection: keep-alive"); list = curl_slist_append(list, "Connection: keep-alive");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
/* specify URL to get */
/* For reference, this url works cprintf(cbf, "%s", url);
"http://192.36.171.239:8086/db/nordunet/series?q=select%20tcmp2%20from%20%22dk-ore%22%20limit%201" if (strlen(start)||strlen(stop))
*/ cprintf(cbf, "?");
if (strlen(start)){
if ((encoded = curl_easy_escape(curl, start, 0)) == NULL)
goto done;
cprintf(cbf, "start-time=%s", encoded);
curl_free(encoded);
encoded = NULL;
}
if (strlen(stop)){
if (strlen(start))
cprintf(cbf, "&");
if ((encoded = curl_easy_escape(curl, stop, 0)) == NULL)
goto done;
cprintf(cbf, "stop-time=%s", encoded);
curl_free(encoded);
encoded = NULL;
}
clicon_debug(1, "url: %s\n", cbuf_get(cbf)); clicon_debug(1, "url: %s\n", cbuf_get(cbf));
curl_easy_setopt(curl, CURLOPT_URL, cbuf_get(cbf)); curl_easy_setopt(curl, CURLOPT_URL, cbuf_get(cbf));
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_get_cb); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_get_cb);
@ -159,9 +166,7 @@ url_get(char *url,
field, so we provide one */ field, so we provide one */
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err); curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err);
// curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
// curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
ret = curl_easy_perform(curl); ret = curl_easy_perform(curl);
if (ret != CURLE_OPERATION_TIMEDOUT && ret != CURLE_OK){ if (ret != CURLE_OPERATION_TIMEDOUT && ret != CURLE_OK){
@ -190,7 +195,16 @@ url_get(char *url,
static int static int
usage(char *argv0) usage(char *argv0)
{ {
fprintf(stderr, "usage:%s <url> <timeout>.\n\tInput on stdin\n", argv0); fprintf(stderr, "usage:%s <options>*\n"
"where options are:\n"
"\t-h\t\tHelp\n"
"\t-D <level>\tDebug level\n"
"\t-u <url>\tURL (mandatory)\n"
"\t-s <start>\tStart-time (format: 2018-10-21T19:22:16 OR +/-<x>s\n"
"\t-e <end>\tStop-time (same format as start)\n"
"\t-t <timeout>\tTimeout (default: 10)\n"
, argv0);
exit(0); exit(0);
} }
@ -198,22 +212,69 @@ int
main(int argc, char **argv) main(int argc, char **argv)
{ {
cbuf *cb = cbuf_new(); cbuf *cb = cbuf_new();
char *url; char *url = NULL;
char *query = NULL;
char *getdata = NULL; char *getdata = NULL;
int timeout; int timeout = 10;
char start[27] = {0,}; /* strlen = 0 */
char stop[27] = {0,};
char c;
char *argv0 = argv[0];
struct timeval now;
if (argc != 3){ clicon_log_init("xpath", LOG_DEBUG, CLICON_LOG_STDERR);
gettimeofday(&now, NULL);
optind = 1;
opterr = 0;
while ((c = getopt(argc, argv, "hDu:s:e:t:")) != -1)
switch (c) {
case 'h':
usage(argv0);
break;
case 'D':
debug++;
break;
case 'u': /* URL */
url = optarg;
break;
case 's': /* start-time */
if (*optarg == '+' || *optarg == '-'){
struct timeval t;
t = now;
t.tv_sec += atoi(optarg);
if (time2str(t, start, sizeof(start)) < 0)
goto done;
}
else
strcpy(start, optarg);
break;
case 'e': /* stop-time */
if (*optarg == '+' || *optarg == '-'){
struct timeval t;
t = now;
t.tv_sec += atoi(optarg);
if (time2str(t, stop, sizeof(stop)) < 0)
goto done;
}
else
strcpy(stop, optarg);
break;
case 't': /* timeout */
timeout = atoi(optarg);
break;
default:
usage(argv[0]);
break;
}
if (url == NULL)
usage(argv[0]); usage(argv[0]);
return 0; curl_global_init(0);
} if (stream_url_get(url, start, stop, timeout, &getdata) < 0)
url = argv[1];
timeout = atoi(argv[2]);
if (url_get(url, query, timeout, &getdata) < 0)
goto done; goto done;
fprintf(stdout, "%s", getdata); if (getdata)
fprintf(stdout, "%s", getdata);
fflush(stdout); fflush(stdout);
done: done:
curl_global_cleanup();
if (getdata) if (getdata)
free(getdata); free(getdata);
if (cb) if (cb)