test stream util function

example/example_backend.c

@@ -304,8 +304,7 @@ clixon_plugin_init(clicon_handle h)
      */
     if (clicon_option_exists(h, "CLICON_STREAM_RETENTION"))
 	retention.tv_sec = clicon_option_int(h, "CLICON_STREAM_RETENTION");
-    if (stream_add(h, "EXAMPLE", "Example event stream", 1,
-		   retention.tv_sec?&retention:NULL) < 0)
+    if (stream_add(h, "EXAMPLE", "Example event stream", 1, &retention) < 0)
 	goto done;
     /* assumes: CLIXON_PUBLISH_STREAMS, eg configure --enable-publish
      */

lib/src/clixon_stream.c

@@ -199,7 +199,8 @@ stream_get_xml(clicon_handle h,
 	    cprintf(cb, "<name>%s</name>", es->es_name);
 	    if (es->es_description)
 		cprintf(cb, "<description>%s</description>", es->es_description);
-	    cprintf(cb, "<replay-support>false</replay-support>");
+	    cprintf(cb, "<replay-support>%s</replay-support>",
+		    es->es_replay_enabled?"true":"false");
 	    if (access){
 		cprintf(cb, "<access>");
 		cprintf(cb, "<encoding>xml</encoding>");

util/Makefile.in

@@ -70,6 +70,7 @@ APPSRC    = clixon_util_xml.c
 APPSRC   += clixon_util_json.c
 APPSRC   += clixon_util_yang.c
 APPSRC   += clixon_util_xpath.c
+# APPSRC   += clixon_util_stream.c # Needs curl
 
 APPS	  = $(APPSRC:.c=)
 
@@ -91,6 +92,9 @@ clixon_util_yang: clixon_util_yang.c $(MYLIB)
 clixon_util_xpath: clixon_util_xpath.c $(MYLIB)
 	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@
 
+clixon_util_stream: clixon_util_stream.c $(MYLIB)
+	$(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -lcurl -o $@
+
 distclean: clean
 	rm -f Makefile *~ .depend
 

util/clixon_util_stream.c

@@ -0,0 +1,224 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2018 Olof Hagsand and Benny Holmgren
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+ * Stream restconf support functions.
+ * (Original in grideye)
+ * Example: clixon_util_stream http://localhost/streams/EXAMPLE 10
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "clixon_config.h" /* generated by config & autoconf */
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <limits.h>
+#include <fnmatch.h>
+#include <stdint.h>
+#include <assert.h>
+#include <curl/curl.h>
+
+/* cligen */
+#include <cligen/cligen.h>
+
+/* clixon */
+#include "clixon/clixon.h"
+
+/*
+ * Types (curl)
+ */
+struct curlbuf{
+    size_t b_len;
+    char  *b_buf;
+};
+
+/*
+ * For the asynchronous case. I think we must handle the case where of many of these
+ * come in before we can handle them in the upper-level polling routine.
+ * realloc. Therefore, we append new data to the userdata buffer.
+ */
+static size_t
+curl_get_cb(void *ptr, 
+	    size_t size, 
+	    size_t nmemb, 
+	    void *userdata)
+{
+    struct curlbuf *buf = (struct curlbuf *)userdata;
+    int len;
+
+    len = size*nmemb;
+    if ((buf->b_buf = realloc(buf->b_buf, buf->b_len+len+1)) == NULL)
+	return 0;
+    memcpy(buf->b_buf+buf->b_len, ptr, len);
+    buf->b_len += len;
+    buf->b_buf[buf->b_len] = '\0';
+    //    fprintf(stderr, "%s\n", buf->b_buf);
+    return len;
+}
+
+/*! Given an URL and data to post, do a (curl) get request with data.
+ *
+ * If getdata is set, return the (malloced) data (which should be freed).
+ *
+ * @param[in] query 'q' parameter that should be URL-encoded, ie ?q=<encoded>
+ *                   XXX: dont add q=, there may be more parameters.
+ * @retval    -1    fatal error
+ * @retval     1    ok
+ *
+ * @note curl_easy_perform blocks
+ * @note New handle is created every time, the handle can be re-used for 
+ * better TCP performance
+ */
+int
+url_get(char  *url, 
+	char  *query, 
+	int    timeout,
+	char **getdata)
+{
+    int        retval = -1;
+    CURL      *curl;
+    char      *err = NULL;
+    char      *encoded = NULL;
+    struct curlbuf cb = {0, };
+    cbuf      *cbf = NULL;
+    struct curl_slist *list = NULL;
+    int       ret;
+
+    clicon_debug(1, "%s:  curl -G %s", __FUNCTION__, url);
+    /* Set up curl for doing the communication with the controller */
+    if ((curl = curl_easy_init()) == NULL) {
+	clicon_err(OE_PLUGIN, errno, "curl_easy_init");
+	goto done;
+    }
+    if ((cbf = cbuf_new()) == NULL)
+	goto done;
+
+    if (query){
+	if ((encoded = curl_easy_escape(curl, query, 0)) == NULL){
+	    clicon_debug(1, "curl_easy_escape");
+	    goto done;
+	}
+    }
+    if ((err = malloc(CURL_ERROR_SIZE)) == NULL) {
+	clicon_debug(1, "%s: malloc", __FUNCTION__);
+	goto done;
+    }
+    /* specify URL to get */ 
+    if (query)
+	cprintf(cbf, "%s?q=%s", url, encoded);
+    else
+	cprintf(cbf, "%s", url);
+    curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
+    list = curl_slist_append(list, "Accept: text/event-stream");
+    list = curl_slist_append(list, "Cache-Control: no-cache");
+    list = curl_slist_append(list, "Connection: keep-alive");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+
+    /* For reference, this url works 
+       "http://192.36.171.239:8086/db/nordunet/series?q=select%20tcmp2%20from%20%22dk-ore%22%20limit%201"
+    */
+    clicon_debug(1, "url: %s\n", cbuf_get(cbf));
+    curl_easy_setopt(curl, CURLOPT_URL, cbuf_get(cbf));
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_get_cb);
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cb);
+    
+    /* some servers don't like requests that are made without a user-agent
+       field, so we provide one */ 
+
+    curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, err);
+    //    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout);
+    //    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L);
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
+    curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);  
+    ret = curl_easy_perform(curl);
+    if (ret != CURLE_OPERATION_TIMEDOUT && ret != CURLE_OK){
+	fprintf(stderr, "curl: %s %d", err, ret);
+	goto done;
+    }
+    if (getdata && cb.b_buf){
+	*getdata = cb.b_buf;
+	cb.b_buf = NULL;
+    }
+    retval = 1;
+  done:
+    if (cbf)
+	cbuf_free(cbf);
+    if (err)
+	free(err);
+    if (encoded)
+	curl_free(encoded);
+    if (cb.b_buf)
+	free(cb.b_buf);
+    if (curl)
+	curl_easy_cleanup(curl);   /* cleanup */ 
+    return retval;
+}
+
+static int
+usage(char *argv0)
+{
+    fprintf(stderr, "usage:%s <url> <timeout>.\n\tInput on stdin\n", argv0);
+    exit(0);
+}
+
+int
+main(int argc, char **argv)
+{
+    cbuf  *cb = cbuf_new();
+    char  *url;
+    char  *query = NULL;
+    char  *getdata = NULL;
+    int    timeout;
+
+    if (argc != 3){
+	usage(argv[0]);
+	return 0;
+    }
+    url = argv[1];
+    timeout = atoi(argv[2]);
+    if (url_get(url, query, timeout, &getdata) < 0)
+	goto done;
+    fprintf(stdout, "%s", getdata);
+    fflush(stdout);
+ done:
+    if (getdata)
+	free(getdata);
+    if (cb)
+	cbuf_free(cb);
+    return 0;
+}
+
+

yang/clixon-config@2018-10-21.yang

@@ -420,7 +420,7 @@ module clixon-config {
 	}
 	leaf CLICON_STREAM_RETENTION {
 	    type uint32;
-	    default 0;
+	    default 3600;
 	    units s;
 	    description "Retention for stream replay buffers in seconds, ie how much
                          data to store before dropping. 0 means no retention";