* New clixon-config@2020-02-22.yang revision

* C-code changes:
  - Replaced stream uri:s w constants
  - Replaced large debug print code with single clicon_log_xml
  - Restconf put and post handling refactored using new parse API

CHANGELOG.md

@@ -38,7 +38,10 @@ Expected: February 2020
   * For more info, see docs at [paths](https://clixon-docs.readthedocs.io/en/latest/paths.html) and 
 [search](https://clixon-docs.readthedocs.io/en/latest/xml.html#searching-in-xml)
   * Experimental: explicit search index, ie index any list variable, not just keys
+	
 ### API changes on existing features (you may need to change your code)
+* New clixon-config@2020-02-22.yang revision
+  * Added search index extension
 * JSON parse error messages change from ` on line x: syntax error,..` to `json_parse: line x: syntax error`
 * Unknown-element error message is more descriptive, eg from `namespace is: urn:example:clixon` to: `Failed to find YANG spec of XML node: x with parent: xp in namespace urn:example:clixon`.
 * C-API parse and validation API more capable
@@ -59,7 +62,7 @@ Expected: February 2020
   
 ### Minor changes
 
-* C-API: Added instrumentation: `xml_size` and `xml_stats_get`.
+* C-API: Added instrumentation: `xml_stats` and `xml_stats_global`.
 * Obsoleted and removed XMLDB format "tree". This function did not work. Only xml and json allowed.
 * Test framework
   * Added `-- -S <file>` command-line to main example to be able to return any state to main example.

apps/backend/backend_client.c

@@ -297,7 +297,6 @@ client_statedata(clicon_handle h,
 	    clicon_err(OE_YANG, ENOENT, "clixon-rfc5277 namespace not found");
 	    goto done;
 	}
-
 	cprintf(cb, "<netconf xmlns=\"%s\"/>", namespace);
 	if (xml_parse_string2(cbuf_get(cb), YB_TOP, yspec, xret, NULL) < 0)
 	    goto done;
@@ -1035,15 +1034,8 @@ from_client_get(clicon_handle h,
     if (ret > 0 && (ret = xml_yang_validate_add(h, xret, &xerr)) < 0)
 	goto done;
     if (ret == 0){
-#if 1
+	if (debug)
-	if (debug){
+	    clicon_log_xml(LOG_DEBUG, xret, "VALIDATE_STATE");
-	    cbuf *ccc=cbuf_new();
-	    if (clicon_xml2cbuf(ccc, xret, 0, 0, -1) < 0)
-		goto done;
-	    clicon_debug(1, "%s FAIL: %s", __FUNCTION__, cbuf_get(ccc));
-	    cbuf_free(ccc);
-	}
-#endif
 	if ((xr = xpath_first(xerr, NULL, "//error-tag")) != NULL &&
 	    (xb = xml_body_get(xr))){
 	    if (xml_value_set(xb, "operation-failed") < 0)
@@ -1258,7 +1250,7 @@ from_client_create_subscription(clicon_handle h,
     struct timeval       stop;
     cvec                *nsc = NULL;
     
-    if ((nsc = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:netmod:notification")) == NULL)
+    if ((nsc = xml_nsctx_init(NULL, EVENT_RFC5277_NAMESPACE)) == NULL)
 	goto done;
     if ((x = xpath_first(xe, nsc, "//stream")) != NULL)
 	stream = xml_find_value(x, "body");
@@ -1721,7 +1713,7 @@ backend_rpc_init(clicon_handle h)
 
     /* In backend_client.? RPC from RFC 5277 */
     if (rpc_callback_register(h, from_client_create_subscription, NULL,
-		      "urn:ietf:params:xml:ns:netmod:notification", "create-subscription") < 0)
+		      EVENT_RFC5277_NAMESPACE, "create-subscription") < 0)
 	goto done;
     /* Clixon RPC */
     if (rpc_callback_register(h, from_client_debug, NULL,

apps/backend/backend_plugin.c

@@ -125,13 +125,8 @@ clixon_plugin_statedata(clicon_handle    h,
 	if (fn(h, nsc, xpath, x) < 0)
 	    goto fail;  /* Dont quit here on user callbacks */
 #if 1
-	if (debug){
+	if (debug)
-	    cbuf *ccc=cbuf_new();
+	    clicon_log_xml(LOG_DEBUG, x, "%s STATE:", __FUNCTION__);
-	    if (clicon_xml2cbuf(ccc, x, 0, 0, -1) < 0)
-		goto done;
-	    clicon_debug(1, "%s STATE: %s", __FUNCTION__, cbuf_get(ccc));
-	    cbuf_free(ccc);
-	}
 #endif
 	if (xml_spec_populate(x, yspec, NULL) < 0)
 	    goto done;

apps/netconf/netconf_rpc.c

@@ -442,7 +442,7 @@ netconf_notification_cb(int   s,
     if (clicon_msg_decode(reply, yspec, NULL, &xt) < 0) 
 	goto done;
 
-    if ((nsc = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:netconf:notification:1.0")) == NULL)
+    if ((nsc = xml_nsctx_init(NULL, NOTIFICATION_RFC5277_NAMESPACE)) == NULL)
 	goto done;
     if ((xn = xpath_first(xt, nsc, "notification")) == NULL)
 	goto ok;

apps/restconf/restconf_main.c

@@ -232,9 +232,7 @@ api_root(clicon_handle  h,
     FCGX_FPrintF(r->out, "Content-Type: %s\r\n", restconf_media_int2str(media_out));
     FCGX_FPrintF(r->out, "\r\n");
 
-    if (xml_parse_string("<restconf xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf\"><data/><operations/><yang-library-version>2016-06-21</yang-library-version></restconf>", NULL, &xt) < 0)
+    if (xml_parse_string("<restconf xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf\"><data/><operations/><yang-library-version>2016-06-21</yang-library-version></restconf>", yspec, &xt) < 0)
-	goto done;
-    if (xml_spec_populate(xt, yspec, NULL) < 0)
 	goto done;
     if ((cb = cbuf_new()) == NULL){
 	clicon_err(OE_XML, errno, "cbuf_new");

apps/restconf/restconf_methods.c

@@ -261,9 +261,9 @@ api_data_write(clicon_handle h,
     int            ret;
     char          *namespace = NULL;
     char          *dname;
-    int        nullspec = 0;
     cbuf          *cbpath = NULL;    
     cvec          *nsc = NULL;
+    enum yang_bind yb;
 
     clicon_debug(1, "%s api_path:\"%s\"",  __FUNCTION__, api_path0);
     clicon_debug(1, "%s data:\"%s\"", __FUNCTION__, data);
@@ -306,13 +306,8 @@ api_data_write(clicon_handle h,
 
     }
 #if 0
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xret, "%s xret:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xret, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s XRET: %s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
     if (xml_child_nr(xret) == 0){ /* Object does not exist */
 	if (plain_patch){    /* If the target resource instance does not exist, the server MUST NOT create it. */
@@ -370,12 +365,35 @@ api_data_write(clicon_handle h,
      */
     if ((xdata0 = xml_new("data0", NULL, NULL)) == NULL)
 	goto done;
-    if (xml_copy_one(api_path?xml_parent(xbot):xbot, xdata0) < 0)
+    { /* XXX mv to copy? */
+	cxobj *xfrom;
+	cxobj *xac;
+	
+	xfrom = api_path?xml_parent(xbot):xbot;
+	if (xml_copy_one(xfrom, xdata0) < 0)
 	    goto done;
-    /* Parse input data as json or xml into xml */
+	xa = NULL;
+	while ((xa = xml_child_each(xfrom, xa, CX_ATTR)) != NULL) {
+	    if ((xac = xml_new(xml_name(xa), xdata0, NULL)) == NULL)
+		goto done;
+	    if (xml_copy(xa, xac) < 0) /* recursion */
+		goto done;
+	}
+    }
+    if (xml_spec(xdata0)==NULL)
+	yb = YB_TOP;
+    else
+	yb = YB_PARENT;
+
+    /* Parse input data as json or xml into xml 
+     * Note that in POST (api_data_post) the new object is grafted on xbot, since it is a new
+     * object. In that case all yang bindings can be made since xbot is available.
+     * Here the new object replaces xbot and is therefore more complicated to make when parsing.
+     * Instead, xbots parent is copied into xdata0 (but not its children).
+     */
     switch (media_in){
     case YANG_DATA_XML:
-	if (xml_parse_string(data, yspec, &xdata0) < 0){
+	if ((ret = xml_parse_string2(data, yb, yspec, &xdata0, &xerr)) < 0){
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@@ -386,15 +404,18 @@ api_data_write(clicon_handle h,
 		goto done;
 	    goto ok;
 	}
+	if (ret == 0){
+	    if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
+		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
+		goto done;
+	    }
+	    if (api_return_err(h, r, xe, pretty, media_out, 0) < 0)
+		goto done;
+	    goto ok;
+	}
 	break;
     case YANG_DATA_JSON:
-	/* Data here cannot cannot be Yang populated since it is loosely
+	if ((ret = json_parse_str2(data, yb, yspec, &xdata0, &xerr)) < 0){
-	 * hanging without top symbols.
-	 * And if it is not yang populated, it cant be translated properly
-	 * from JSON to XML.
-	 * Therefore, yang population is done later after addsub below
-	 */
-	if ((ret = json_parse_str(data, yspec, &xdata0, &xerr)) < 0){
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@@ -436,15 +457,6 @@ api_data_write(clicon_handle h,
 	goto ok;
     }
     xdata = xml_child_i_type(xdata0, 0, CX_ELMNT);
-#if 0
-    if (debug){
-	cbuf *ccc=cbuf_new();
-	if (clicon_xml2cbuf(ccc, xdata, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s DATA:%s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
-#endif
     /* If the api-path (above) defines a module, then xdata must have a prefix
      * and it match the module defined in api-path
      * This does not apply if api-path is / (no module)
@@ -568,23 +580,6 @@ api_data_write(clicon_handle h,
 	xml_purge(xbot);
 	if (xml_addsub(xparent, xdata) < 0)
 	    goto done;
-	nullspec = (xml_spec(xdata) == NULL);
-	/* xbot is already populated, resolve yang for added xdata too */
-	if (xml_spec_populate0(xdata, yspec, NULL) < 0)
-	    goto done;
-	if (media_in == YANG_DATA_JSON && nullspec){
-	    /* json2xml decode could not be done above in json_parse,
-	     * need to be done here instead 
-	     * UNLESS it is root resource, then json-parse has already done it
-	     */
-	    if ((ret = json2xml_decode(xdata, &xerr)) < 0)
-		goto done;
-	    if (ret == 0){
-		if (api_return_err(h, r, xerr, pretty, media_out, 0) < 0)
-		    goto done;
-		goto ok;
-	    }
-	}
 	/* If restconf insert/point attributes are present, translate to netconf */
 	if (restconf_insert_attributes(xdata, qvec) < 0)
 	    goto done;
@@ -864,9 +859,11 @@ api_data_delete(clicon_handle h,
     if ((xa = xml_new("operation", xbot, NULL)) == NULL)
 	goto done;
     xml_type_set(xa, CX_ATTR);
-    xml_prefix_set(xa, NETCONF_BASE_PREFIX);
     if (xml_value_set(xa, xml_operation2str(op)) < 0)
 	goto done;
+    if (xml_namespace_change(xa, NETCONF_BASE_NAMESPACE, NETCONF_BASE_PREFIX) < 0)
+	goto done;
+
     if ((cbx = cbuf_new()) == NULL)
 	goto done;
     /* For internal XML protocol: add username attribute for access control

apps/restconf/restconf_methods_get.c

@@ -210,12 +210,8 @@ api_data_get2(clicon_handle h,
      * We need to cut that tree to only the object.
      */
 #if 0 /* DEBUG */
-    if (debug){
+    if (debug)
-	cbuf *cb = cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xret, "%s xret:", __FUNCTION__);
-	clicon_xml2cbuf(cb, xret, 0, 0, -1);
-	clicon_debug(1, "%s xret:%s", __FUNCTION__, cbuf_get(cb));
-	cbuf_free(cb);
-    }
 #endif
     /* Check if error return  */
     if ((xe = xpath_first(xret, NULL, "//rpc-error")) != NULL){

apps/restconf/restconf_methods_post.c

@@ -129,6 +129,7 @@ api_data_post(clicon_handle h,
     int            ret;
     restconf_media media_in;
     int            nrchildren0 = 0;
+    enum yang_bind yb;
     
     clicon_debug(1, "%s api_path:\"%s\"", __FUNCTION__, api_path);
     clicon_debug(1, "%s data:\"%s\"", __FUNCTION__, data);
@@ -178,11 +179,18 @@ api_data_post(clicon_handle h,
 	nrchildren0++;
 	xml_flag_set(x, XML_FLAG_MARK);
     }
-    /* Parse input data as json or xml into xml */
+    if (xml_spec(xbot)==NULL)
+	yb = YB_TOP;
+    else
+	yb = YB_PARENT;
+    /* Parse input data as json or xml into xml 
+     * If xbot is top-level (api_path=null) it does not have a spec therefore look for 
+     * top-level (yspec) otherwise assume parent (xbot) is populated.
+     */
     media_in = restconf_content_type(r);
     switch (media_in){
     case YANG_DATA_XML:
-	if (xml_parse_string(data, yspec, &xbot) < 0){
+	if ((ret = xml_parse_string2(data, yb, yspec, &xbot, &xerr)) < 0){
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@@ -193,12 +201,18 @@ api_data_post(clicon_handle h,
 		goto done;
 	    goto ok;
 	}
+	if (ret == 0){
+	    if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
+		clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
+		goto done;
+	    }
+	    if (api_return_err(h, r, xe, pretty, media_out, 0) < 0)
+		goto done;
+	    goto ok;
+	}
 	break;
     case YANG_DATA_JSON:	
-	/* If xbot is top-level (api_path=null) it does not have a spec therefore look for 
+	if ((ret = json_parse_str2(data, yb, yspec, &xbot, &xerr)) < 0){
-	 * top-level (yspec) otherwise assume parent (xbot) is populated.
-	 */
-	if ((ret = json_parse_str(data, yspec, &xbot, &xerr)) < 0){
 	    if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
 		goto done;
 	    if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@@ -256,14 +270,8 @@ api_data_post(clicon_handle h,
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa, xml_operation2str(op)) < 0)
 	goto done;
-
-#if 0 /* XXX postpone this, there is something wrong with NETCONF_BASE_NAMESPACE not appearing here
-      * but later it does due to default handling,... */
     if (xml_namespace_change(xa, NETCONF_BASE_NAMESPACE, NETCONF_BASE_PREFIX) < 0)
 	goto done;
-#else
-    xml_prefix_set(xa, NETCONF_BASE_PREFIX); /* XXX: But this assumes proper namespace set */
-#endif
 
     if (ys_module_by_xml(yspec, xdata, &ymoddata) < 0)
 	goto done;
@@ -299,13 +307,8 @@ api_data_post(clicon_handle h,
     if (restconf_insert_attributes(xdata, qvec) < 0)
 	goto done;
 #if 1
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xdata, "%s xdata:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xdata, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s XDATA:%s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
 
     /* Create text buffer for transfer to backend */
@@ -495,13 +498,8 @@ api_operations_post_input(clicon_handle h,
      * <data><input xmlns="urn:example:clixon">...</input></data>
      */
 #if 1
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xdata, "%s xdata:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xdata, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s DATA:%s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
     /* Validate that exactly only <input> tag */
     if ((xinput = xml_child_i_type(xdata, 0, CX_ELMNT)) == NULL ||
@@ -602,13 +600,8 @@ api_operations_post_output(clicon_handle h,
     xml_name_set(xoutput, "output");
     /* xoutput should now look: <output><x xmlns="uri">0</x></output> */
 #if 1
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xoutput, "%s xoutput:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xoutput, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s XOUTPUT:%s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
 
     /* Sanity check of outgoing XML 
@@ -840,20 +833,15 @@ api_operations_post(clicon_handle h,
     /* Here xtop is: 
       <rpc username="foo"><myfn xmlns="uri"><x>42</x></myfn></rpc> */
 #if 1
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xtop, "%s 5. Translate input args:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xtop, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s 5. Translate input args: %s",
-		     __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
-    /* 6. Validate incoming RPC and fill in defaults */
+    /* 6. Validate outgoing RPC and fill in defaults */
     if (xml_spec_populate_rpc(xtop, yspec, NULL) < 0) /*  */
 	goto done;
     if ((ret = xml_yang_validate_rpc(h, xtop, &xret)) < 0)
 	goto done;
+
     if (ret == 0){
 	if ((xe = xpath_first(xret, NULL, "rpc-error")) == NULL){
 	    clicon_err(OE_XML, EINVAL, "rpc-error not found (internal error)");
@@ -867,13 +855,8 @@ api_operations_post(clicon_handle h,
      * <rpc username="foo"><myfn xmlns="uri"><x>42</x><y>99</y></myfn></rpc>
     */
 #if 0
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xtop, "%s 6. Validate and defaults:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xtop, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s 6. Validate and defaults:%s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
     /* 7. Send to RPC handler, either local or backend
      * Note (1) xtop is <rpc><method> xbot is <method>
@@ -907,13 +890,8 @@ api_operations_post(clicon_handle h,
      *       <rpc-reply><x xmlns="uri">0</x></rpc-reply>
      */
 #if 1
-    if (debug){
+    if (debug)
-	cbuf *ccc=cbuf_new();
+	clicon_log_xml(LOG_DEBUG, xret, "%s Receive reply:", __FUNCTION__);
-	if (clicon_xml2cbuf(ccc, xret, 0, 0, -1) < 0)
-	    goto done;
-	clicon_debug(1, "%s 8. Receive reply:%s", __FUNCTION__, cbuf_get(ccc));
-	cbuf_free(ccc);
-    }
 #endif
     youtput = yang_find(yrpc, Y_OUTPUT, NULL);
     if ((ret = api_operations_post_output(h, r, xret, yspec, youtput, namespace,

apps/restconf/restconf_stream.c

@@ -249,7 +249,8 @@ restconf_stream(clicon_handle h,
 	clicon_err(OE_XML, errno, "cbuf_new");
 	goto done;
     }
-    cprintf(cb, "<rpc><create-subscription xmlns=\"urn:ietf:params:xml:ns:netmod:notification\"><stream>%s</stream>", name);
+    cprintf(cb, "<rpc><create-subscription xmlns=\"%s\"><stream>%s</stream>",
+	    EVENT_RFC5277_NAMESPACE, name);
     /* Print all fields */
     for (i=0; i<cvec_len(qvec); i++){
         cv = cvec_i(qvec, i);

example/hello/hello.xml

@@ -1,5 +1,5 @@
 <clixon-config xmlns="http://clicon.org/config">
-  <CLICON_CONFIGFILE>/usr/local/etc/example.xml</CLICON_CONFIGFILE>
+  <CLICON_CONFIGFILE>/usr/local/etc/clixon.xml</CLICON_CONFIGFILE>
   <CLICON_FEATURE>*:*</CLICON_FEATURE>
   <CLICON_YANG_DIR>/usr/local/share/clixon</CLICON_YANG_DIR>
   <CLICON_YANG_MODULE_MAIN>clixon-hello</CLICON_YANG_MODULE_MAIN>

include/clixon_custom.h

@@ -47,6 +47,7 @@
  * NETCONF namespace (see rfc6241 3.1)
  * Undefine it if you want to ensure strict namespace assignment on all netconf and
  * XML statements.
+ * The base namespace is defined as NETCONF_BASE_NAMESPACE
  */
 #define USE_NETCONF_NS_AS_DEFAULT
 

lib/clixon/clixon_json.h

@@ -50,6 +50,7 @@ int xml2json(FILE *f, cxobj *x, int pretty);
 int json_print(FILE *f, cxobj *x);
 int xml2json_vec(FILE *f, cxobj **vec, size_t veclen, int pretty);
 int json_parse_str(char *str, yang_stmt *yspec, cxobj **xt, cxobj **xret);
+int json_parse_str2(char *str, enum yang_bind yb, yang_stmt *yspec, cxobj **xt, cxobj **xret);
 int json_parse_file(int fd, yang_stmt *yspec, cxobj **xt, cxobj **xret);
 
 #endif /* _CLIXON_JSON_H */

lib/clixon/clixon_stream.h

@@ -36,6 +36,26 @@
 #ifndef _CLIXON_STREAM_H_
 #define _CLIXON_STREAM_H_
 
+/*
+ * Constants
+ */
+/* Default STREAM namespace (see rfc5277 3.1)
+ * From RFC8040: 
+ *  The structure of the event data is based on the <notification>
+ *  element definition in Section 4 of [RFC5277].  It MUST conform to the
+ *  schema for the <notification> element in Section 4 of [RFC5277],
+ *  using the XML namespace as defined in the XSD as follows:
+ *     urn:ietf:params:xml:ns:netconf:notification:1.0
+ * It is used everywhere in yangmodels, but not in openconfig
+ */
+#define NOTIFICATION_RFC5277_NAMESPACE "urn:ietf:params:xml:ns:netconf:notification:1.0"
+
+/*
+ * Then there is also this namespace that is only used in RFC5277 seems to be for "netconf"
+ * events. The usage seems wrong here,...
+ */
+#define EVENT_RFC5277_NAMESPACE "urn:ietf:params:xml:ns:netmod:notification"
+
 /*
  * Types
  */

lib/clixon/clixon_xml.h

@@ -86,9 +86,11 @@ enum cxobj_type {CX_ERROR=-1,
 
 /* How to bind yang to XML top-level when parsing */
 enum yang_bind{ 
+    YB_UNKNOWN=0, /* System derive binding: top if parent not exist or no spec, otherwise parent */
     YB_NONE,    /* Dont try to do Yang binding */
     YB_PARENT,  /* Get yang binding from parents yang */
     YB_TOP,     /* Get yang binding from top-level modules */
+    YB_RPC,     /* Assume top-level xml is an netconf RPC message */
 };
 
 #define CX_ANY CX_ERROR /* catch all and error is same */
@@ -117,8 +119,8 @@ typedef int (xml_applyfn_t)(cxobj *x, void *arg);
  * Prototypes
  */
 char     *xml_type2str(enum cxobj_type type);
-int       xml_stats_get(uint64_t *nr);
+int       xml_stats_global(uint64_t *nr);
-size_t    xml_size(cxobj *xt, size_t *szp);
+size_t    xml_stats(cxobj *xt, uint64_t *nrp, size_t *szp);
 char     *xml_name(cxobj *xn);
 int       xml_name_set(cxobj *xn, char *name);
 char     *xml_prefix(cxobj *xn);

lib/src/clixon_json.c

@@ -1066,7 +1066,6 @@ json_xmlns_translate(yang_stmt *yspec,
     yang_stmt *ymod;
     char      *namespace;
     char      *modname = NULL;
-    char      *prefix;
     cxobj     *xc;
     int        ret;
     
@@ -1080,8 +1079,13 @@ json_xmlns_translate(yang_stmt *yspec,
 	    goto fail;
 	}
 	namespace = yang_find_mynamespace(ymod);
-	prefix = yang_find_myprefix(ymod);
+	/* It would be possible to use canonical prefixes here, but probably not
-	if (xml_namespace_change(x, namespace, prefix) < 0)
+	 * necessary or even right. Therefore, the namespace given by the JSON prefix / module
+	 * is always the default namespace with prefix NULL.
+	 * If not, this would be the prefix to pass instead of NULL
+	 * prefix = yang_find_myprefix(ymod);
+	 */
+	if (xml_namespace_change(x, namespace, NULL) < 0)
 	    goto done;
     }
     xc = NULL;
@@ -1172,6 +1176,8 @@ _json_parse(char          *str,
 	 * XXX should be xml_spec_populate0_parent() sometimes.
 	 */
 	switch (yb){
+	case YB_RPC:
+	case YB_UNKNOWN:
 	case YB_NONE:
 	    break;
 	case YB_PARENT:
@@ -1231,6 +1237,25 @@ _json_parse(char          *str,
  * @retval       -1     Error with clicon_err called
  * @see json_parse_file with a file descriptor (and more description)
  */
+int 
+json_parse_str2(char          *str, 
+		enum yang_bind yb,
+		yang_stmt     *yspec,
+		cxobj        **xt,
+		cxobj        **xerr)
+{
+    clicon_debug(1, "%s", __FUNCTION__);
+    if (xt==NULL){
+	clicon_err(OE_XML, EINVAL, "xt is NULL");
+	return -1;
+    }
+    if (*xt == NULL){
+	if ((*xt = xml_new("top", NULL, NULL)) == NULL)
+	    return -1;
+    }
+    return _json_parse(str, yb, yspec, *xt, xerr);
+}
+
 int 
 json_parse_str(char      *str, 
 	       yang_stmt *yspec,

lib/src/clixon_proto_client.c

@@ -73,6 +73,7 @@
 #include "clixon_xpath.h"
 #include "clixon_proto.h"
 #include "clixon_err.h"
+#include "clixon_stream.h"
 #include "clixon_err_string.h"
 #include "clixon_xml_nsctx.h"
 #include "clixon_xml_map.h"
@@ -750,7 +751,6 @@ clicon_rpc_get(clicon_handle   h,
     return retval;
 }
 
-
 /*! Close a (user) session
  * @param[in] h        CLICON handle
  * @retval    0        OK
@@ -964,11 +964,12 @@ clicon_rpc_create_subscription(clicon_handle    h,
 	goto done;
     username = clicon_username_get(h);
     if ((msg = clicon_msg_encode(session_id,
-				 "<rpc username=\"%s\"><create-subscription xmlns=\"urn:ietf:params:xml:ns:netmod:notification\">"
+				 "<rpc username=\"%s\"><create-subscription xmlns=\"%s\">"
 				 "<stream>%s</stream>"
 				 "<filter type=\"xpath\" select=\"%s\" />"
 				 "</create-subscription></rpc>", 
 				 username?username:"",
+				 EVENT_RFC5277_NAMESPACE,
 				 stream?stream:"", filter?filter:"")) == NULL)
 	goto done;
     if (clicon_rpc_msg(h, msg, &xret, s0) < 0)

lib/src/clixon_stream.c

@@ -581,7 +581,8 @@ stream_notify(clicon_handle h,
 	goto done;
     }
     /* From RFC5277 */
-    cprintf(cb, "<notification xmlns=\"urn:ietf:params:xml:ns:netconf:notification:1.0\"><eventTime>%s</eventTime>%s</notification>", timestr, str);
+    cprintf(cb, "<notification xmlns=\"%s\"><eventTime>%s</eventTime>%s</notification>",
+	    NOTIFICATION_RFC5277_NAMESPACE, timestr, str);
     if (xml_parse_string(cbuf_get(cb), yspec, &xev) < 0)
 	goto done;
     if (xml_rootchild(xev, 0, &xev) < 0)
@@ -644,7 +645,9 @@ stream_notify_xml(clicon_handle h,
 	clicon_err(OE_UNIX, errno, "time2str");
 	goto done;
     }
-    cprintf(cb, "<notification xmlns=\"urn:ietf:params:xml:ns:netconf:notification:1.0\"><eventTime>%s</eventTime>NULL</notification>", timestr); /* XXX str is always NULL */
+    cprintf(cb, "<notification xmlns=\"%s\"><eventTime>%s</eventTime>NULL</notification>",
+	    NOTIFICATION_RFC5277_NAMESPACE,
+	    timestr); /* XXX str is always NULL */
     if (xml_parse_string(cbuf_get(cb), yspec, &xev) < 0)
 	goto done;
     if (xml_rootchild(xev, 0, &xev) < 0)

lib/src/clixon_xml.c

@@ -173,7 +173,7 @@ uint64_t _stats_nr = 0;
 /*! Get global statistics about XML objects
  */
 int
-xml_stats_get(uint64_t *nr)
+xml_stats_global(uint64_t *nr)
 {
     if (nr)
 	*nr = _stats_nr;
@@ -187,7 +187,7 @@ xml_stats_get(uint64_t *nr)
  * (baseline: 96 bytes per object on x86-64)
  */
 static int
-xml_size_one(cxobj  *x,
+xml_stats_one(cxobj    *x,
 	      size_t   *szp)
 {
     size_t sz = 0;
@@ -209,24 +209,26 @@ xml_size_one(cxobj  *x,
     return 0;
 }
 
-/*! Return the alloced memory of a XML obj tree recursively
+/*! Return statistics of an XML tree recursively
  * @param[in]   x    XML object
  * @param[out]  szp  Size of this XML obj recursively
  * @retval      0    OK
  */
 size_t
-xml_size(cxobj  *xt,
+xml_stats(cxobj    *xt,
+	  uint64_t *nrp,
 	  size_t   *szp)
 {
     size_t sz = 0;
     cxobj *xc;
 
-    xml_size_one(xt, &sz);
+    *nrp += 1;
+    xml_stats_one(xt, &sz);
     if (szp)
 	*szp += sz;
     xc = NULL;
     while ((xc = xml_child_each(xt, xc, -1)) != NULL) {
-	xml_size(xc, &sz);
+	xml_stats(xc, nrp, &sz);
 	if (szp)
 	    *szp += sz;
     }
@@ -1737,8 +1739,6 @@ xml_free(cxobj *x)
  *------------------------------------------------------------------------*/
 
 /*! Print an XML tree structure to an output stream and encode chars "<>&"
- *
- * Uses clicon_xml2cbuf internally
  *
  * @param[in]   f           UNIX output stream
  * @param[in]   xn          clicon xml tree
@@ -2070,6 +2070,8 @@ _xml_parse(const char    *str,
 	/* Populate, ie associate xml nodes with yang specs 
 	 */
 	switch (yb){
+	case YB_RPC:
+	case YB_UNKNOWN:
 	case YB_NONE:
 	    break;
 	case YB_PARENT:
@@ -2325,7 +2327,8 @@ xml_parse_string2(const char    *str,
  * @see xml_parse_file
  * @see xml_parse_va
  * @note You need to free the xml parse tree after use, using xml_free()
- * @note If empty on entry, a new TOP xml will be created named "top"
+ * @note If xt is empty on entry, a new TOP xml will be created named "top" and yang binding 
+ *       assumed to be TOP
  */
 int 
 xml_parse_string(const char *str, 

lib/src/clixon_xml_map.c

@@ -801,10 +801,16 @@ xml_tree_prune_flagged(cxobj *xt,
 }
 
 /*! Add prefix:namespace pair to xml node, set cache, prefix, etc
+ * @param[in]  x         XML node whose namespace should change
+ * @param[in]  xp        XML node where namespace attribute should be declared (can be same)
+ * @param[in]  prefix1   Use this prefix
+ * @param[in]  namespace Use this namespace
+ * @note x and xp must be different if x is an attribute and may be different otherwise
  */
 static int
-add_namespace(cxobj *x1, /* target */
+add_namespace(cxobj *x,
-	      char  *prefix1,
+	      cxobj *xp,
+	      char  *prefix,
 	      char  *namespace)
 {
     int    retval = -1;
@@ -813,26 +819,26 @@ add_namespace(cxobj *x1, /* target */
     /* Add binding to x1p. We add to parent due to heurestics, so we dont
      * end up in adding it to large number of siblings 
      */
-    if (nscache_set(x1, prefix1, namespace) < 0)
+    if (nscache_set(x, prefix, namespace) < 0)
 	goto done;
     /* Create xmlns attribute to x1p/x1 XXX same code v */
-    if (prefix1){
+    if (prefix){
-	if ((xa = xml_new(prefix1, x1, NULL)) == NULL)
+	if ((xa = xml_new(prefix, xp, NULL)) == NULL)
 	    goto done;
 	if (xml_prefix_set(xa, "xmlns") < 0)
 	    goto done;
     }
     else{
-	if ((xa = xml_new("xmlns", x1, NULL)) == NULL)
+	if ((xa = xml_new("xmlns", xp, NULL)) == NULL)
 	    goto done;
     }
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa, namespace) < 0)
 	goto done;
-    xml_sort(x1, NULL); /* Ensure attr is first / XXX xml_insert? */
+    xml_sort(xp, NULL); /* Ensure attr is first / XXX xml_insert? */
 	
-    /* 5. Add prefix to x1, if any */
+    /* 5. Add prefix to x, if any */
-    if (prefix1 && xml_prefix_set(x1, prefix1) < 0)
+    if (prefix && xml_prefix_set(x, prefix) < 0)
 	goto done;
     retval = 0;
  done:
@@ -855,6 +861,7 @@ xml_namespace_change(cxobj *x,
     int    retval = -1;
     char  *ns0 = NULL;     /* existing namespace */
     char  *prefix0 = NULL; /* existing prefix */
+    cxobj *xp;
 
     ns0 = NULL;
     if (xml2ns(x, xml_prefix(x), &ns0) < 0)
@@ -863,7 +870,7 @@ xml_namespace_change(cxobj *x,
 	goto ok; /* Already has right namespace */ 
     /* Is namespace already declared? */
     if (xml2prefix(x, namespace, &prefix0) == 1){
-       /* Yes it is declared and the prefix is pexists */
+       /* Yes it is declared and the prefix is prefix0 */
        if (xml_prefix_set(x, prefix0) < 0)
            goto done;
     }
@@ -871,7 +878,11 @@ xml_namespace_change(cxobj *x,
 	/* Clear old prefix if any */
        if (xml_prefix_set(x, NULL) < 0)
            goto done;
-       if (add_namespace(x, prefix0, namespace) < 0)
+       if (xml_type(x) == CX_ELMNT) /* If not element, do the namespace addition to the element */
+	   xp = x;
+       else
+	   xp = xml_parent(x);
+       if (add_namespace(x, xp, prefix, namespace) < 0)
 	   goto done;	   
     }
  ok:
@@ -933,7 +944,7 @@ xml_default(cxobj *xt,
 				clicon_err(OE_UNIX, errno, "strdup");
 				goto done;
 			    }
-			    if (add_namespace(xc, prefix, namespace) < 0)
+			    if (add_namespace(xc, xc, prefix, namespace) < 0)
 				goto done;
 			}
 		    }
@@ -1634,7 +1645,7 @@ assign_namespaces(cxobj *x0, /* source */
 		}
 	    }
 	}
-	if (add_namespace(x1, prefix1, namespace) < 0)
+	if (add_namespace(x1, x1, prefix1, namespace) < 0)
 	    goto done;
     }
  ok:

lib/src/clixon_yang_parse_lib.c

@@ -161,8 +161,8 @@ yang_augment_node(yang_stmt *ys,
     int        retval = -1;
     char      *schema_nodeid;
     yang_stmt *ytarget = NULL;
+    yang_stmt *yc0;
     yang_stmt *yc;
-    int        i;
     yang_stmt *ymod;
 
     if ((ymod = ys_module(ys)) == NULL){
@@ -179,8 +179,9 @@ yang_augment_node(yang_stmt *ys,
     /* Extend ytarget with ys' children
      * First enlarge ytarget vector 
      */
-    for (i=0; i<ys->ys_len; i++){
+    yc0 = NULL;
-	if ((yc = ys_dup(ys->ys_stmt[i])) == NULL)
+    while ((yc0 = yn_each(ys, yc0)) != NULL) {
+	if ((yc = ys_dup(yc0)) == NULL)
 	    goto done;
 	yc->ys_mymodule = ymod;
 	if (yn_insert(ytarget, yc) < 0)
@@ -203,11 +204,11 @@ yang_augment_spec(yang_stmt *ysp,
     int        i;
     int        j; 
 
-    i = modnr;
+    i = modnr; /* cant use yang_each here since you dont start at 0 */
-    while (i<ysp->ys_len){ /* Loop through modules and sub-modules */
+    while (i < yang_len_get(ysp)){ /* Loop through modules and sub-modules */
 	ym = ysp->ys_stmt[i++];
 	j = 0;
-	while (j<ym->ys_len){ /* Top-level symbols in modules */
+	while (j < yang_len_get(ym)){ /* Top-level symbols in modules */
 	    ys = ym->ys_stmt[j++];
 	    switch (yang_keyword_get(ys)){
 	    case Y_AUGMENT: /* top-level */
@@ -260,7 +261,7 @@ ys_do_refine(yang_stmt *yr,
 	case Y_MAX_ELEMENTS:
 	case Y_EXTENSION:
 	    /* Remove old matching, dont increment due to prune in loop */
-	    for (i=0; i<yt->ys_len; ){
+	    for (i=0; i<yang_len_get(yt); ){
 		ytc = yt->ys_stmt[i];
 		if (keyw != yang_keyword_get(ytc)){
 		    i++;
@@ -321,7 +322,7 @@ yang_expand_grouping(yang_stmt *yn)
 
     /* Cannot use yang_apply here since child-list is modified (is destructive) */
     i = 0;
-    while (i<yn->ys_len){
+    while (i < yang_len_get(yn)){
 	ys = yn->ys_stmt[i]; 
 	switch(yang_keyword_get(ys)){
 	case Y_USES:
@@ -359,15 +360,15 @@ yang_expand_grouping(yang_stmt *yn)
 	    /* Replace ys with ygrouping,... 
 	     * First enlarge parent vector 
 	     */
-	    glen = ygrouping2->ys_len;
+	    glen = yang_len_get(ygrouping2);
 	    /* 
 	     * yn is parent: the children of ygrouping replaces ys.
 	     * Is there a case when glen == 0?  YES AND THIS BREAKS
 	     */
 	    if (glen != 1){
-		size = (yn->ys_len - i - 1)*sizeof(struct yang_stmt *);
+		size = (yang_len_get(yn) - i - 1)*sizeof(struct yang_stmt *);
 		yn->ys_len += glen - 1;
-		if (glen && (yn->ys_stmt = realloc(yn->ys_stmt, (yn->ys_len)*sizeof(yang_stmt *))) == 0){
+		if (glen && (yn->ys_stmt = realloc(yn->ys_stmt, (yang_len_get(yn))*sizeof(yang_stmt *))) == 0){
 		    clicon_err(OE_YANG, errno, "realloc");
 		    goto done;
 		}
@@ -411,7 +412,7 @@ yang_expand_grouping(yang_stmt *yn)
 	    /* Remove 'uses' node */
 	    ys_free(ys); 
 	    /* Remove the grouping copy */
-	    ygrouping2->ys_len = 0;
+	    ygrouping2->ys_len = 0; /* Cant do with get access function */
 	    ys_free(ygrouping2);
 	    break; /* Note same child is re-iterated since it may be changed */
 	default:
@@ -420,7 +421,7 @@ yang_expand_grouping(yang_stmt *yn)
 	}
     }
     /* Second pass since length may have changed */
-    for (i=0; i<yn->ys_len; i++){
+    for (i=0; i<yang_len_get(yn); i++){
 	ys = yn->ys_stmt[i];
 	if (yang_expand_grouping(ys) < 0)
 	    goto done;
@@ -903,22 +904,22 @@ yang_parse_post(clicon_handle h,
     /* 1: Parse from text to yang parse-tree. 
      * Iterate through modules and detect module/submodules to parse
      * - note the list may grow on each iteration */
-    for (i=modnr; i<yspec->ys_len; i++)
+    for (i=modnr; i<yang_len_get(yspec); i++)
 	if (yang_parse_recurse(h, yspec->ys_stmt[i], yspec) < 0)
 	    goto done;
 
     /* 2. Check cardinality maybe this should be done after grouping/augment */
-    for (i=modnr; i<yspec->ys_len; i++) 
+    for (i=modnr; i<yang_len_get(yspec); i++) 
 	if (yang_cardinality(h, yspec->ys_stmt[i], yang_argument_get(yspec->ys_stmt[i])) < 0)
 	    goto done;
     
     /* 3: Check features: check if enabled and remove disabled features */
-    for (i=modnr; i<yspec->ys_len; i++) /* XXX */
+    for (i=modnr; i<yang_len_get(yspec); i++) /* XXX */
 	if (yang_features(h, yspec->ys_stmt[i]) < 0)
 	    goto done;
     
     /* 4: Go through parse tree and populate it with cv types */
-    for (i=modnr; i<yspec->ys_len; i++)
+    for (i=modnr; i<yang_len_get(yspec); i++)
 	if (yang_apply(yspec->ys_stmt[i], -1, ys_populate, (void*)h) < 0)
 	    goto done;
 
@@ -926,7 +927,7 @@ yang_parse_post(clicon_handle h,
      * from ys_populate step.
      * Must be done using static binding.
      */
-    for (i=modnr; i<yspec->ys_len; i++)
+    for (i=modnr; i<yang_len_get(yspec); i++)
 	if (yang_apply(yspec->ys_stmt[i], Y_TYPE, ys_resolve_type, h) < 0)
 	    goto done;
 
@@ -938,7 +939,7 @@ yang_parse_post(clicon_handle h,
      */
 
     /* 6: Macro expansion of all grouping/uses pairs. Expansion needs marking */
-    for (i=modnr; i<yspec->ys_len; i++){
+    for (i=modnr; i<yang_len_get(yspec); i++){
 	if (yang_expand_grouping(yspec->ys_stmt[i]) < 0)
 	    goto done;
 	yang_apply(yspec->ys_stmt[i], -1, (yang_applyfn_t*)yang_flag_reset, (void*)YANG_FLAG_MARK);
@@ -949,12 +950,12 @@ yang_parse_post(clicon_handle h,
 	goto done;
 
     /* 4: Go through parse tree and do 2nd step populate (eg default) */
-    for (i=modnr; i<yspec->ys_len; i++)
+    for (i=modnr; i<yang_len_get(yspec); i++)
 	if (yang_apply(yspec->ys_stmt[i], -1, ys_populate2, (void*)h) < 0)
 	    goto done;
 
     /* 8: sanity check of schemanode references, need more here */
-    for (i=modnr; i<yspec->ys_len; i++)
+    for (i=modnr; i<yang_len_get(yspec); i++)
 	if (yang_apply(yspec->ys_stmt[i], -1, ys_schemanode_check, NULL) < 0)
 	    goto done;
     retval = 0;
@@ -990,7 +991,7 @@ yang_spec_parse_module(clicon_handle h,
 	goto done;
     }
     /* Apply steps 2.. on new modules, ie ones after modnr. */
-    modnr = yspec->ys_len;
+    modnr = yang_len_get(yspec);
     /* Do not load module if it already exists */
     if (yang_find(yspec, Y_MODULE, module) != NULL)
 	goto ok;
@@ -1026,7 +1027,7 @@ yang_spec_parse_file(clicon_handle h,
     char       *base = NULL;;
 
     /* Apply steps 2.. on new modules, ie ones after modnr. */
-    modnr = yspec->ys_len;
+    modnr = yang_len_get(yspec);
     /* Find module, and do not load file if module already exists */
     if (basename(filename) == NULL){
 	clicon_err(OE_YANG, errno, "No basename");
@@ -1101,7 +1102,7 @@ yang_spec_load_dir(clicon_handle h,
 	clicon_log(LOG_WARNING, "%s: No yang files found in %s",
 		   __FUNCTION__, dir);
     /* Apply post steps on new modules, ie ones after modnr. */
-    modnr = yspec->ys_len;
+    modnr = yang_len_get(yspec);
     /* Load all yang files in dir */
     for (i = 0; i < ndp; i++) {
 	/* base = module name [+ @rev ] + .yang */
@@ -1151,7 +1152,7 @@ yang_spec_load_dir(clicon_handle h,
 	if (revm && rev0){
 	    if (revm > rev0) /* Loaded module is older or eq -> remove ym */
 		ym = ym0;
-	    for (j=0; j<yspec->ys_len; j++)
+	    for (j=0; j<yang_len_get(yspec); j++)
 		if (yspec->ys_stmt[j] == ym)
 		    break;
 	    ys_prune(yspec, j);

test/test_augment.sh

@@ -227,7 +227,8 @@ XML=$(cat <<EOF
 EOF
    )
 
-# Test for multi-module path where an augment stretches across modules
+# XXX: Since derived-from etc are NOT implemented, this test may have false positives
+# revisit when it is implemented.
 new "restconf PUT augment multi-namespace path e1 (whole path)"
 expectpart "$(curl -s -X PUT -H 'Content-Type: application/yang-data+xml' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e1 -d "$XML")" 0 ''
 
@@ -239,9 +240,9 @@ EOF
 new "restconf POST augment multi-namespace path e2 (middle path)"
 expectpart "$(curl -s -X POST -H 'Content-Type: application/yang-data+xml' http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e2 -d "$XML" )" 0 ''
 
-# XXX: All interfaces should have ospf, but only 1 or 2 have?
+
 new "restconf GET augment multi-namespace top"
-#expectpart "$(curl -si -X GET http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 'HTTP/1.1 200 OK' '{"ietf-interfaces:interfaces":{"interface":\[{"name":"e1","type":"example-augment:some-new-iftype","example-augment:ospf":{"reference-bandwidth":23},"example-augment:mandatory-leaf":"true","example-augment:port":80,"example-augment:lport":8080},{"name":"e2","type":"fddi","example-augment:ospf":{"reference-bandwidth":23},"example-augment:mandatory-leaf":"true","example-augment:other":"ietf-interfaces:fddi","example-augment:port":80,"example-augment:lport":8080},{"name":"e3","type":"fddi","example-augment:mandatory-leaf":"true","example-augment:me":"you","example-augment:port":80,"example-augment:lport":8080}\]}}'
+expectpart "$(curl -si -X GET http://localhost/restconf/data/ietf-interfaces:interfaces)" 0 'HTTP/1.1 200 OK' '{"ietf-interfaces:interfaces":{"interface":\[{"name":"e1","type":"example-augment:some-new-iftype","example-augment:ospf":{"reference-bandwidth":23},"example-augment:mandatory-leaf":"true","example-augment:port":80,"example-augment:lport":8080},{"name":"e2","type":"fddi","example-augment:ospf":{"reference-bandwidth":23},"example-augment:mandatory-leaf":"true","example-augment:other":"ietf-interfaces:fddi","example-augment:port":80,"example-augment:lport":8080},{"name":"e3","type":"fddi","example-augment:mandatory-leaf":"true","example-augment:me":"you","example-augment:port":80,"example-augment:lport":8080}\]}}'
 
 new "restconf GET augment multi-namespace level 1"
 expectpart "$(curl -si -X GET http://localhost/restconf/data/ietf-interfaces:interfaces/interface=e1)" 0 'HTTP/1.1 200 OK' '{"ietf-interfaces:interface":\[{"name":"e1","type":"example-augment:some-new-iftype","example-augment:ospf":{"reference-bandwidth":23},"example-augment:mandatory-leaf":"true","example-augment:port":80,"example-augment:lport":8080}\]}'

test/test_perf.sh

@@ -11,10 +11,10 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 : ${format:=xml}
 
 # Number of list/leaf-list entries in file
-: ${perfnr:=1000}
+: ${perfnr:=10000}
 
 # Number of requests made get/put
-: ${perfreq:=100}
+: ${perfreq:=10}
 
 # time function (this is a mess to get right on freebsd/linux)
 # -f %e gives elapsed wall clock time but is not available on all systems

test/test_perf_state.sh

@@ -14,7 +14,7 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 : ${perfnr:=1000}
 
 # Number of requests made get/put
-: ${perfreq:=100}
+: ${perfreq:=10}
 
 # time function (this is a mess to get right on freebsd/linux)
 : ${TIMEFN:=time -p} # portability: 2>&1 | awk '/real/ {print $2}'

test/test_stream.sh

@@ -118,49 +118,51 @@ fi
 new "waiting"
 wait_backend
 
-new "kill old restconf daemon"
+if [ $RC -ne 0 ]; then
-sudo pkill -u $wwwuser -f clixon_restconf
+    new "kill old restconf daemon"
+    sudo pkill -u $wwwuser -f clixon_restconf
       
-new "start restconf daemon"
+    new "start restconf daemon"
-start_restconf -f $cfg
+    start_restconf -f $cfg
 
-new "waiting"
+    new "waiting"
-wait_restconf
+    wait_restconf
+fi
 
 #
 # 1. Netconf RFC5277 stream testing
 new "1. Netconf RFC5277 stream testing"
 # 1.1 Stream discovery
 new "netconf event stream discovery RFC5277 Sec 3.2.5"
-expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get><filter type="xpath" select="n:netconf/n:streams" xmlns:n="urn:ietf:params:xml:ns:netmod:notification"/></get></rpc>]]>]]>' '<rpc-reply><data><netconf xmlns="urn:ietf:params:xml:ns:netmod:notification"><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>true</replay-support></stream></streams></netconf></data></rpc-reply>]]>]]>'
+expecteof "$clixon_netconf -D $DBG -qf $cfg" 0 '<rpc><get><filter type="xpath" select="n:netconf/n:streams" xmlns:n="urn:ietf:params:xml:ns:netmod:notification"/></get></rpc>]]>]]>' '<rpc-reply><data><netconf xmlns="urn:ietf:params:xml:ns:netmod:notification"><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>true</replay-support></stream></streams></netconf></data></rpc-reply>]]>]]>'
 
 new "netconf event stream discovery RFC8040 Sec 6.2"
-expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><get><filter type="xpath" select="r:restconf-state/r:streams" xmlns:r="urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"/></get></rpc>]]>]]>' '<rpc-reply><data><restconf-state xmlns="urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>true</replay-support><access><encoding>xml</encoding><location>https://localhost/streams/EXAMPLE</location></access></stream></streams></restconf-state></data></rpc-reply>]]>]]>'
+expecteof "$clixon_netconf -D $DBG -qf $cfg" 0 '<rpc><get><filter type="xpath" select="r:restconf-state/r:streams" xmlns:r="urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"/></get></rpc>]]>]]>' '<rpc-reply><data><restconf-state xmlns="urn:ietf:params:xml:ns:yang:ietf-restconf-monitoring"><streams><stream><name>EXAMPLE</name><description>Example event stream</description><replay-support>true</replay-support><access><encoding>xml</encoding><location>https://localhost/streams/EXAMPLE</location></access></stream></streams></restconf-state></data></rpc-reply>]]>]]>'
 
 #
 # 1.2 Netconf stream subscription
 new "netconf EXAMPLE subscription"
-expectwait "$clixon_netconf -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' $NCWAIT
+expectwait "$clixon_netconf -D $DBG -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' $NCWAIT
 
 new "netconf subscription with empty startTime"
-expecteof "$clixon_netconf -qf $cfg" 0 '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream><startTime/></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>bad-element</error-tag><error-info><bad-element>startTime</bad-element></error-info><error-severity>error</error-severity><error-message>regexp match fail:'
+expecteof "$clixon_netconf -D $DBG -qf $cfg" 0 '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream><startTime/></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>bad-element</error-tag><error-info><bad-element>startTime</bad-element></error-info><error-severity>error</error-severity><error-message>regexp match fail:'
 
 new "netconf EXAMPLE subscription with simple filter"
-expectwait "$clixon_netconf -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream><filter type="xpath" select="event"/></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' $NCWAIT
+expectwait "$clixon_netconf -D $DBG -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream><filter type="xpath" select="event"/></create-subscription></rpc>]]>]]>' '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' $NCWAIT
 
 new "netconf EXAMPLE subscription with filter classifier"
-expectwait "$clixon_netconf -qf $cfg" "<rpc><create-subscription xmlns=\"urn:ietf:params:xml:ns:netmod:notification\"><stream>EXAMPLE</stream><filter type=\"xpath\" select=\"event[event-class='fault']\"/></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' $NCWAIT
+expectwait "$clixon_netconf -D $DBG -qf $cfg" "<rpc><create-subscription xmlns=\"urn:ietf:params:xml:ns:netmod:notification\"><stream>EXAMPLE</stream><filter type=\"xpath\" select=\"event[event-class='fault']\"/></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' $NCWAIT
 
 new "netconf NONEXIST subscription"
-expectwait "$clixon_netconf -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>NONEXIST</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>invalid-value</error-tag><error-severity>error</error-severity><error-message>No such stream</error-message></rpc-error></rpc-reply>]]>]]>$' $NCWAIT
+expectwait "$clixon_netconf -D $DBG -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>NONEXIST</stream></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>invalid-value</error-tag><error-severity>error</error-severity><error-message>No such stream</error-message></rpc-error></rpc-reply>]]>]]>$' $NCWAIT
 
 new "netconf EXAMPLE subscription with wrong date"
-expectwait "$clixon_netconf -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream><startTime>kallekaka</startTime></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>bad-element</error-tag><error-info><bad-element>startTime</bad-element></error-info><error-severity>error</error-severity><error-message>regexp match fail:' 0
+expectwait "$clixon_netconf -D $DBG -qf $cfg" '<rpc><create-subscription xmlns="urn:ietf:params:xml:ns:netmod:notification"><stream>EXAMPLE</stream><startTime>kallekaka</startTime></create-subscription></rpc>]]>]]>' '^<rpc-reply><rpc-error><error-type>application</error-type><error-tag>bad-element</error-tag><error-info><bad-element>startTime</bad-element></error-info><error-severity>error</error-severity><error-message>regexp match fail:' 0
 
 #new "netconf EXAMPLE subscription with replay"
 #NOW=$(date +"%Y-%m-%dT%H:%M:%S")
 #sleep 10
-#expectwait "$clixon_netconf -qf $cfg" "<rpc><create-subscription xmlns=\"urn:ietf:params:xml:ns:netmod:notification\"><stream>EXAMPLE</stream><startTime>$NOW</startTime></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 10
+#expectwait "$clixon_netconf -D $DBG -qf $cfg" "<rpc><create-subscription xmlns=\"urn:ietf:params:xml:ns:netmod:notification\"><stream>EXAMPLE</stream><startTime>$NOW</startTime></create-subscription></rpc>]]>]]>" '^<rpc-reply><ok/></rpc-reply>]]>]]><notification xmlns="urn:ietf:params:xml:ns:netconf:notification:1.0"><eventTime>20' 10
 sleep 1
 
 #
@@ -286,8 +288,10 @@ echo "Eg: curl -H \"Accept: text/event-stream\" -s -X GET http://localhost/sub/E
 
 #-----------------
 sleep 5
-new "Kill restconf daemon"
+if [ $RC -ne 0 ]; then
-stop_restconf 
+    new "Kill restconf daemon"
+    stop_restconf
+fi
 
 if [ $BE -eq 0 ]; then
     exit # BE

yang/clixon/Makefile.in

@@ -42,6 +42,7 @@ datarootdir     = @datarootdir@
 YANG_INSTALLDIR   = @YANG_INSTALLDIR@
 
 YANGSPECS	 = clixon-config@2019-09-11.yang
+YANGSPECS	+= clixon-config@2020-02-22.yang
 YANGSPECS	+= clixon-lib@2019-08-13.yang
 YANGSPECS	+= clixon-rfc5277@2008-07-01.yang
 YANGSPECS	+= clixon-xml-changelog@2019-03-21.yang

yang/clixon/clixon-config@2019-09-11.yang

@@ -12,7 +12,7 @@ module clixon-config {
     description
       "Clixon configuration file
        ***** BEGIN LICENSE BLOCK *****
-       Copyright (C) 2009-2020 Olof Hagsand
+       Copyright (C) 2009-2019 Olof Hagsand
        
        This file is part of CLIXON
 
@@ -67,10 +67,6 @@ module clixon-config {
 	description
 	    "Released in Clixon 3.8";
     }
-    extension search_index {
-      description "This list argument acts as a search index using optimized binary search.
-                  ";
-    }
     typedef startup_mode{
 	description
 	    "Which method to boot/start clicon backend.

yang/clixon/clixon-config@2020-02-22.yang

@@ -667,6 +667,34 @@ module clixon-config {
                          data to store before dropping. 0 means no retention";
 
 	}
+    }
+    container clixon-stats{
+	config false;
+	description "Clixon backend statistics.";
+	container global{
+	    description "Clixon global statistics";
+	    leaf nr{
+		description "Number of cxobj:ects. That is number of residing xml/json objects
+                             in the internal 'cxobj' representation.";
+		type uint64;
+	    }
 
 	}
+	list datastore{
+	    description "Datastore statistics";
+	    key "name";
+	    leaf name{
+		description "name of datastore (eg running).";
+		type string;
+	    }
+	    leaf nr{
+		description "Number bytes of internal datastore cache of datastore tree.";
+		type uint64;
+	    }
+	    leaf size{
+		description "Size in bytes of internal datastore cache of datastore tree.";
+		type uint64;
+	    }
+	}
+    }
 }