Test: make streams optional, remove dependency on main example

CHANGELOG.md

@@ -119,6 +119,7 @@ e`
 
 ### Corrected Bugs
 
+* [Replace operation](https://github.com/clicon/clixon/issues/350)
 * Fixed: [When multiple lists have same key name, need more elaborate error message in case of configuration having duplicate keys](https://github.com/clicon/clixon/issues/362)
   * Solved by implementing RFC7950 Sec 5.1 correctly
 * Fixed: [All values in list don't appear when writing "show <list>" in cli](https://github.com/clicon/clixon/issues/359)

example/main/example_backend.c

@@ -36,6 +36,7 @@
   * The example have the following optional arguments that you can pass as 
   * argc/argv after -- in clixon_backend:
   *  -a <..> Register callback for this yang action
+  *  -n  Notification streams example
   *  -r  enable the reset function 
   *  -s  enable the state function
   *  -S <file>  read state data from file, otherwise construct it programmatically (requires -s)
@@ -67,7 +68,7 @@
 #include <clixon/clixon_backend.h> 
 
 /* Command line options to be passed to getopt(3) */
-#define BACKEND_EXAMPLE_OPTS "a:rsS:x:iuUtV:"
+#define BACKEND_EXAMPLE_OPTS "a:nrsS:x:iuUtV:"
 
 /*! Yang action
  * Start backend with -- -a <instance-id>
@@ -76,6 +77,12 @@
  */
 static char *_action_instanceid = NULL;
 
+/*! Notification stream
+ * Enable notification streams for netconf/restconf 
+ * Start backend with -- -n
+ */
+static int _notification_stream = 0;
+
 /*! Variable to control if reset code is run.
  * The reset code inserts "extra XML" which assumes ietf-interfaces is
  * loaded, and this is not always the case.
@@ -1314,6 +1321,9 @@ clixon_plugin_init(clicon_handle h)
 	case 'a':
 	    _action_instanceid = optarg;
 	    break;
+	case 'n':
+	    _notification_stream = 1;
+	    break;
 	case 'r':
 	    _reset = 1;
 	    break;
@@ -1355,24 +1365,25 @@ clixon_plugin_init(clicon_handle h)
 	}
     }
 	
-    /* Example stream initialization:
+    if (_notification_stream){
-     * 1) Register EXAMPLE stream 
+	/* Example stream initialization:
-     * 2) setup timer for notifications, so something happens on stream
+	 * 1) Register EXAMPLE stream 
-     * 3) setup stream callbacks for notification to push channel
+	 * 2) setup timer for notifications, so something happens on stream
-     */
+	 * 3) setup stream callbacks for notification to push channel
-    if (clicon_option_exists(h, "CLICON_STREAM_RETENTION"))
+	 */
-	retention.tv_sec = clicon_option_int(h, "CLICON_STREAM_RETENTION");
+	if (clicon_option_exists(h, "CLICON_STREAM_RETENTION"))
-    if (stream_add(h, "EXAMPLE", "Example event stream", 1, &retention) < 0)
+	    retention.tv_sec = clicon_option_int(h, "CLICON_STREAM_RETENTION");
-	goto done;
+	if (stream_add(h, "EXAMPLE", "Example event stream", 1, &retention) < 0)
-    /* Enable nchan pub/sub streams
+	    goto done;
-     * assumes: CLIXON_PUBLISH_STREAMS, eg configure --enable-publish
+	/* Enable nchan pub/sub streams
-     */
+	 * assumes: CLIXON_PUBLISH_STREAMS, eg configure --enable-publish
-    if (clicon_option_exists(h, "CLICON_STREAM_PUB") &&
+	 */
-	stream_publish(h, "EXAMPLE") < 0)
+	if (clicon_option_exists(h, "CLICON_STREAM_PUB") &&
-	goto done;
+	    stream_publish(h, "EXAMPLE") < 0)
-    if (example_stream_timer_setup(h) < 0)
+	    goto done;
-	goto done;
+	if (example_stream_timer_setup(h) < 0)
-
+	    goto done;
+    }
     /* Register callback for routing rpc calls 
      */
     /* From example.yang (clicon) */

test/config.sh.in

@@ -76,7 +76,6 @@ CLIXON_AUTOCLI_REV="2022-02-11"
 CLIXON_LIB_REV="2021-12-05"
 CLIXON_CONFIG_REV="2022-03-21"
 CLIXON_RESTCONF_REV="2022-08-01"
-CLIXON_EXAMPLE_REV="2020-12-01"
 
 # Length of TSL RSA key
 # Problem with small key such as 1024 not allowed in centos8 for example (why is this)

test/test_netconf.sh

@@ -11,6 +11,7 @@ APPNAME=example
 
 cfg=$dir/conf_yang.xml
 tmp=$dir/tmp.x
+fyang=$dir/clixon-example.yang
 
 # Use yang in example
 
@@ -19,9 +20,10 @@ cat <<EOF > $cfg
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
   <CLICON_FEATURE>ietf-netconf:startup</CLICON_FEATURE>
   <CLICON_MODULE_SET_ID>42</CLICON_MODULE_SET_ID>
+  <CLICON_YANG_DIR>$dir</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
   <CLICON_YANG_DIR>$IETFRFC</CLICON_YANG_DIR>
-  <CLICON_YANG_MODULE_MAIN>clixon-example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>	
   <CLICON_CLISPEC_DIR>/usr/local/lib/$APPNAME/clispec</CLICON_CLISPEC_DIR>
   <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
   <CLICON_BACKEND_REGEXP>example_backend.so$</CLICON_BACKEND_REGEXP>
@@ -36,6 +38,101 @@ cat <<EOF > $cfg
 </clixon-config>
 EOF
 
+cat <<EOF > $fyang
+module clixon-example{
+   yang-version 1.1;
+   namespace "urn:example:clixon";
+   prefix ex;
+   import ietf-interfaces { 
+	prefix if;
+   }
+   import ietf-ip {
+	prefix ip;
+   }
+   /* Example interface type for tests, local callbacks, etc */
+   identity eth {
+	base if:interface-type;
+   }
+    /* Generic config data */
+    container table{
+	list parameter{
+	    key name;
+	    leaf name{
+		type string;
+	    }
+	}
+    }
+   /* State data (not config) for the example application*/
+   container state {
+	config false;
+	description "state data for the example application (must be here for example get operation)";
+	leaf-list op {
+            type string;
+	}
+   }
+   augment "/if:interfaces/if:interface" {
+	container my-status {
+	    config false;
+	    description "For testing augment+state";
+	    leaf int {
+		type int32;
+	    }
+	    leaf str {
+		type string;
+	    }
+	}
+    }
+    rpc client-rpc {
+	description "Example local client-side RPC that is processed by the
+                     the netconf/restconf and not sent to the backend.
+                     This is a clixon implementation detail: some rpc:s
+                     are better processed by the client for API or perf reasons";
+	input {
+	    leaf x {
+		type string;
+	    }
+	}
+	output {
+	    leaf x {
+		type string;
+	    }
+	}
+    }
+    rpc empty {
+	description "Smallest possible RPC with no input or output sections";
+    }
+    rpc example {
+	description "Some example input/output for testing RFC7950 7.14.
+                     RPC simply echoes the input for debugging.";
+	input {
+	    leaf x {
+		description
+         	    "If a leaf in the input tree has a 'mandatory' statement with
+                   the value 'true', the leaf MUST be present in an RPC invocation.";
+		type string;
+		mandatory true;
+	    }
+	    leaf y {
+		description
+		    "If a leaf in the input tree has a 'mandatory' statement with the
+                  value 'true', the leaf MUST be present in an RPC invocation.";
+		type string;
+		default "42";
+	    }
+	}
+	output {
+	    leaf x {
+		type string;
+	    }
+	    leaf y {
+		type string;
+	    }
+	}
+    }
+
+}
+EOF
+
 new "test params: -f $cfg -- -s"
 # Bring your own backend
 if [ $BE -ne 0 ]; then

test/test_netconf_notifications.sh

@@ -97,8 +97,8 @@ if [ $BE -ne 0 ]; then
     if [ $? -ne 0 ]; then
 	err
     fi
-    new "start backend -s init -f $cfg"
+    new "start backend -s init -f $cfg -- -n"
-    start_backend -s init -f $cfg
+    start_backend -s init -f $cfg -- -n # create example notification stream
 fi
 
 new "waiting"

test/test_restconf.sh

@@ -27,17 +27,11 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 APPNAME=example
 
 cfg=$dir/conf.xml
+fyang=$dir/clixon-example.yang
 
-# clixon-example and clixon-restconf is used in the test, need local copy
+# clixon-restconf is used in the test, need local copy
 # This is a kludge: look in src otherwise assume it is installed in /usr/local/share
 # Note that revisions may change and may need to be updated
-y="clixon-example@${CLIXON_EXAMPLE_REV}.yang"
-
-if [ -d ${TOP_SRCDIR}/example/main/$y ]; then 
-    cp ${TOP_SRCDIR}/example/main/$y $dir/
-else
-    cp /usr/local/share/clixon/$y $dir/
-fi
 y=clixon-restconf@${CLIXON_RESTCONF_REV}.yang
 if [ -d ${TOP_SRCDIR}/yang/clixon ]; then 
     cp ${TOP_SRCDIR}/yang/clixon/$y $dir/
@@ -119,6 +113,146 @@ cat <<EOF > $cfg
 </clixon-config>
 EOF
 
+cat <<EOF > $fyang
+module clixon-example {
+    yang-version 1.1;
+    namespace "urn:example:clixon";
+    prefix ex;
+    import ietf-interfaces { 
+	/* is in yang/optional which means clixon must be installed using --opt-yang-installdir */
+	prefix if;
+    }
+    import ietf-ip {
+	prefix ip;
+    }
+    import iana-if-type {
+	prefix ianaift;
+    }
+    import ietf-datastores {
+	prefix ds;
+    }
+    import clixon-autocli{
+	prefix autocli;
+    }
+    description
+	"Clixon example used as a part of the Clixon test suite.
+         It can be used as a basis for making new Clixon applications.
+         Note, may change without updating revision, just for testing current master.
+         ";
+    /* Example interface type for tests, local callbacks, etc */
+    identity eth {
+	base if:interface-type;
+    }
+    identity loopback {
+	base if:interface-type;
+    }
+    /* Generic config data */
+    container table{
+	list parameter{
+	    key name;
+	    leaf name{
+		type string;
+	    }
+	    leaf value{
+		type string;
+	    }
+	    leaf hidden{
+		type string;
+		autocli:hide;
+	    }
+	    leaf stat{
+		description "Inline state data for example application";
+		config false;
+		type int32;
+	    }
+	}
+    }
+    /* State data (not config) for the example application*/
+    container state {
+	config false;
+	description "state data for the example application (must be here for example get operation)";
+	leaf-list op {
+            type string;
+	}
+    }
+    augment "/if:interfaces/if:interface" {
+	container my-status {
+	    config false;
+	    description "For testing augment+state";
+	    leaf int {
+		type int32;
+	    }
+	    leaf str {
+		type string;
+	    }
+	}
+    }
+    /* yang extension implemented by the example backend code. */
+    extension e4 {
+	description
+	    "The first child of the ex:e4 (unknown) statement is inserted into 
+	    the module as a regular data statement. This means that 'uses bar;'
+	    in the ex:e4 statement below is a valid data node";
+	argument arg;
+    }
+    grouping bar {
+	leaf bar{
+	    type string;
+	}
+    }
+    ex:e4 arg1{
+	uses bar;
+    }
+    rpc client-rpc {
+	description "Example local client-side RPC that is processed by the
+                     the netconf/restconf and not sent to the backend.
+                     This is a clixon implementation detail: some rpc:s
+                     are better processed by the client for API or perf reasons";
+	input {
+	    leaf x {
+		type string;
+	    }
+	}
+	output {
+	    leaf x {
+		type string;
+	    }
+	}
+    }
+    rpc empty {
+	description "Smallest possible RPC with no input or output sections";
+    }
+    rpc example {
+	description "Some example input/output for testing RFC7950 7.14.
+                     RPC simply echoes the input for debugging.";
+	input {
+	    leaf x {
+		description
+         	    "If a leaf in the input tree has a 'mandatory' statement with
+                   the value 'true', the leaf MUST be present in an RPC invocation.";
+		type string;
+		mandatory true;
+	    }
+	    leaf y {
+		description
+		    "If a leaf in the input tree has a 'mandatory' statement with the
+                  value 'true', the leaf MUST be present in an RPC invocation.";
+		type string;
+		default "42";
+	    }
+	}
+	output {
+	    leaf x {
+		type string;
+	    }
+	    leaf y {
+		type string;
+	    }
+         }
+   }
+}
+EOF
+
 # Restconf test routine with arguments:
 # 1. proto:http/https
 # 2: addr: 127.0.0.1/::1 # IPv4 or IPv6
@@ -320,11 +454,11 @@ function testrun()
 
     # Should be alphabetically ordered
     new "restconf get restconf/operations. RFC8040 3.3.2 (json)"
-    expectpart "$(curl $CURLOPTS -X GET $proto://$addr/restconf/operations)" 0 "HTTP/$HVER 200" '{"operations":{' '"clixon-example:client-rpc":\[null\],"clixon-example:empty":\[null\],"clixon-example:optional":\[null\],"clixon-example:example":\[null\]' '"clixon-lib:debug":\[null\],"clixon-lib:ping":\[null\],"clixon-lib:stats":\[null\],"clixon-lib:restart-plugin":\[null\]' '"ietf-netconf:get-config":\[null\],"ietf-netconf:edit-config":\[null\],"ietf-netconf:copy-config":\[null\],"ietf-netconf:delete-config":\[null\],"ietf-netconf:lock":\[null\],"ietf-netconf:unlock":\[null\],"ietf-netconf:get":\[null\],"ietf-netconf:close-session":\[null\],"ietf-netconf:kill-session":\[null\],"ietf-netconf:commit":\[null\],"ietf-netconf:discard-changes":\[null\],"ietf-netconf:validate":\[null\]'
+    expectpart "$(curl $CURLOPTS -X GET $proto://$addr/restconf/operations)" 0 "HTTP/$HVER 200" '{"operations":{' '"clixon-example:empty":\[null\]' '"clixon-lib:debug":\[null\],"clixon-lib:ping":\[null\],"clixon-lib:stats":\[null\],"clixon-lib:restart-plugin":\[null\]' '"ietf-netconf:get-config":\[null\],"ietf-netconf:edit-config":\[null\],"ietf-netconf:copy-config":\[null\],"ietf-netconf:delete-config":\[null\],"ietf-netconf:lock":\[null\],"ietf-netconf:unlock":\[null\],"ietf-netconf:get":\[null\],"ietf-netconf:close-session":\[null\],"ietf-netconf:kill-session":\[null\],"ietf-netconf:commit":\[null\],"ietf-netconf:discard-changes":\[null\],"ietf-netconf:validate":\[null\]'
 
     new "restconf get restconf/operations. RFC8040 3.3.2 (xml)"
     ret=$(curl $CURLOPTS -X GET -H "Accept: application/yang-data+xml" $proto://$addr/restconf/operations)
-    expect='<operations><client-rpc xmlns="urn:example:clixon"/><empty xmlns="urn:example:clixon"/><optional xmlns="urn:example:clixon"/><example xmlns="urn:example:clixon"/>'
+    expect='<operations><client-rpc xmlns="urn:example:clixon"/><empty xmlns="urn:example:clixon"/>'
     match=`echo $ret | grep --null -Eo "$expect"`
     if [ -z "$match" ]; then
 	err "$expect" "$ret"
@@ -345,7 +479,7 @@ function testrun()
     expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+json' $proto://$addr/restconf/data/ietf-yang-library:yang-library/module-set=default/module=ietf-interfaces)" 0 "HTTP/$HVER 200" '{"ietf-yang-library:module":\[{"name":"ietf-interfaces","revision":"2018-02-20","namespace":"urn:ietf:params:xml:ns:yang:ietf-interfaces"}\]}'
 
     new "restconf schema resource, mod-state top-level"
-    expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+json' $proto://$addr/restconf/data/ietf-yang-library:yang-library/module-set=default)" 0 "HTTP/$HVER 200" "{\"ietf-yang-library:module-set\":\[{\"name\":\"default\",\"module\":\[{\"name\":\"clixon-autocli\",\"revision\":\"${CLIXON_AUTOCLI_REV}\",\"namespace\":\"http://clicon.org/autocli\"},{\"name\":\"clixon-example\",\"revision\":\"${CLIXON_EXAMPLE_REV}\",\"namespace\":\"urn:example:clixon\"},{\"name\":\"clixon-lib\",\"revision\":\"${CLIXON_LIB_REV}\",\""
+    expectpart "$(curl $CURLOPTS -X GET -H 'Accept: application/yang-data+json' $proto://$addr/restconf/data/ietf-yang-library:yang-library/module-set=default)" 0 "HTTP/$HVER 200" "{\"ietf-yang-library:module-set\":\[{\"name\":\"default\",\"module\":\[{\"name\":\"clixon-autocli\",\"revision\":\"${CLIXON_AUTOCLI_REV}\",\"namespace\":\"http://clicon.org/autocli\"}" "{\"name\":\"clixon-lib\",\"revision\":\"${CLIXON_LIB_REV}\",\""
 
     new "restconf options. RFC 8040 4.1"
     expectpart "$(curl $CURLOPTS -X OPTIONS $proto://$addr/restconf/data)" 0 "HTTP/$HVER 200" "Allow: OPTIONS,HEAD,GET,POST,PUT,PATCH,DELETE"

test/test_restconf_notifications.sh

@@ -33,11 +33,6 @@ if [ "${WITH_RESTCONF}" != "fcgi" -o "$RCPROTO" = https ]; then
     if [ "$s" = $0 ]; then exit 0; else return 0; fi # skip
 fi
 
-# Skip regardless, broken in 5.7
-if true; then
-    rm -rf $dir
-    if [ "$s" = $0 ]; then exit 0; else return 0; fi # skip
-fi
 : ${SLEEP2:=1}
 SLEEP5=.5
 APPNAME=example
@@ -139,8 +134,8 @@ if [ $BE -ne 0 ]; then
     if [ $? -ne 0 ]; then
 	err
     fi
-    new "start backend -s init -f $cfg"
+    new "start backend -s init -f $cfg -- -n"
-    start_backend -s init -f $cfg
+    start_backend -s init -f $cfg -- -n # create example notification stream
 fi
 
 new "waiting"
@@ -181,7 +176,6 @@ new "restconf monitor event nonexist stream"
 # partial returns like expectpart can
 expectwait "curl -sk -X GET -H \"Accept: text/event-stream\" -H \"Cache-Control: no-cache\" -H \"Connection: keep-alive\" $RCPROTO://localhost/streams/NOTEXIST" 0 "" "" 2 '<errors xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf\"><error><error-type>application</error-type><error-tag>invalid-value</error-tag><error-severity>error</error-severity><error-message>No such stream</error-message></error></errors>'
 
-
 # 2a) start subscription 8s - expect 1-2 notifications
 new "2a) start subscriptions 8s - expect 1-2 notifications"