Added event_poll function; optimization experiments

apps/netconf/netconf_main.c

@@ -170,6 +170,9 @@ ed");
 /*! Get netconf message: detect end-of-msg 
  * @param[in]   s    Socket where input arrived. read from this.
  * @param[in]   arg  Clicon handle.
+ * This routine continuously reads until no more data on s. There could
+ * be risk of starvation, but the netconf client does little else than
+ * read data so I do not see a danger of true starvation here.
  */
 static int
 netconf_input_cb(int   s, 
@@ -182,6 +185,7 @@ netconf_input_cb(int   s,
     int           len;
     cbuf         *cb=NULL;
     int           xml_state = 0;
+    int           poll;
 
     if ((cb = cbuf_new()) == NULL){
 	clicon_err(OE_XML, errno, "%s: cbuf_new", __FUNCTION__);
@@ -200,6 +204,7 @@ netconf_input_cb(int   s,
 	if (len == 0){ 	/* EOF */
 	    cc_closed++;
 	    close(s);
+	    clicon_log(LOG_ERR, "read close", __FUNCTION__);
 	    retval = 0;
 	    goto done;
 	}
@@ -220,7 +225,12 @@ netconf_input_cb(int   s,
 		cbuf_reset(cb);
 	    }
 	}
-    }
+	/* poll==1 if more, poll==0 if none */
+	if ((poll = event_poll(s)) < 0)
+	    goto done;
+	if (poll == 0)
+	    break; /* No data to read */
+    } /* while */
     retval = 0;
   done:
     if (cb)

datastore/text/clixon_xmldb_text.c

@@ -425,6 +425,13 @@ match_base_child(cxobj     *x0,
 	if ((cvk = yang_arg2cvec(ykey, " ")) == NULL)
 	    goto done;
 	x0c = NULL;
+	/* XXX: room for optimization? on 1K calls we have 1M body calls and
+	   500K xml_child_each/cvec_each calls. 
+	   The outer loop is large for large lists
+	   The inner loop is small
+	   Major time in xml_find_body()
+	   Can one do a binary search in the x0 list?
+	*/
 	while ((x0c = xml_child_each(x0, x0c, CX_ELMNT)) != NULL) {
 	    if (strcmp(xml_name(x0c), cname))
 		continue;
@@ -482,6 +489,8 @@ text_modify(cxobj              *x0,
     cxobj     *x1c; /* mod child */
     char      *x1bstr; /* mod body string */
     yang_stmt *yc;  /* yang child */
+    cxobj    **x0vec = NULL;
+    int        i;
 
     assert(x1 && xml_type(x1) == CX_ELMNT);
     assert(y0);
@@ -576,6 +585,38 @@ text_modify(cxobj              *x0,
 		if (op==OP_NONE)
 		    xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */
 	    }
+#if 0 		/* Find x1c in x0 */
+	    /* First pass: mark existing children in base */
+	    /* Loop through children of the modification tree */
+	    if ((x0vec = calloc(xml_child_nr(x1), sizeof(x1))) == NULL){
+		clicon_err(OE_UNIX, errno, "calloc");
+		goto done;
+	    }
+	    x1c = NULL; 
+	    i = 0;
+	    while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
+		x1cname = xml_name(x1c);
+		/* Get yang spec of the child */
+		if ((yc = yang_find_datanode(y0, x1cname)) == NULL){
+		    clicon_err(OE_YANG, errno, "No yang node found: %s", x1cname);
+		    goto done;
+		}
+		/* See if there is a corresponding node in the base tree */
+		x0c = NULL;
+		if (match_base_child(x0, x1c, yc, &x0c) < 0)
+		    goto done;
+		x0vec[i++] = x0c;
+	    }
+	    /* Second pass: modify tree */
+	    x1c = NULL;
+	    i = 0;
+	    while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
+		yc = yang_find_datanode(y0, x1cname);
+		if (text_modify(x0vec[i++], (yang_node*)yc, x0, x1c, op) < 0)
+		    goto done;
+	    }
+#else
+	    i = 0; i = i; /* XXX */
 	    /* Loop through children of the modification tree */
 	    x1c = NULL;
 	    while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
@@ -587,11 +628,12 @@ text_modify(cxobj              *x0,
 		}
 		/* See if there is a corresponding node in the base tree */
 		x0c = NULL;
-		if (yc && match_base_child(x0, x1c, yc, &x0c) < 0)
+		if (match_base_child(x0, x1c, yc, &x0c) < 0)
 		    goto done;
 		if (text_modify(x0c, (yang_node*)yc, x0, x1c, op) < 0)
 		    goto done;
 	    }
+#endif
 	    break;
 	case OP_DELETE:
 	    if (x0==NULL){
@@ -609,6 +651,8 @@ text_modify(cxobj              *x0,
     // ok:
     retval = 0;
  done:
+    if (x0vec)
+	free(x0vec);
     return retval;
 }
 

lib/clixon/clixon_event.h

@@ -54,6 +54,8 @@ int event_reg_timeout(struct timeval t,  int (*fn)(int, void*),
 
 int event_unreg_timeout(int (*fn)(int, void*), void *arg);
 
+int event_poll(int fd);
+
 int event_loop(void);
 
 int event_exit(void);

lib/src/clixon_event.c

@@ -251,6 +251,26 @@ event_unreg_timeout(int (*fn)(int, void*),
     return found?0:-1;
 }
 
+/*! Poll to see if there is any data available on this file descriptor.
+ * @param[in]  fd   File descriptor
+ * @retval    -1    Error
+ * @retval     0    Nothing to read/empty fd
+ * @retval     1    Something to read on fd
+ */
+int 
+event_poll(int fd)
+{
+    int            retval = -1;
+    fd_set         fdset;
+    struct timeval tnull = {0,};
+
+    FD_ZERO(&fdset);
+    FD_SET(fd, &fdset);
+    if ((retval = select(FD_SETSIZE, &fdset, NULL, NULL, &tnull)) < 0)
+	clicon_err(OE_EVENTS, errno, "%s select1: %s", __FUNCTION__, strerror(errno));
+    return retval;
+}
+
 /*! Dispatch file descriptor events (and timeouts) by invoking callbacks.
  * There is an issue with fairness that timeouts may take over all events
  * One could try to poll the file descriptors after a timeout?
@@ -258,9 +278,12 @@ event_unreg_timeout(int (*fn)(int, void*),
 int
 event_loop(void)
 {
-    struct event_data *e, *e_next;
+    struct event_data *e;
+    struct event_data *e_next;
     int                n;
-    struct timeval t, t0, tnull={0,};
+    struct timeval     t;
+    struct timeval     t0;
+    struct timeval     tnull = {0,};
     fd_set             fdset;
     int                retval = -1;
 

test/lib.sh

@@ -80,6 +80,24 @@ EOF
   fi
 }
 
+# clixon tester read from file for large tests
+expecteof_file(){
+  cmd=$1
+  file=$2
+  expect=$3
+
+# Do while read stuff
+ret=$($cmd<$file)
+  # Match if both are empty string
+  if [ -z "$ret" -a -z "$expect" ]; then
+      return
+  fi
+  match=`echo "$ret" | grep -Eo "$expect"`
+  if [ -z "$match" ]; then
+      err "$expect" "$ret"
+  fi
+}
+
 # clixon tester. First arg is command second is stdin and
 # third is expected outcome, fourth is how long to wait
 expectwait(){

test/test_perf.sh

@@ -1,42 +1,73 @@
 #!/bin/bash
-# Test2: backend and netconf basic functionality
+# Scaling test
 
-number=10000
+if [ $# = 0 ]; then
+    number=1000
+elif [ $# = 1 ]; then
+    number=$1
+else
+    echo "Usage: $0 [<number>]"
+    exit 1
+fi
+
+fyang=/tmp/scaling.yang
+fconfig=/tmp/config
 
 # include err() and new() functions
 . ./lib.sh
 
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+# clixon_netconf="valgrind --tool=callgrind clixon_netconf 
 clixon_netconf=clixon_netconf
 
+cat <<EOF > $fyang
+module ietf-ip{
+   container x {
+    list y {
+      key "a";
+      leaf a {
+        type string;
+      }
+    }
+    container z {
+      leaf-list c {
+        type string;
+      }
+    }
+  }
+}
+EOF
+
 # kill old backend (if any)
 new "kill old backend"
-sudo clixon_backend -zf $clixon_cf
+echo "clixon_backend -zf $clixon_cf -y $fyang"
+sudo clixon_backend -zf $clixon_cf -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
+
 new "start backend"
 # start new backend
-sudo clixon_backend -If $clixon_cf
+sudo clixon_backend -If $clixon_cf -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
 
-new "netconf perf tests"
+new "generate large config"
-
+echo -n "<rpc><edit-config><target><candidate/></target><config><x>" > $fconfig
-str="<rpc><edit-config><target><candidate/></target><config><interfaces>"
+for (( i=0; i<$number; i++ )); do  
-for (( i=0; i<$number; i++ ))
+    echo -n "<y><a>$i</a></y>" >> $fconfig
-do  
-   str+="<interface><name>eth$i</name></interface>"
 done
-str+="</interfaces></config></edit-config></rpc>]]>]]>"
+echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
 
 new "netconf edit large config"
-expecteof "$clixon_netconf -qf $clixon_cf" "$str" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof_file "time $clixon_netconf -qf $clixon_cf -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" < $fconfig
+
+rm $fconfig
 
 new "netconf get large config"
-expecteof "$clixon_netconf -qf $clixon_cf" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><interfaces><interface><name>eth0</name><enabled>true</enabled></interface>"
+expecteof "time $clixon_netconf -qf $clixon_cf  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>0</a></y><y><a>1</a>"
 
 new "Kill backend"
 # Check if still alive