Merge branch 'develop' for 3.5.0

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Clixon Changelog
 
+## 3.5.0 (Upcoming)
+
+### Major changes:
+* Major Restconf feature update to compy to RFC 8040. Thanks Stephen Jones for getting right.
+  * GET: Always return object referenced (and nothing else). ie, GET /restconf/data/X returns X. 
+  * GET Added support for the following resources: Well-known, top-level resource, and yang library version,
+  * GET Single element JSON lists use {list:[element]}, not {list:element}.
+  * PUT Whole datastore
+  
+### Minor changes:
+
+* Changed signature of plugin_credentials() restconf callback. Added a "user" parameter. To enable authentication and in preparation for access control a la RFC 6536.
+* Added RFC 6536 ietf-netconf-acm@2012-02-22.yang access control (but not implemented).
+* The following backward compatible options to configure have been _obsoleted_. If you havent already migrated this code you must do this now.
+  * `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted.
+  * `configure --with-config-compat`. The template clicon.conf.cpp files are also removed.
+  * `configure --with-xml-compat`
+  
+* New configuration option: CLICON_RESTCONF_PRETTY. Default true. Set to false to get more compact Restconf output.
+
+
+* Default configure file handling generalized by Renato Botelho/Matt Smith. Config file FILE is selected in the following priority order:
+  * Provide -f FILE option when starting a program (eg clixon_backend -F FILE)
+  * Provide --with-configfile=FILE when configuring
+  * Provide --with-sysconfig=<dir> when configuring, then FILE is <dir>/clixon.xml
+  * Provide --sysconfig=<dir> when configuring then FILE is <dir>/etc/clixon.xml
+  * FILE is /usr/local/etc/clixon.xml
+	
+### Corrected Bugs
+* yang max keyword was not supported for string type. Corrected by setting "max" to MAXPATHLEN
+* Corrected "No yang spec" printed on tty when using leafref in CLI.
+* Fixed error in xml2cvec. If a (for example) int8 value has range error (eg 1000), it was treated as an error and the program terminated. Now this is just logged and skipped. Reported by Fredrik Pettai.
+	
+### Known issues
+
 ## 3.4.0 (1 January 2018)
 
 ### Major changes:

Makefile.in

@@ -56,7 +56,7 @@ SUBDIRS = lib apps include etc datastore yang
 
 .PHONY:	doc all clean depend $(SUBDIRS) install loc TAGS .config.status docker
 
-all:	$(SUBDIRS) clixon.conf.cpp clixon.mk
+all:	$(SUBDIRS) clixon.mk
 
 $(SUBDIRS):
 	(cd $@ && $(MAKE) $(MFLAGS) all)
@@ -65,16 +65,11 @@ depend:
 	for i in $(SUBDIRS) doc example docker; \
 	do (cd $$i && $(MAKE) $(MFLAGS) depend); done
 
-# template clixon.conf file
-clixon.conf.cpp:  clixon.conf.cpp.cpp
-	$(CPP) -P -x assembler-with-cpp -Dprefix=$(prefix) -Dlocalstatedir=$(localstatedir) -Dsysconfdir=$(sysconfdir) -Ddatadir=$(datadir) -Dlibdir=$(libdir) $< > $@
-
 clixon.mk:  clixon.mk.cpp
 	$(CPP) -P -traditional-cpp -x assembler-with-cpp -Dprefix=$(prefix) -Dlocalstatedir=$(localstatedir) -Dsysconfdir=$(sysconfdir) -Ddatadir=$(datadir) -Dlibdir=$(libdir) $< > $@
 
-install:	clixon.conf.cpp clixon.mk
+install:  clixon.mk
 	install -d -m 755 $(DESTDIR)$(datadir)/clixon
-	install -m 755 clixon.conf.cpp $(DESTDIR)$(datadir)/clixon
 	install -m 755 clixon.mk $(DESTDIR)$(datadir)/clixon
 	for i in $(SUBDIRS) doc; \
 	do (cd $$i; $(MAKE) $(MFLAGS) $@)||exit 1; done; \
@@ -88,7 +83,6 @@ install-include:
 uninstall:
 	for i in $(SUBDIRS) doc example docker; \
 	do (cd $$i && $(MAKE) $(MFLAGS) $@)||exit 1; done; 
-	rm -f $(DESTDIR)$(datadir)/clixon/clixon.conf.cpp
 	rm -f $(DESTDIR)$(datadir)/clixon/clixon.mk
 
 doc:	
@@ -106,8 +100,7 @@ clean:
 
 distclean:
 	rm -f Makefile TAGS config.status config.log *~ .depend
-	rm -rf autom4te.cache
+	rm -rf autom4te.cache clixon.mk build-root/rpmbuild
-	rm -rf clixon.conf.cpp clixon.mk build-root/rpmbuild
 	rm -f build-root/*.tar.xz build-root/*.rpm extras/rpm/Makefile
 	for i in $(SUBDIRS) doc example docker; \
 		do (cd $$i && $(MAKE) $(MFLAGS) $@); done

apps/backend/backend_client.c

@@ -1095,8 +1095,8 @@ from_client(int   s,
 	    goto done;
     retval = 0;
   done:
+    clicon_debug(1, "%s retval=%d", __FUNCTION__, retval);
     if (msg)
 	free(msg);
-    clicon_debug(1, "%s retval=%d", __FUNCTION__, retval);
     return retval; /* -1 here terminates backend */
 }

apps/backend/backend_main.c

@@ -73,11 +73,7 @@
 #include "backend_handle.h"
 
 /* Command line options to be passed to getopt(3) */
-#ifdef BACKEND_STARTUP_COMPAT
-#define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:IRCrg:y:x:" /* substitute s: for IRCc:r */
-#else
 #define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:g:y:x:" /* substitute s: for IRCc:r */
-#endif
 
 /*! Terminate. Cannot use h after this */
 static int
@@ -142,12 +138,6 @@ usage(char *argv0, clicon_handle h)
     	    "    -P <file>\tPid filename (default: %s)\n"
 	    "    -s <mode>\tSpecify backend startup mode: none|startup|running|init (replaces -IRCr\n"
 	    "    -c <file>\tLoad extra xml configuration, but don't commit.\n"
-#ifdef BACKEND_STARTUP_COMPAT
-    	    "    -I\t\tInitialize running state database\n"
-    	    "    -R\t\tCall plugin_reset() in plugins to reset system state in running db (use with -I)\n"
-	    "    -C\t\tCall plugin_reset() in plugins to reset system state in candidate db (use with -I)\n"
-	    "    -r\t\tReload running database\n"
-#endif /* BACKEND_STARTUP_COMPAT */
 	    "    -g <group>\tClient membership required to this group (default: %s)\n"
 	    "    -y <file>\tOverride yang spec file (dont include .yang suffix)\n"
 	    "    -x <plugin>\tXMLDB plugin\n",
@@ -270,161 +260,6 @@ plugin_start_useroptions(clicon_handle h,
     return 0;
 }
 
-#ifdef BACKEND_STARTUP_COMPAT
-/*! Initialize running-config from file application configuration
- *
- * @param[in] h                clicon handle
- * @param[in] extraxml_file  clicon application configuration file
- * @param[in] running_db       Name of running db
- * @retval    0                OK
- * @retval   -1                Error. clicon_err set
- */
-static int
-rundb_main(clicon_handle h, 
-	   char         *extraxml_file)
-{
-    int        retval = -1;
-    int        fd = -1;
-    cxobj     *xt = NULL;
-    cxobj     *xn;
-
-    if (xmldb_create(h, "tmp") < 0)
-	goto done;
-    if (xmldb_copy(h, "running", "tmp") < 0)
-	goto done;
-    if ((fd = open(extraxml_file, O_RDONLY)) < 0){
-	clicon_err(OE_UNIX, errno, "open(%s)", extraxml_file);
-	goto done;
-    }
-    if (xml_parse_file(fd, &xt, "</clicon>") < 0)
-	goto done;
-    if ((xn = xml_child_i(xt, 0)) != NULL)
-	if (xmldb_put(h, "tmp", OP_MERGE, xn) < 0)
-	    goto done;
-    if (candidate_commit(h, "tmp") < 0)
-	goto done;
-    if (xmldb_delete(h, "tmp") < 0)
-	goto done;
-    retval = 0;
-done:
-    if (xt)
-	xml_free(xt);
-    if (fd != -1)
-	close(fd);
-    return retval;
-}
-
-static int
-candb_reset(clicon_handle h)
-{
-    int   retval = -1;
-
-    if (xmldb_copy(h, "running", "tmp") < 0)
-	goto done;
-    /* Request plugins to reset system state, eg initiate running from system 
-     * -R
-     */
-    if (plugin_reset_state(h, "tmp") < 0)  
-	goto done;
-    if (candidate_commit(h, "tmp") < 0)
-	goto done;
-    retval = 0;
-  done:
-    return retval;
-}
-
-/*! Legacy (old-style) startup mode where flags -IRCcr was used
- */
-static int
-fragmented_startup_mode(clicon_handle h,
-			char         *argv0,
-			int           argc,
-			char         *argv[],
-			int           reload_running,
-			int           init_rundb,
-			int           reset_state_candidate,
-			int           reset_state_running,
-			char         *extraxml_file)
-{
-    int retval = -1;
-
-    /* First check for startup config */
-    if (clicon_option_int(h, "CLICON_USE_STARTUP_CONFIG") > 0){
-	if (xmldb_exists(h, "startup") == 1){
-	    /* copy startup config -> running */
-	    if (xmldb_copy(h, "startup", "running") < 0)
-		goto done;
-	}
-	else
-	    if (db_reset(h, "running") < 0)
-		goto done;
-	if (xmldb_create(h, "candidate") < 0)
-	    goto done;
-	if (xmldb_copy(h, "running", "candidate") < 0)
-	    goto done;
-    }
-    /* If running exists and reload_running set, make a copy to candidate */
-    if (reload_running){
-	if (xmldb_exists(h, "running") != 1){
-	    clicon_log(LOG_NOTICE, "%s: -r (reload running) option given but no running_db found, proceeding without", __PROGRAM__);
-	    reload_running = 0; /* void it, so we dont commit candidate below */
-	}
-	else
-	    if (xmldb_copy(h, "running", "candidate") < 0)
-		goto done;
-    }
-    /* Init running db 
-     * -I or if it isnt there
-     */
-    if (init_rundb || xmldb_exists(h, "running") != 1){
-	if (db_reset(h, "running") < 0)
-	    goto done;
-    }
-    /* If candidate does not exist, create it from running */
-    if (xmldb_exists(h, "candidate") != 1){
-	if (xmldb_create(h, "candidate") < 0)
-	    goto done;
-	if (xmldb_copy(h, "running", "candidate") < 0)
-	    goto done;
-    }
-
-    /* Load plugins and call plugin_init() */
-    if (plugin_initiate(h) != 0) 
-	goto done;
-
-    if (reset_state_candidate){
-	if (candb_reset(h) < 0) 
-	    goto done;
-    }
-    else
-	if (reset_state_running){
-	    if (plugin_reset_state(h, "running") < 0) 
-		goto done;
-	}
-
-    if (plugin_start_useroptions(h, argv0, argc, argv) <0)
-	goto done;
-
-    if (reload_running){
-	/* This could be a failed validation, and we should not fail for that */
-	(void)candidate_commit(h, "candidate");
-    }
-
-    /* Have we specified a config file to load? eg 
-     * -c [<file>]
-     */
-    if (extraxml_file)
-	if (rundb_main(h, extraxml_file) < 0)
-	    goto done;
-    /* Initiate the shared candidate. */
-    if (xmldb_copy(h, "running", "candidate") < 0)
-	goto done;
-    retval = 0;
- done:
-    return retval;
-}
-#endif /* BACKEND_STARTUP_COMPAT */
-
 /*! Merge xml in filename into database
  */
 static int
@@ -648,12 +483,6 @@ main(int argc, char **argv)
     int           foreground;
     int           once;
     enum startup_mode_t startup_mode;
-#ifdef BACKEND_STARTUP_COMPAT
-    int           init_rundb = 0;
-    int           reset_state_running = 0;
-    int           reset_state_candidate = 0;
-    int           reload_running = 0;
-#endif
     char         *extraxml_file;
     char         *config_group;
     char         *argv0 = argv[0];
@@ -769,20 +598,6 @@ main(int argc, char **argv)
 	case 'c': /* Load application config */
 	    extraxml_file = optarg;
 	    break;
-#ifdef BACKEND_STARTUP_COMPAT
-	case 'I': /* Initiate running db */
-	    init_rundb++;
-	    break;
-	case 'R': /* Reset state directly into running */
-	    reset_state_running++;
-	    break;
-	case 'C': /* Reset state into candidate and then commit it */
-	    reset_state_candidate++;
-	    break;
-	case 'r': /* Reload running */
-	    reload_running++;
-	    break;
-#endif /* BACKEND_STARTUP_COMPAT */
 	case 'g': /* config socket group */
 	    clicon_option_str_set(h, "CLICON_SOCK_GROUP", optarg);
 	    break;
@@ -890,19 +705,9 @@ main(int argc, char **argv)
 	    goto done;
     /* If startup mode is not defined, eg via OPTION or -s, assume old method */
     startup_mode = clicon_startup_mode(h);
-    if (startup_mode == -1){ 	/* Old style, fragmented mode, phase out */
+    if (startup_mode == -1){ 	
-#ifdef BACKEND_STARTUP_COMPAT
-	if (fragmented_startup_mode(h,
-				    argv0, argc, argv,
-				    reload_running, init_rundb,
-				    reset_state_candidate, reset_state_running, 
-				    extraxml_file
-				    ) < 0)
-	    goto done;
-#else
 	clicon_log(LOG_ERR, "Startup mode undefined. Specify option CLICON_STARTUP_MODE or specify -s option to clicon_backend.\n"); 
 	goto done;
-#endif	
     }
     else {
 	/* Init running db if it is not there

apps/cli/cli_generate.c

@@ -238,9 +238,18 @@ yang2cli_var_sub(clicon_handle h,
 		goto done;
 	    }
 	}
+
 	else{ /* Cligen does not have 'max' keyword in range so need to find actual
-		 max value of type if yang range expression is 0..max */
+		 max value of type if yang range expression is 0..max 
-	    if ((r = cvtype_max2str_dup(cvtype)) == NULL){
+	      */
+	    if (cvtype==CGV_STRING){
+		if ((r = malloc(512)) == NULL){
+		    clicon_err(OE_UNIX, errno, "malloc");
+		    goto done;
+		}
+		snprintf(r, 512, "%d", MAXPATHLEN);
+	    }
+	    else if ((r = cvtype_max2str_dup(cvtype)) == NULL){
 		    clicon_err(OE_UNIX, errno, "cvtype_max2str");
 		    goto done;
 		}

apps/cli/cli_show.c

@@ -79,7 +79,7 @@
  * Returns an expand-type list of commands as used by cligen 'expand' 
  * functionality.
  *
- * Assume callback given in a cligen spec: a <x:int expand_dbvar("arg")
+ * Assume callback given in a cligen spec: a <x:int expand_dbvar("db" "<xmlkeyfmt>")
  * @param[in]   h        clicon handle 
  * @param[in]   name     Name of this function (eg "expand_dbvar")
  * @param[in]   cvv      The command so far. Eg: cvec [0]:"a 5 b"; [1]: x=5;
@@ -88,7 +88,6 @@
  * @param[out]  commands vector of function pointers to callback functions
  * @param[out]  helptxt  vector of pointers to helptexts
  * @see cli_expand_var_generate  This is where arg is generated
- * XXX: helptexts?
  */
 int
 expand_dbvar(void   *h, 

apps/restconf/README.md

@@ -3,13 +3,14 @@
 
 Clixon restconf is a daemon based on FASTCGI. Instructions are available to
 run with NGINX.
-The implementatation supports plain OPTIONS, HEAD, GET, POST, PUT, PATCH, DELETE.
+The implementatation is based on [RFC 8040: RESTCONF Protocol](https://tools.ietf.org/html/rfc8040).
-and is based on draft-ietf-netconf-restconf-13. 
+The following featires are supported:
-There is currently (2017) a [RFC 8040: RESTCONF Protocol](https://tools.ietf.org/html/rfc8040), many of those features are _not_ implemented,
+- OPTIONS, HEAD, GET, POST, PUT, DELETE
-including:
+The following are not implemented
+- PATCH
 - query parameters (section 4.9)
 - notifications (sec 6)
-- only rudimentary error reporting exists (sec 7)
+- schema resource
 
 ### Installation using Nginx
 

apps/restconf/restconf_lib.c

@@ -148,7 +148,60 @@ restconf_code2reason(int code)
     return clicon_int2str(http_reason_phrase_map, code);
 }
 
-/*!
+/*! HTTP error 400
+ * @param[in]  r        Fastcgi request handle
+ */
+int
+badrequest(FCGX_Request *r)
+{
+    char *path;
+
+    clicon_debug(1, "%s", __FUNCTION__);
+    path = FCGX_GetParam("DOCUMENT_URI", r->envp);
+    FCGX_FPrintF(r->out, "Status: 400\r\n"); /* 400 bad request */
+    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
+    FCGX_FPrintF(r->out, "<h1>Clixon Bad request/h1>\n");
+    FCGX_FPrintF(r->out, "The requested URL %s or data is in some way badly formed.\n",
+		 path);
+    return 0;
+}
+
+/*! HTTP error 401
+ * @param[in]  r        Fastcgi request handle
+ */
+int
+unauthorized(FCGX_Request *r)
+{
+    char *path;
+
+    clicon_debug(1, "%s", __FUNCTION__);
+    path = FCGX_GetParam("DOCUMENT_URI", r->envp);
+    FCGX_FPrintF(r->out, "Status: 401\r\n"); /* 401 unauthorized */
+    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
+    FCGX_FPrintF(r->out, "<error-tag>access-denied</error-tag>\n");
+    FCGX_FPrintF(r->out, "The requested URL %s was unauthorized.\n", path);
+   return 0;
+}
+
+/*! HTTP error 403
+ * @param[in]  r        Fastcgi request handle
+ */
+int
+forbidden(FCGX_Request *r)
+{
+    char *path;
+
+    clicon_debug(1, "%s", __FUNCTION__);
+    path = FCGX_GetParam("DOCUMENT_URI", r->envp);
+    FCGX_FPrintF(r->out, "Status: 403\r\n"); /* 403 forbidden */
+    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
+    FCGX_FPrintF(r->out, "<h1>Grideye Forbidden</h1>\n");
+    FCGX_FPrintF(r->out, "The requested URL %s was forbidden.\n", path);
+   return 0;
+}
+
+/*! HTTP error 404
+ * @param[in]  r        Fastcgi request handle
  */
 int
 notfound(FCGX_Request *r)
@@ -165,31 +218,9 @@ notfound(FCGX_Request *r)
     return 0;
 }
 
-int
+/*! HTTP error 409
-badrequest(FCGX_Request *r)
+ * @param[in]  r        Fastcgi request handle
-{
+ */
-    char *path;
-
-    clicon_debug(1, "%s", __FUNCTION__);
-    path = FCGX_GetParam("DOCUMENT_URI", r->envp);
-    FCGX_FPrintF(r->out, "Status: 400\r\n"); /* 400 bad request */
-    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
-    FCGX_FPrintF(r->out, "<h1>Clixon Bad request/h1>\n");
-    FCGX_FPrintF(r->out, "The requested URL %s or data is in some way badly formed.\n",
-		 path);
-    return 0;
-}
-
-int
-notimplemented(FCGX_Request *r)
-{
-    clicon_debug(1, "%s", __FUNCTION__);
-    FCGX_FPrintF(r->out, "Status: 501\r\n"); 
-    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
-    FCGX_FPrintF(r->out, "<h1>Not Implemented/h1>\n");
-    return 0;
-}
-
 int
 conflict(FCGX_Request *r)
 {
@@ -200,6 +231,35 @@ conflict(FCGX_Request *r)
     return 0;
 }
 
+/*! HTTP error 500
+ * @param[in]  r        Fastcgi request handle
+ */
+int
+internal_server_error(FCGX_Request *r)
+{
+    char *path;
+
+    clicon_debug(1, "%s", __FUNCTION__);
+    path = FCGX_GetParam("DOCUMENT_URI", r->envp);
+    FCGX_FPrintF(r->out, "Status: 500\r\n"); /* 500 internal server error */
+    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
+    FCGX_FPrintF(r->out, "<h1>Grideye Internal server error when accessing %s</h1>\n", path);
+    return 0;
+}
+
+/*! HTTP error 501
+ * @param[in]  r        Fastcgi request handle
+ */
+int
+notimplemented(FCGX_Request *r)
+{
+    clicon_debug(1, "%s", __FUNCTION__);
+    FCGX_FPrintF(r->out, "Status: 501\r\n"); 
+    FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n");
+    FCGX_FPrintF(r->out, "<h1>Not Implemented/h1>\n");
+    return 0;
+}
+
 /*! Specialization of clicon_debug with xml tree */
 int 
 clicon_debug_xml(int    dbglevel, 
@@ -296,11 +356,10 @@ readdata(FCGX_Request *r)
     return cb;
 }
 
-typedef int (credentials_t)(clicon_handle h, FCGX_Request *r); 
 
 static int nplugins = 0;
 static plghndl_t *plugins = NULL;
-static credentials_t *p_credentials = NULL; /* Credentials callback */
+static plgcredentials_t *_credentials_fn = NULL; /* Credentials callback */
 
 /*! Load all plugins you can find in CLICON_RESTCONF_DIR
  */
@@ -330,7 +389,10 @@ restconf_plugin_load(clicon_handle h)
 		     (int)strlen(filename), filename);
 	if ((handle = plugin_load(h, filename, RTLD_NOW)) == NULL)
 	    goto quit;
-	p_credentials    = dlsym(handle, PLUGIN_CREDENTIALS);
+	if ((_credentials_fn    = dlsym(handle, PLUGIN_CREDENTIALS)) == NULL)
+	    clicon_debug(1, "Failed to load %s", PLUGIN_CREDENTIALS); 
+	else
+	    clicon_debug(1, "%s callback loaded", PLUGIN_CREDENTIALS); 
 	if ((plugins = realloc(plugins, (nplugins+1) * sizeof (*plugins))) == NULL) {
 	    clicon_err(OE_UNIX, errno, "realloc");
 	    goto quit;
@@ -384,26 +446,36 @@ restconf_plugin_start(clicon_handle h,
     return 0;
 }
 
+/*! Run the restconf user-defined credentials callback if present
+ * The callback is expected to return the authenticated user, or NULL if not
+ * authenticasted.
+ * If no callback exists, return user "none"
+ * @param[in]  h    Clicon handle
+ * @param[in]  r    Fastcgi request handle
+ * @param[out] user The authenticated user (or NULL). Malloced, must be freed.
+ */
 int 
-plugin_credentials(clicon_handle h,     
+restconf_credentials(clicon_handle h,     
 		     FCGX_Request *r,
-		   int          *auth)
+		     char        **user)
 {
     int retval = -1;
 
     clicon_debug(1, "%s", __FUNCTION__);
     /* If no authentication callback then allow anything. Is this OK? */
-    if (p_credentials == 0){
+    if (_credentials_fn == NULL){
-	*auth = 1;
+	if ((*user = strdup("none")) == NULL){
-	retval = 0;
+	    clicon_err(OE_XML, errno, "strdup");
 	    goto done;
 	}
-    if (p_credentials(h, r) < 0) 
+	goto ok;
-	*auth = 0;
+    }
-    else
+    if (_credentials_fn(h, r, user) < 0) 
-	*auth = 1;
+	*user = NULL;
+ ok:
     retval = 0;
  done:
+    clicon_debug(1, "%s retval:%d user:%s", __FUNCTION__, retval, *user);
     return retval;
 }
 

apps/restconf/restconf_lib.h

@@ -45,10 +45,15 @@
  */
 int restconf_err2code(char *tag);
 const char *restconf_code2reason(int code);
-int notfound(FCGX_Request *r);
+
 int badrequest(FCGX_Request *r);
-int notimplemented(FCGX_Request *r);
+int unauthorized(FCGX_Request *r);
+int forbidden(FCGX_Request *r);
+int notfound(FCGX_Request *r);
 int conflict(FCGX_Request *r);
+int internal_server_error(FCGX_Request *r);
+int notimplemented(FCGX_Request *r);
+
 int clicon_debug_xml(int dbglevel, char *str, cxobj *cx);
 int test(FCGX_Request *r, int dbg);
 cbuf *readdata(FCGX_Request *r);
@@ -56,7 +61,7 @@ cbuf *readdata(FCGX_Request *r);
 int restconf_plugin_load(clicon_handle h);
 int restconf_plugin_start(clicon_handle h, int argc, char **argv);
 int restconf_plugin_unload(clicon_handle h);
-int plugin_credentials(clicon_handle h, FCGX_Request *r, int *auth);
+int restconf_credentials(clicon_handle h, FCGX_Request *r, char **user);
 int get_user_cookie(char *cookiestr, char  *attribute, char **val);
 
 

apps/restconf/restconf_main.c

@@ -73,9 +73,14 @@
 /* Command line options to be passed to getopt(3) */
 #define RESTCONF_OPTS "hDf:p:y:"
 
-/* Should be discovered via  "/.well-known/host-meta"
+/* RESTCONF enables deployments to specify where the RESTCONF API is 
-   resource ([RFC6415]) */
+   located.  The client discovers this by getting the "/.well-known/host-meta"
-#define RESTCONF_API_ROOT    "/restconf/"
+   resource 
+*/
+#define RESTCONF_WELL_KNOWN  "/.well-known/host-meta"
+
+#define RESTCONF_API       "restconf"
+#define RESTCONF_API_ROOT "/restconf"
 
 /*! Generic REST method, GET, PUT, DELETE, etc
  * @param[in]  h      CLIXON handle
@@ -117,6 +122,7 @@ api_data(clicon_handle h,
 	retval = api_data_delete(h, r, api_path, pi);
     else
 	retval = notfound(r);
+    clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     return retval;
 }
 
@@ -136,7 +142,8 @@ api_operations(clicon_handle h,
 	       cvec         *pcvec, 
 	       int           pi,
 	       cvec         *qvec, 
-	       char         *data)
+	       char         *data,
+	       char         *username)
 {
     int     retval = -1;
     char   *request_method;
@@ -144,18 +151,118 @@ api_operations(clicon_handle h,
     clicon_debug(1, "%s", __FUNCTION__);
     request_method = FCGX_GetParam("REQUEST_METHOD", r->envp);
     clicon_debug(1, "%s method:%s", __FUNCTION__, request_method);
-    if (strcmp(request_method, "POST")==0)
+    if (strcmp(request_method, "GET")==0)
-	retval = api_operation_post(h, r, path, pcvec, pi, qvec, data);
+	retval = api_operation_get(h, r, path, pcvec, pi, qvec, data);
+    else if (strcmp(request_method, "POST")==0)
+	retval = api_operation_post(h, r, path, pcvec, pi, qvec, data, username);
     else
 	retval = notfound(r);
     return retval;
 }
 
+/*! Retrieve the Top-Level API Resource
+ * @note Only returns null for operations and data,...
+ */
+static int
+api_root(clicon_handle h,
+	 FCGX_Request *r)
+{
+    int    retval = -1;
+    char  *media_accept;
+    int   use_xml = 0; /* By default use JSON */
+    cxobj *xt = NULL;
+    cbuf  *cb = NULL;
+    int    pretty;
+    
+    clicon_debug(1, "%s", __FUNCTION__);
+    pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY");
+    media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp);
+    if (strcmp(media_accept, "application/yang-data+xml")==0)
+	use_xml++;
+    FCGX_SetExitStatus(200, r->out); /* OK */
+    FCGX_FPrintF(r->out, "Content-Type: application/yang-data+%s\r\n", use_xml?"xml":"json");
+    FCGX_FPrintF(r->out, "\r\n");
+    if (xml_parse_string("<restconf><data></data><operations></operations><yang-library-version>2016-06-21</yang-library-version></restconf>", NULL, &xt) < 0)
+	goto done;
+    if ((cb = cbuf_new()) == NULL){
+	clicon_err(OE_XML, errno, "cbuf_new");
+	goto done;
+    }
+    if (xml_rootchild(xt, 0, &xt) < 0)
+	goto done;
+    if (use_xml){
+	if (clicon_xml2cbuf(cb, xt, 0, pretty) < 0)
+	    goto done;
+    }
+    else
+	if (xml2json_cbuf(cb, xt, pretty) < 0)
+	    goto done;
+    FCGX_FPrintF(r->out, "%s", cb?cbuf_get(cb):"");
+    FCGX_FPrintF(r->out, "\r\n\r\n");
+    retval = 0;
+ done:
+    if (cb)
+        cbuf_free(cb);
+    if (xt)
+	xml_free(xt);
+    return retval;
+}
+
+/*!
+ * See https://tools.ietf.org/html/rfc7895
+ */
+static int
+api_yang_library_version(clicon_handle h,
+			 FCGX_Request *r)
+{
+    int    retval = -1;
+    char  *media_accept;
+    int    use_xml = 0; /* By default use JSON */
+    cxobj *xt = NULL;
+    cbuf  *cb = NULL;
+    int    pretty;
+
+    clicon_debug(1, "%s", __FUNCTION__);
+    pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY");
+    media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp);
+    if (strcmp(media_accept, "application/yang-data+xml")==0)
+	use_xml++;
+    FCGX_SetExitStatus(200, r->out); /* OK */
+    FCGX_FPrintF(r->out, "Content-Type: application/yang-data+%s\r\n", use_xml?"xml":"json");
+    FCGX_FPrintF(r->out, "\r\n");
+    if (xml_parse_string("<yang-library-version>2016-06-21</yang-library-version>", NULL, &xt) < 0)
+	goto done;
+    if (xml_rootchild(xt, 0, &xt) < 0)
+	goto done;
+    if ((cb = cbuf_new()) == NULL){
+	clicon_err(OE_XML, errno, "cbuf_new");
+	goto done;
+    }
+    if (use_xml){
+	if (clicon_xml2cbuf(cb, xt, 0, pretty) < 0)
+	    goto done;
+    }
+    else{
+	if (xml2json_cbuf(cb, xt, pretty) < 0)
+	    goto done;
+    }
+    clicon_debug(1, "%s cb%s", __FUNCTION__, cbuf_get(cb));
+    FCGX_FPrintF(r->out, "%s\r\n", cb?cbuf_get(cb):"");
+    FCGX_FPrintF(r->out, "\r\n\r\n");
+    retval = 0;
+ done:
+    if (cb)
+        cbuf_free(cb);
+    if (xt)
+	xml_free(xt);
+    return retval;
+}
+
 /*! Process a FastCGI request
  * @param[in]  r        Fastcgi request handle
  */
 static int
-request_process(clicon_handle h,
+api_restconf(clicon_handle h,
 		FCGX_Request *r)
 {
     int    retval = -1;
@@ -169,14 +276,35 @@ request_process(clicon_handle h,
     cvec  *pcvec = NULL; /* for rest api */
     cbuf  *cb = NULL;
     char  *data;
-    int    auth = 0;
+    char  *username = NULL;
 
     clicon_debug(1, "%s", __FUNCTION__);
     path = FCGX_GetParam("REQUEST_URI", r->envp);
     query = FCGX_GetParam("QUERY_STRING", r->envp);
     if ((pvec = clicon_strsep(path, "/", &pn)) == NULL)
 	goto done;
-
+    /* Sanity check of path. Should be /restconf/ */
+    if (pn < 2){
+	retval = notfound(r);
+	goto done;
+    }
+    if (strlen(pvec[0]) != 0){
+	retval = notfound(r);
+	goto done;
+    }
+    if (strcmp(pvec[1], RESTCONF_API)){
+	retval = notfound(r);
+	goto done;
+    }
+    if (pn == 2){
+	retval = api_root(h, r);
+	goto done;
+    }
+    if ((method = pvec[2]) == NULL){
+	retval = notfound(r);
+	goto done;
+    }
+    clicon_debug(1, "method=%s", method);
     if (str2cvec(query, '&', '=', &qvec) < 0)
       goto done;
     if (str2cvec(path, '/', '=', &pcvec) < 0) /* rest url eg /album=ricky/foo */
@@ -188,29 +316,38 @@ request_process(clicon_handle h,
     clicon_debug(1, "DATA=%s", data);
     if (str2cvec(data, '&', '=', &dvec) < 0)
       goto done;
-    if ((method = pvec[2]) == NULL){
+
-	retval = notfound(r);
+    test(r, 1);
+    /* If present, check credentials. See "plugin_credentials" in plugin  
+     * See RFC 8040 section 2.5
+     */
+    if (restconf_credentials(h, r, &username) < 0)
+	goto done;
+    clicon_debug(1, "%s username:%s", __FUNCTION__, username);
+    clicon_debug(1, "%s credentials ok username:%s (should be non-NULL)",
+		 __FUNCTION__, username);
+    if (username == NULL){
+	unauthorized(r);
+	goto ok;
+    }
+    if (strcmp(method, "yang-library-version")==0){
+	if (api_yang_library_version(h, r) < 0)
 	    goto done;
     }
-    retval = 0;
+    else if (strcmp(method, "data") == 0){ /* restconf, skip /api/data */
-    test(r, 1);
+	if (api_data(h, r, path, pcvec, 2, qvec, data) < 0)
-    /* If present, check credentials */
-    if (plugin_credentials(h, r, &auth) < 0)
 	    goto done;
-    clicon_debug(1, "%s credentials ok auth:%d (should be 1)",
+    }
-		 __FUNCTION__, auth);
+    else if (strcmp(method, "operations") == 0){ /* rpc */
-    if (auth == 0)
+	if (api_operations(h, r, path, pcvec, 2, qvec, data, username) < 0)
 	    goto done;
-    clicon_debug(1, "%s credentials ok 2", __FUNCTION__);
+    }
-
-    if (strcmp(method, "data") == 0) /* restconf, skip /api/data */
-	retval = api_data(h, r, path, pcvec, 2, qvec, data);
-    else if (strcmp(method, "operations") == 0) /* rpc */
-	retval = api_operations(h, r, path, pcvec, 2, qvec, data);
     else if (strcmp(method, "test") == 0)
-	retval = test(r, 0);
+	test(r, 0);
     else
-	retval = notfound(r);
+	notfound(r);
+ ok:
+    retval = 0;
  done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     if (pvec)
@@ -223,9 +360,29 @@ request_process(clicon_handle h,
 	cvec_free(pcvec);
     if (cb)
 	cbuf_free(cb);
+    if (username)
+	free(username);
     return retval;
 }
 
+/*! Process a FastCGI request
+ * @param[in]  r        Fastcgi request handle
+ */
+static int
+api_well_known(clicon_handle h,
+	       FCGX_Request *r)
+{
+    clicon_debug(1, "%s", __FUNCTION__);
+    FCGX_FPrintF(r->out, "Content-Type: application/xrd+xml\r\n");
+    FCGX_FPrintF(r->out, "\r\n");
+    FCGX_SetExitStatus(200, r->out); /* OK */
+    FCGX_FPrintF(r->out, "<XRD xmlns='http://docs.oasis-open.org/ns/xri/xrd-1.0'>\r\n");
+    FCGX_FPrintF(r->out, "   <Link rel='restconf' href='/restconf'/>\r\n");
+    FCGX_FPrintF(r->out, "</XRD>\r\n");
+
+    return 0;
+}
+
 static int
 restconf_terminate(clicon_handle h)
 {
@@ -384,13 +541,17 @@ main(int    argc,
 	}
 	clicon_debug(1, "------------");
 	if ((path = FCGX_GetParam("REQUEST_URI", r->envp)) != NULL){
-	    if (strncmp(path, RESTCONF_API_ROOT, strlen(RESTCONF_API_ROOT)) == 0 ||
+	    clicon_debug(1, "path:%s", path);
-		strncmp(path, RESTCONF_API_ROOT, strlen(RESTCONF_API_ROOT)-1) == 0)
+	    if (strncmp(path, RESTCONF_API_ROOT, strlen(RESTCONF_API_ROOT)) == 0)
-		request_process(h, r); /* This is the function */
+		api_restconf(h, r); /* This is the function */
+	    else if (strncmp(path, RESTCONF_WELL_KNOWN, strlen(RESTCONF_WELL_KNOWN)) == 0) {
+		api_well_known(h, r); /* This is the function */
+	    }
 	    else{
-		clicon_debug(1, "top-level not found");
+		clicon_debug(1, "top-level %s not found", path);
 		notfound(r);
 	    }
+	    
 	}
 	else
 	    clicon_debug(1, "NULL URI");

apps/restconf/restconf_methods.c

@@ -140,61 +140,27 @@ api_data_options(clicon_handle h,
     return 0;
 }
 
-/*! Generic GET (both HEAD and GET)
+/*! Return error on get/head request
+ * @param[in]  h      Clixon handle
+ * @param[in]  r      Fastcgi request handle
+ * @param[in]  xerr   XML error message from backend
  */
 static int
-api_data_get_gen(clicon_handle h,
+api_data_get_err(clicon_handle h,
 		 FCGX_Request *r,
-		 cvec         *pcvec,
+		 cxobj     *xerr)
-		 int           pi,
-		 cvec         *qvec,
-		 int           head)
 {
     int        retval = -1;
-    cbuf      *path = NULL;
+    cbuf      *cbj = NULL;
-    cbuf      *cbx = NULL;
-    cxobj    **vec = NULL;
-    yang_spec *yspec;
-    cxobj     *xret = NULL;
-    cxobj     *xerr;
     cxobj     *xtag;
-    cbuf      *cbj = NULL;;
     int        code;	
     const char *reason_phrase;
-    char      *media_accept;
-    int        use_xml = 0; /* By default use JSON */
 
-    clicon_debug(1, "%s", __FUNCTION__);
-    media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp);
-    if (strcmp(media_accept, "application/yang-data+xml")==0)
-	use_xml++;
-    yspec = clicon_dbspec_yang(h);
-    if ((path = cbuf_new()) == NULL)
-        goto done;
-    cprintf(path, "/");
-    if (api_path2xpath_cvv(yspec, pcvec, pi, path) < 0){
-	notfound(r);
-	goto done;
-    }
-    clicon_debug(1, "%s path:%s", __FUNCTION__, cbuf_get(path));
-    if (clicon_rpc_get(h, cbuf_get(path), &xret) < 0){
-	notfound(r);
-	goto done;
-    }
-#if 0 /* DEBUG */
-    {
-	cbuf *cb = cbuf_new();
-	xml2json_cbuf(cb, xret, 1);
-	clicon_debug(1, "%s xret:%s", __FUNCTION__, cbuf_get(cb));
-	cbuf_free(cb);
-    }
-#endif
-    if ((xerr = xpath_first(xret, "/rpc-error")) != NULL){
     if ((cbj = cbuf_new()) == NULL)
 	goto done;
     if ((xtag = xpath_first(xerr, "/error-tag")) == NULL){
 	notfound(r); /* bad reply? */
-	    goto done;
+	goto ok;
     }
     code = restconf_err2code(xml_body(xtag));
     if ((reason_phrase = restconf_code2reason(code)) == NULL)
@@ -214,8 +180,100 @@ api_data_get_gen(clicon_handle h,
     FCGX_FPrintF(r->out, "    %s", cbuf_get(cbj));
     FCGX_FPrintF(r->out, "  }\r\n");
     FCGX_FPrintF(r->out, "}\r\n");
+ ok:
+    retval = 0;
+ done:
+    if (cbj)
+        cbuf_free(cbj);
+    return retval;
+}
+
+/*! Generic GET (both HEAD and GET)
+ * According to restconf 
+ * @param[in]  h      Clixon handle
+ * @param[in]  r      Fastcgi request handle
+ * @param[in]  pcvec  Vector of path ie DOCUMENT_URI element 
+ * @param[in]  pi     Offset, where path starts  
+ * @param[in]  qvec   Vector of query string (QUERY_STRING)
+ * @param[in]  head   If 1 is HEAD, otherwise GET
+ * @code
+ *  curl -G http://localhost/restconf/data/interfaces/interface=eth0
+ * @endcode                                     
+ * XXX: cant find a way to use Accept request field to choose Content-Type  
+ *      I would like to support both xml and json.           
+ * Request may contain                                        
+ *     Accept: application/yang.data+json,application/yang.data+xml   
+ * Response contains one of:                           
+ *     Content-Type: application/yang-data+xml    
+ *     Content-Type: application/yang-data+json  
+ * NOTE: If a retrieval request for a data resource representing a YANG leaf-
+ * list or list object identifies more than one instance, and XML
+ * encoding is used in the response, then an error response containing a
+ * "400 Bad Request" status-line MUST be returned by the server.
+ * Netconf: <get-config>, <get>                        
+ */
+static int
+api_data_get2(clicon_handle h,
+	      FCGX_Request *r,
+	      cvec         *pcvec,
+	      int           pi,
+	      cvec         *qvec,
+	      int           head)
+{
+    int        retval = -1;
+    cbuf      *cbpath = NULL;
+    char      *path;
+    cbuf      *cbx = NULL;
+    yang_spec *yspec;
+    cxobj     *xret = NULL;
+    cxobj     *xerr;
+    char      *media_accept;
+    int        use_xml = 0; /* By default use JSON */
+    cxobj    **xvec = NULL;
+    size_t     xlen;
+    int        pretty;
+    int        i;
+    cxobj     *x;
+
+    clicon_debug(1, "%s", __FUNCTION__);
+    pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY");
+    media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp);
+    if (strcmp(media_accept, "application/yang-data+xml")==0)
+	use_xml++;
+    yspec = clicon_dbspec_yang(h);
+    if ((cbpath = cbuf_new()) == NULL)
+        goto done;
+    cprintf(cbpath, "/");
+    clicon_debug(1, "%s pi:%d", __FUNCTION__, pi);
+    /* We know "data" is element pi-1 */
+    if (api_path2xpath_cvv(yspec, pcvec, pi, cbpath) < 0){
+	notfound(r);
 	goto ok;
     }
+    path = cbuf_get(cbpath);
+    clicon_debug(1, "%s path:%s", __FUNCTION__, path);
+    if (clicon_rpc_get(h, path, &xret) < 0){
+	notfound(r);
+	goto ok;
+    }
+    /* We get return via netconf which is complete tree from root 
+     * We need to cut that tree to only the object.
+     */
+#if 1 /* DEBUG */
+    {
+	cbuf *cb = cbuf_new();
+	clicon_xml2cbuf(cb, xret, 0, 0);
+	clicon_debug(1, "%s xret:%s", __FUNCTION__, cbuf_get(cb));
+	cbuf_free(cb);
+    }
+#endif
+    /* Check if error return */
+    if ((xerr = xpath_first(xret, "/rpc-error")) != NULL){
+	if (api_data_get_err(h, r, xerr) < 0)
+	    goto done;
+	goto ok;
+    }
+    /* Normal return, no error */
     if ((cbx = cbuf_new()) == NULL)
 	goto done;
     FCGX_SetExitStatus(200, r->out); /* OK */
@@ -223,18 +281,32 @@ api_data_get_gen(clicon_handle h,
     FCGX_FPrintF(r->out, "\r\n");
     if (head)
 	goto ok;
-    clicon_debug(1, "%s name:%s child:%d", __FUNCTION__, xml_name(xret), xml_child_nr(xret));
+    if (path==NULL || strcmp(path,"/")==0){ /* Special case: data root */
-
-    clicon_debug(1, "%s xretnr:%d", __FUNCTION__, xml_child_nr(xret));
 	if (use_xml){
-	if (clicon_xml2cbuf(cbx, xret, 0, 1) < 0) /* Dont print top object?  */
+	    if (clicon_xml2cbuf(cbx, xret, 0, pretty) < 0) /* Dont print top object?  */
 		goto done;
 	}
 	else{
-	vec = xml_childvec_get(xret);
+	    if (xml2json_cbuf(cbx, xret, pretty) < 0)
-	if (xml2json_cbuf_vec(cbx, vec, xml_child_nr(xret), 0) < 0)
 		goto done;
 	}
+    }
+    else{
+	if (xpath_vec(xret, path, &xvec, &xlen) < 0)
+	    goto done;
+	clicon_debug(1, "%s: xpath:%s xlen:%d", __FUNCTION__, path, xlen);
+	if (use_xml){
+	    for (i=0; i<xlen; i++){
+		x = xvec[i];
+		if (clicon_xml2cbuf(cbx, x, 0, pretty) < 0) /* Dont print top object?  */
+		    goto done;
+	    }
+	}
+	else
+	    if (xml2json_cbuf_vec(cbx, xvec, xlen, pretty) < 0)
+		goto done;
+    }
+
     clicon_debug(1, "%s cbuf:%s", __FUNCTION__, cbuf_get(cbx));
     FCGX_FPrintF(r->out, "%s", cbx?cbuf_get(cbx):"");
     FCGX_FPrintF(r->out, "\r\n\r\n");
@@ -244,12 +316,12 @@ api_data_get_gen(clicon_handle h,
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
     if (cbx)
         cbuf_free(cbx);
-    if (cbj)
+    if (cbpath)
-        cbuf_free(cbj);
+	cbuf_free(cbpath);
-    if (path)
-	cbuf_free(path);
     if (xret)
 	xml_free(xret);
+    if (xvec)
+	free(xvec);
     return retval;
 }
 
@@ -271,7 +343,7 @@ api_data_head(clicon_handle h,
              int           pi,
              cvec         *qvec)
 {
-    return api_data_get_gen(h, r, pcvec, pi, qvec, 1);
+    return api_data_get2(h, r, pcvec, pi, qvec, 1);
 }
 
 /*! REST GET method
@@ -304,10 +376,10 @@ api_data_get(clicon_handle h,
              int           pi,
              cvec         *qvec)
 {
-    return api_data_get_gen(h, r, pcvec, pi, qvec, 0);
+    return api_data_get2(h, r, pcvec, pi, qvec, 0);
 }
 
-/*! REST POST  method 
+/*! Generic REST POST  method 
  * @param[in]  h      CLIXON handle
  * @param[in]  r      Fastcgi request handle
  * @param[in]  api_path According to restconf (Sec 3.5.3.1 in rfc8040)
@@ -315,7 +387,7 @@ api_data_get(clicon_handle h,
  * @param[in]  pi     Offset, where to start pcvec
  * @param[in]  qvec   Vector of query string (QUERY_STRING)
  * @param[in]  data   Stream input data
- * @note We map post to edit-config create. 
+ * @note restconf POST is mapped to edit-config create. 
  POST:
    target resource type is datastore --> create a top-level resource
    target resource type is  data resource --> create child resource
@@ -342,12 +414,12 @@ api_data_post(clicon_handle h,
 	      cvec         *qvec, 
 	      char         *data)
 {
-    enum operation_type op = OP_CREATE;
     int        retval = -1;
+    enum operation_type op = OP_CREATE;
     int        i;
     cxobj     *xdata = NULL;
-    cxobj     *xtop = NULL; /* xpath root */
     cbuf      *cbx = NULL;
+    cxobj     *xtop = NULL; /* xpath root */
     cxobj     *xbot = NULL;
     cxobj     *x;
     yang_node *y = NULL;
@@ -360,7 +432,8 @@ api_data_post(clicon_handle h,
 		 __FUNCTION__, 
 		 api_path, data);
     media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp);
-    if (strcmp(media_content_type, "application/yang-data+xml")==0)
+    if (media_content_type &&
+	strcmp(media_content_type, "application/yang-data+xml")==0)
 	parse_xml++;
     if ((yspec = clicon_dbspec_yang(h)) == NULL){
 	clicon_err(OE_FATAL, 0, "No DB_SPEC");
@@ -371,56 +444,61 @@ api_data_post(clicon_handle h,
     /* Create config top-of-tree */
     if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
+    /* Translate api_path to xtop/xbot */
     xbot = xtop;
     if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0)
 	goto done;
     /* Parse input data as json or xml into xml */
     if (parse_xml){
 	if (xml_parse_string(data, NULL, &xdata) < 0){
-	    clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
+	    badrequest(r);
-	    goto done;
+	    goto ok;
 	}
     }
     else if (json_parse_str(data, &xdata) < 0){
-	clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
+	badrequest(r);
-	goto done;
+	goto ok;
     }
-    /* Add xdata to xbot */
+    /* The message-body MUST contain exactly one instance of the
-    x = NULL;
+     * expected data resource. 
-    while ((x = xml_child_each(xdata, x, CX_ELMNT)) != NULL) {
+     */
+    if (xml_child_nr(xdata) != 1){
+	badrequest(r);
+	goto ok;
+    }
+    x = xml_child_i(xdata,0);
+    /* Add operation (create/replace) as attribute */
     if ((xa = xml_new("operation", x, NULL)) == NULL)
 	goto done;
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa, xml_operation2str(op)) < 0)
 	goto done;
+    /* Replace xbot with x, ie bottom of api-path with data */
     if (xml_addsub(xbot, x) < 0)
 	goto done;
-    }
+    /* Create text buffer for transfer to backend */
     if ((cbx = cbuf_new()) == NULL)
 	goto done;
     if (clicon_xml2cbuf(cbx, xtop, 0, 0) < 0)
 	goto done;
-    clicon_debug(1, "%s xml: %s",__FUNCTION__, cbuf_get(cbx));
+    clicon_debug(1, "%s xml: %s api_path:%s",__FUNCTION__, cbuf_get(cbx), api_path);
     if (clicon_rpc_edit_config(h, "candidate", 
 			       OP_NONE,
 			       cbuf_get(cbx)) < 0){
-	//	notfound(r); /* XXX */
 	conflict(r);
-	goto done;
+	goto ok;
     }
-    if (clicon_rpc_validate(h, "candidate") < 0){
+    /* Assume this is validation failed since commit includes validate */
+    if (clicon_rpc_commit(h) < 0){
 	if (clicon_rpc_discard_changes(h) < 0)
 	    goto done;
 	badrequest(r);
-	retval = 0;
 	goto done;
     }
-    if (clicon_rpc_commit(h) < 0)
-	goto done;
     FCGX_SetExitStatus(201, r->out); /* Created */
     FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n");
-    // XXX api_path can be null    FCGX_FPrintF(r->out, "Location: %s\r\n", api_path);
     FCGX_FPrintF(r->out, "\r\n");
+ ok:
     retval = 0;
  done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
@@ -431,6 +509,58 @@ api_data_post(clicon_handle h,
      if (cbx)
 	cbuf_free(cbx); 
    return retval;
+} /* api_data_post */
+
+
+/*! Check matching keys
+ *
+ * @param[in] y        Yang statement, should be list or leaf-list
+ * @param[in] xdata    XML data tree
+ * @param[in] xapipath XML api-path tree
+ * @retval    0        Yes, keys match
+ * @retval    -1        No keys do not match
+ * If the target resource represents a YANG leaf-list, then the PUT
+ * method MUST NOT change the value of the leaf-list instance.
+ *
+ * If the target resource represents a YANG list instance, then the key
+ * leaf values, in message-body representation, MUST be the same as the
+ * key leaf values in the request URI.  The PUT method MUST NOT be used
+ * to change the key leaf values for a data resource instance.
+ */
+static int
+match_list_keys(yang_stmt *y,
+		cxobj     *xdata,
+		cxobj     *xapipath)
+{
+    int        retval = -1;
+    cvec      *cvk = NULL; /* vector of index keys */
+    cg_var    *cvi;
+    char      *keyname;
+    cxobj     *xkeya; /* xml key object in api-path */
+    cxobj     *xkeyd; /* xml key object in data */
+    char      *keya;
+    char      *keyd;
+
+    if (y->ys_keyword != Y_LIST &&y->ys_keyword != Y_LEAF_LIST)
+	return -1;
+    cvk = y->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */
+    cvi = NULL;
+    while ((cvi = cvec_each(cvk, cvi)) != NULL) {
+	keyname = cv_string_get(cvi);	    
+	if ((xkeya = xml_find(xapipath, keyname)) == NULL)
+	    goto done; /* No key in api-path */
+	    
+	keya = xml_body(xkeya);
+	if ((xkeyd = xml_find(xdata, keyname)) == NULL)
+	    goto done; /* No key in data */
+	keyd = xml_body(xkeyd);
+	if (strcmp(keya, keyd) != 0)
+	    goto done; /* keys dont match */
+    }
+    retval = 0;
+ done:
+    clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
+    return retval;
 }
 
 /*! Generic REST PUT  method 
@@ -441,6 +571,7 @@ api_data_post(clicon_handle h,
  * @param[in]  pi     Offset, where to start pcvec
  * @param[in]  qvec   Vector of query string (QUERY_STRING)
  * @param[in]  data   Stream input data
+ * @note restconf PUT is mapped to edit-config replace. 
  * @example
       curl -X PUT -d '{"enabled":"false"}' http://127.0.0.1/restconf/data/interfaces/interface=eth1
  *
@@ -454,7 +585,7 @@ api_data_post(clicon_handle h,
 int
 api_data_put(clicon_handle h,
 	     FCGX_Request *r, 
-	     char         *api_path, 
+	     char         *api_path0, 
 	     cvec         *pcvec, 
 	     int           pi,
 	     cvec         *qvec, 
@@ -465,61 +596,91 @@ api_data_put(clicon_handle h,
     int        i;
     cxobj     *xdata = NULL;
     cbuf      *cbx = NULL;
-    cxobj     *x;
+    cxobj     *xtop = NULL; /* xpath root */
     cxobj     *xbot = NULL;
-    cxobj     *xtop = NULL;
+    cxobj     *xparent;
-    cxobj     *xp;
+    cxobj     *x;
     yang_node *y = NULL;
     yang_spec *yspec;
     cxobj     *xa;
     char      *media_content_type;
     int        parse_xml = 0; /* By default expect and parse JSON */
+    char      *api_path;
 
     clicon_debug(1, "%s api_path:\"%s\" json:\"%s\"",
-		 __FUNCTION__, 
+		 __FUNCTION__, api_path0, data);
-		 api_path, data);
     media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp);
-    if (strcmp(media_content_type, "application/yang-data+xml")==0)
+    if (media_content_type &&
+	strcmp(media_content_type, "application/yang-data+xml")==0)
 	parse_xml++;
     if ((yspec = clicon_dbspec_yang(h)) == NULL){
 	clicon_err(OE_FATAL, 0, "No DB_SPEC");
 	goto done;
     }
+    api_path=api_path0;
     for (i=0; i<pi; i++)
 	api_path = index(api_path+1, '/');
     /* Create config top-of-tree */
     if ((xtop = xml_new("config", NULL, NULL)) == NULL)
 	goto done;
+    /* Translate api_path to xtop/xbot */
     xbot = xtop;
     if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0)
 	goto done;
     /* Parse input data as json or xml into xml */
     if (parse_xml){
 	if (xml_parse_string(data, NULL, &xdata) < 0){
-	    clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
+	    badrequest(r);
-	    goto done;
+	    goto ok;
 	}
     }
     else if (json_parse_str(data, &xdata) < 0){
-	clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
+	badrequest(r);
-	goto done;
+	goto ok;
     }
+    /* The message-body MUST contain exactly one instance of the
+     * expected data resource. 
+     */
     if (xml_child_nr(xdata) != 1){
 	badrequest(r);
-	retval = 0;
+	goto ok;
-	goto done;
     }
     x = xml_child_i(xdata,0);
+    /* Add operation (create/replace) as attribute */
     if ((xa = xml_new("operation", x, NULL)) == NULL)
 	goto done;
     xml_type_set(xa, CX_ATTR);
     if (xml_value_set(xa, xml_operation2str(op)) < 0)
 	goto done;
-    /* Replace xbot with x */    
+    /* Replace xparent with x, ie bottom of api-path with data */	    
-    xp = xml_parent(xbot);
+    if (api_path==NULL && strcmp(xml_name(x),"data")==0){
-    xml_purge(xbot);
+	if (xml_addsub(NULL, x) < 0)
-    if (xml_addsub(xp, x) < 0) 	
 	    goto done;
+	if (xtop)
+	    xml_free(xtop);
+	xtop = x;
+	xml_name_set(xtop, "config");
+    }
+    else {
+	/* Check same symbol in api-path as data */	    
+	if (strcmp(xml_name(x), xml_name(xbot))){
+	    badrequest(r);
+	    goto ok;
+	}
+	/* If list or leaf-list, api-path keys must match data keys */	    
+	if (y && (y->yn_keyword == Y_LIST ||y->yn_keyword == Y_LEAF_LIST)){
+	    if (match_list_keys((yang_stmt*)y, x, xbot) < 0){
+		badrequest(r);
+		goto ok;
+	    }
+	}
+	xparent = xml_parent(xbot);
+	xml_purge(xbot);
+	if (xml_addsub(xparent, x) < 0)
+	    goto done;
+    }
+
+    /* Create text buffer for transfer to backend */
     if ((cbx = cbuf_new()) == NULL)
 	goto done;
     if (clicon_xml2cbuf(cbx, xtop, 0, 0) < 0)
@@ -529,21 +690,19 @@ api_data_put(clicon_handle h,
 			       OP_NONE,
 			       cbuf_get(cbx)) < 0){
 	notfound(r);
-	goto done;
+	goto ok;
     }
-    
+    /* Assume this is validation failed since commit includes validate */
-    if (clicon_rpc_validate(h, "candidate") < 0){
+    if (clicon_rpc_commit(h) < 0){
 	if (clicon_rpc_discard_changes(h) < 0) 
 	    goto done;
 	badrequest(r);
-	retval = 0;
 	goto done;
     }
-    if (clicon_rpc_commit(h) < 0)
-	goto done;
     FCGX_SetExitStatus(201, r->out); /* Created */
     FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n");
     FCGX_FPrintF(r->out, "\r\n");
+ ok:
     retval = 0;
  done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);
@@ -554,8 +713,7 @@ api_data_put(clicon_handle h,
      if (cbx)
 	cbuf_free(cbx); 
    return retval;
-
+} /* api_data_put */
-}
 
 /*! Generic REST PATCH  method 
  * @param[in]  h      CLIXON handle
@@ -632,13 +790,19 @@ api_data_delete(clicon_handle h,
 			       OP_NONE, 
 			       cbuf_get(cbx)) < 0){
 	notfound(r);
+	goto ok;
+    }
+    /* Assume this is validation failed since commit includes validate */
+    if (clicon_rpc_commit(h) < 0){
+	if (clicon_rpc_discard_changes(h) < 0) 
+	    goto done;
+	badrequest(r);
 	goto done;
     }
-    if (clicon_rpc_commit(h) < 0)
-	goto done;
     FCGX_SetExitStatus(201, r->out);
     FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n");
     FCGX_FPrintF(r->out, "\r\n");
+ ok:
     retval = 0;
  done:
     if (cbx)
@@ -649,6 +813,20 @@ api_data_delete(clicon_handle h,
    return retval;
 }
 
+/*! NYI
+ */
+int
+api_operation_get(clicon_handle h,
+		   FCGX_Request *r, 
+		   char         *path, 
+		   cvec         *pcvec, 
+		   int           pi,
+		   cvec         *qvec, 
+		   char         *data)
+{
+    return 0;
+}
+
 /*! REST operation POST method 
  * @param[in]  h      CLIXON handle
  * @param[in]  r      Fastcgi request handle
@@ -669,7 +847,8 @@ api_operation_post(clicon_handle h,
 		   cvec         *pcvec, 
 		   int           pi,
 		   cvec         *qvec, 
-		   char         *data)
+		   char         *data,
+		   char         *username)
 {
     int        retval = -1;
     int        i;
@@ -691,12 +870,16 @@ api_operation_post(clicon_handle h,
     int        parse_xml = 0; /* By default expect and parse JSON */
     char      *media_accept;
     int        use_xml = 0; /* By default return JSON */
+    int        pretty;
+    cxobj     *xa;
 	
     clicon_debug(1, "%s json:\"%s\" path:\"%s\"", __FUNCTION__, data, path);
+    pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY");
     if ((media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp)) &&
 	strcmp(media_accept, "application/yang-data+xml")==0)
 	use_xml++;
-    if ((media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp)) &&
+    media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp);
+    if (media_content_type &&
 	strcmp(media_content_type, "application/yang-data+xml")==0)
 	parse_xml++;
     clicon_debug(1, "%s accept:\"%s\" content-type:\"%s\"", 
@@ -714,7 +897,7 @@ api_operation_post(clicon_handle h,
 	goto done;
     if (yrpc == NULL){
 	retval = notfound(r); 
-	goto done;
+	goto ok;
     }
     /* Create an xml message: 
      * <"rpc"><operation><input-args>...
@@ -724,19 +907,28 @@ api_operation_post(clicon_handle h,
     if ((xtop = xml_new("rpc", NULL, NULL)) == NULL)
 	goto done;
     xbot = xtop;
+    /* XXX: something strange for rpc user */
     if (api_path2xml(oppath, yspec, xtop, 1, &xbot, &y) < 0)
 	goto done;
+#if 1
+    {
+	cbuf *c = cbuf_new();
+	clicon_xml2cbuf(c, xtop, 0, 0);
+	clicon_debug(1, "%s xinput:%s", __FUNCTION__, cbuf_get(c));
+	cbuf_free(c);
+    }
+#endif
     if (data && strlen(data)){
 	/* Parse input data as json or xml into xml */
 	if (parse_xml){
 	    if (xml_parse_string(data, NULL, &xdata) < 0){
-		clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
+		badrequest(r);
-		goto done;
+		goto ok;
 	    }
 	}
 	else if (json_parse_str(data, &xdata) < 0){
-	    clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data);
+	    badrequest(r);
-	    goto done;
+	    goto ok;
 	}
 	/* xdata should have format <top><input> */
 	if ((xinput = xpath_first(xdata, "/input")) != NULL){
@@ -759,8 +951,17 @@ api_operation_post(clicon_handle h,
 	    }
 	}
     }
-    /* Non-standard: add cookie as attribute for backend
+    /* Non-standard: add username attribute for backend ACM (RFC 6536)
+     * 
      */
+    if (username){
+	if ((xa = xml_new("username", xtop, NULL)) == NULL)
+	    goto done;
+	xml_type_set(xa, CX_ATTR);
+	if (xml_value_set(xa, username) < 0)
+	    goto done;
+    }
+#ifdef obsolete
     {
 	cxobj *xa;
 	char *cookie;
@@ -777,15 +978,17 @@ api_operation_post(clicon_handle h,
 		free(cookieval);
 	}
     }
+#endif
     /* Send to backend */
     if (clicon_rpc_netconf_xml(h, xtop, &xret, NULL) < 0)
 	goto done;
     if ((cbx = cbuf_new()) == NULL)
 	goto done;
     xoutput=xpath_first(xret, "/");
+    xml_name_set(xoutput, "output");
     if ((youtput = yang_find((yang_node*)yrpc, Y_OUTPUT, NULL)) != NULL &&
 	xoutput){
-	xml_name_set(xoutput, "output");
+
 	clicon_debug(1, "%s xoutput:%s", __FUNCTION__, cbuf_get(cbx));
 	cbuf_reset(cbx);
 	xml_spec_set(xoutput, youtput); /* needed for xml_spec_populate */
@@ -803,16 +1006,17 @@ api_operation_post(clicon_handle h,
     FCGX_FPrintF(r->out, "\r\n");
     if (xoutput){
 	if (use_xml){
-	    if (clicon_xml2cbuf(cbx, xoutput, 0, 1) < 0)
+	    if (clicon_xml2cbuf(cbx, xoutput, 0, pretty) < 0)
 		goto done;
 	}
 	else
-	    if (xml2json_cbuf(cbx, xoutput, 1) < 0)
+	    if (xml2json_cbuf(cbx, xoutput, pretty) < 0)
 		goto done;
 	clicon_debug(1, "%s xoutput:%s", __FUNCTION__, cbuf_get(cbx));
 	FCGX_FPrintF(r->out, "%s", cbx?cbuf_get(cbx):"");
 	FCGX_FPrintF(r->out, "\r\n\r\n");
     }
+ ok:
     retval = 0;
  done:
     clicon_debug(1, "%s retval:%d", __FUNCTION__, retval);

apps/restconf/restconf_methods.h

@@ -60,8 +60,13 @@ int api_data_patch(clicon_handle h, FCGX_Request *r, char *api_path,
 		   cvec *qvec, char *data);
 int api_data_delete(clicon_handle h, FCGX_Request *r, char *api_path, int pi);
 
-int api_operation_post(clicon_handle h, FCGX_Request *r, 
+int api_operation_get(clicon_handle h, FCGX_Request *r, 
 		       char *path,
 		       cvec *pcvec, int pi, cvec *qvec, char *data);
 
+int api_operation_post(clicon_handle h, FCGX_Request *r, 
+		       char *path,
+		       cvec *pcvec, int pi, cvec *qvec, char *data,
+		       char *username);
+
 #endif /* _RESTCONF_METHODS_H_ */

clixon.mk.cpp

@@ -42,19 +42,5 @@ clixon_DBSPECDIR=prefix/share/$(APPNAME)
 clixon_SYSCONFDIR=sysconfdir
 clixon_LOCALSTATEDIR=localstatedir/$(APPNAME)
 clixon_LIBDIR=libdir/$(APPNAME)
-clixon_DATADIR=datadir/clixon
+clixon_DATADIR=datadir/clixon # for system yang files
 
-# Rules for the clixon application configuration file.
-# The clixon applications should be started with this fileas its -f argument.
-# Typically installed in sysconfdir
-# Example: APPNAME=myapp --> clixon_cli -f /usr/local/etc/myapp.conf
-# The two variants are if there is a .conf.local file or not
-.PHONY: $(APPNAME).conf
-ifneq (,$(wildcard ${APPNAME}.conf.local)) 	
-${APPNAME}.conf:  ${clixon_DATADIR}/clixon.conf.cpp ${APPNAME}.conf.local
-	$(CPP) -P -x assembler-with-cpp -DAPPNAME=$(APPNAME) $< > $@
-	cat ${APPNAME}.conf.local >> $@
-else
-${APPNAME}.conf:  ${clixon_DATADIR}/clixon.conf.cpp
-	$(CPP) -P -x assembler-with-cpp -DAPPNAME=$(APPNAME) $< > $@
-endif

configure

@@ -632,9 +632,6 @@ CPP
 OBJEXT
 EXEEXT
 ac_ct_CC
-with_xml_compat
-with_config_compat
-with_startup_compat
 with_keyvalue
 with_restconf
 RANLIB
@@ -712,9 +709,7 @@ with_cligen
 with_restconf
 with_keyvalue
 with_qdbm
-with_startup_compat
+with_configfile
-with_config_compat
-with_xml_compat
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1353,9 +1348,7 @@ Optional Packages:
   --without-restconf      disable support for restconf
   --with-keyvalue         enable support for key-value xmldb datastore
   --with-qdbm=dir         Use QDBM here, if keyvalue
-  --with-startup-compat   Backward compatibility of backend startup commands
+  --with-configfile=FILE  set default path to config file
-  --with-config-compat    Backward compatibility of configuration file
-  --with-xml-compat       Backward compatibility of XML API
 
 Some influential environment variables:
   CC          C compiler command
@@ -2157,9 +2150,9 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 : ${CFLAGS="-O2"}
 
 CLIXON_VERSION_MAJOR="3"
-CLIXON_VERSION_MINOR="4"
+CLIXON_VERSION_MINOR="5"
 CLIXON_VERSION_PATCH="0"
-CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\""
+CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\""
 # Fix to specific version (eg 3.5) or head (3)
 CLIGEN_VERSION="3"
 if test "$prefix" = "NONE"; then
@@ -2357,13 +2350,6 @@ test -n "$target_alias" &&
 
  # If yes, compile apps/restconf
  # If yes, compile datastore/keyvalue
-# If yes, backward compatible with 3.3.2 backend startup
-
-# If yes, backward compatible with 3.3.2 .conf configuration
-
-# If yes, backward compatible with 3.3.3 XML api new and parse functions
-
-
 #
 ac_ext=c
 ac_cpp='$CPP $CPPFLAGS'
@@ -4061,60 +4047,15 @@ fi
 
 fi
 
-# This is for backward compatibility of backend startup commands in 3.3.3
+# Set default config file location
-# Will be removed in 3.4.0
 
-# Check whether --with-startup_compat was given.
+# Check whether --with-configfile was given.
-if test "${with_startup_compat+set}" = set; then :
+if test "${with_configfile+set}" = set; then :
-  withval=$with_startup_compat;
+  withval=$with_configfile; DEFAULT_CONFIG="$withval"
 else
-  with_startup_compat=no
+  DEFAULT_CONFIG="$(eval echo ${sysconfdir}/clixon.xml)"
 fi
 
-if test "x${with_startup_compat}" == xyes; then
-
-cat >>confdefs.h <<_ACEOF
-#define BACKEND_STARTUP_COMPAT $with_startup_compat
-_ACEOF
-
-fi
-
-# This is for backward compatibility of .conf configuration file in 3.3.3
-# Will be removed in 3.4.0
-
-# Check whether --with-config_compat was given.
-if test "${with_config_compat+set}" = set; then :
-  withval=$with_config_compat;
-else
-  with_config_compat=no
-fi
-
-if test "x${with_config_compat}" == xyes; then
-
-cat >>confdefs.h <<_ACEOF
-#define CONFIG_COMPAT $with_config_compat
-_ACEOF
-
-fi
-
-# Clixon 3.4.0 changes XML creation and parse API
-# Set this for backward compat and migration.
-# Will be removed in 3.5.0
-
-# Check whether --with-xml_compat was given.
-if test "${with_xml_compat+set}" = set; then :
-  withval=$with_xml_compat;
-else
-  with_xml_compat=no
-fi
-
-if test "x${with_xml_compat}" == xyes; then
-
-cat >>confdefs.h <<_ACEOF
-#define XML_COMPAT $with_xml_compat
-_ACEOF
-
-fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for crypt in -lcrypt" >&5
 $as_echo_n "checking for crypt in -lcrypt... " >&6; }
@@ -4361,6 +4302,13 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+# Default location for config file
+
+cat >>confdefs.h <<_ACEOF
+#define CLIXON_DEFAULT_CONFIG "${DEFAULT_CONFIG}"
+_ACEOF
+
+
 
 
 # See also datastore/keyvalue/Makefile in with_keyvalue clause above

configure.ac

@@ -42,9 +42,9 @@ AC_INIT(lib/clixon/clixon.h.in)
 : ${CFLAGS="-O2"}
 
 CLIXON_VERSION_MAJOR="3"
-CLIXON_VERSION_MINOR="4"
+CLIXON_VERSION_MINOR="5"
 CLIXON_VERSION_PATCH="0"
-CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\""
+CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\""
 # Fix to specific version (eg 3.5) or head (3) 
 CLIGEN_VERSION="3" 
 if test "$prefix" = "NONE"; then
@@ -86,13 +86,6 @@ AC_SUBST(AR)
 AC_SUBST(RANLIB)
 AC_SUBST(with_restconf) # If yes, compile apps/restconf
 AC_SUBST(with_keyvalue) # If yes, compile datastore/keyvalue
-# If yes, backward compatible with 3.3.2 backend startup
-AC_SUBST(with_startup_compat) 
-# If yes, backward compatible with 3.3.2 .conf configuration
-AC_SUBST(with_config_compat) 
-# If yes, backward compatible with 3.3.3 XML api new and parse functions
-AC_SUBST(with_xml_compat) 
-
 #
 AC_PROG_CC()
 AC_PROG_CPP
@@ -168,36 +161,11 @@ if test "x${with_keyvalue}" == xyes; then
    AC_CONFIG_FILES(datastore/keyvalue/Makefile)
 fi
 
-# This is for backward compatibility of backend startup commands in 3.3.3
+# Set default config file location
-# Will be removed in 3.4.0
+AC_ARG_WITH([configfile],
-AC_ARG_WITH([startup_compat],
+	    [AS_HELP_STRING([--with-configfile=FILE],[set default path to config file])],
-            [AS_HELP_STRING([--with-startup-compat],[Backward compatibility of backend startup commands])],
+	    [DEFAULT_CONFIG="$withval"],
-	    [],
+	    [DEFAULT_CONFIG="$(eval echo ${sysconfdir}/clixon.xml)"])
-            [with_startup_compat=no])
-if test "x${with_startup_compat}" == xyes; then
-   AC_DEFINE_UNQUOTED(BACKEND_STARTUP_COMPAT, $with_startup_compat, [Backward compatible backend startup command-line options])
-fi
-
-# This is for backward compatibility of .conf configuration file in 3.3.3
-# Will be removed in 3.4.0
-AC_ARG_WITH([config_compat],
-            [AS_HELP_STRING([--with-config-compat],[Backward compatibility of configuration file])],
-	    [],
-            [with_config_compat=no])
-if test "x${with_config_compat}" == xyes; then
-   AC_DEFINE_UNQUOTED(CONFIG_COMPAT, $with_config_compat, [Backward compatible of .conf configuration files])
-fi
-
-# Clixon 3.4.0 changes XML creation and parse API
-# Set this for backward compat and migration. 
-# Will be removed in 3.5.0
-AC_ARG_WITH([xml_compat],
-            [AS_HELP_STRING([--with-xml-compat],[Backward compatibility of XML API])],
-	    [],
-            [with_xml_compat=no])
-if test "x${with_xml_compat}" == xyes; then
-   AC_DEFINE_UNQUOTED(XML_COMPAT, $with_xml_compat, [Backward compatible of XML API])
-fi
 
 AC_CHECK_LIB(crypt, crypt)
 AC_CHECK_HEADERS(crypt.h)
@@ -220,6 +188,9 @@ AC_CHECK_FUNCS(inet_aton sigaction sigvec strlcpy strsep strndup alphasort versi
 # same as clixon_DATADIR defined in clixon.mk.cpp for Makefiles
 AC_DEFINE_UNQUOTED(CLIXON_DATADIR, "${prefix}/share/clixon", [Clixon data dir for system yang files etc])
 
+# Default location for config file
+AC_DEFINE_UNQUOTED(CLIXON_DEFAULT_CONFIG,"${DEFAULT_CONFIG}",[Location for apps to find default config file])
+
 AH_BOTTOM([#include <clixon_custom.h>])
 
 # See also datastore/keyvalue/Makefile in with_keyvalue clause above

datastore/text/clixon_xmldb_text.c

@@ -748,14 +748,19 @@ text_modify_top(cxobj              *x0,
     cxobj     *x0c; /* base child */
     cxobj     *x1c; /* mod child */
     yang_stmt *yc;  /* yang child */
+    char      *opstr;
 
     /* Assure top-levels are 'config' */
     assert(x0 && strcmp(xml_name(x0),"config")==0);
     assert(x1 && strcmp(xml_name(x1),"config")==0);
 
+    /* Check for operations embedded in tree according to netconf */
+    if ((opstr = xml_find_value(x1, "operation")) != NULL)
+	if (xml_operation(opstr, &op) < 0)
+	    goto done;
     /* Special case if x1 is empty, top-level only <config/> */
-    if (!xml_child_nr(x1)){ /* base tree not empty */
+    if (!xml_child_nr(x1)){ 
-	if (xml_child_nr(x0))
+	if (xml_child_nr(x0)) /* base tree not empty */
 	    switch(op){ 
 	    case OP_DELETE:
 	    case OP_REMOVE:

etc/Makefile.in

@@ -52,10 +52,10 @@ install:	 clixonrc
 	install -m 755 clixonrc $(DESTDIR)$(sysconfdir)
 
 install-include:	
-	rm -f $(DESTDIR)$(sysconfdir)/clixonrc
+
 
 uninstall:
-
+	rm -f $(DESTDIR)$(sysconfdir)/clixonrc
 
 depend:
 

example/routing_backend.c

@@ -134,6 +134,18 @@ fib_route(clicon_handle h,            /* Clicon handle */
     return 0;
 }
 
+/*! Smallest possible RPC declaration for test */
+static int 
+empty(clicon_handle h,            /* Clicon handle */
+	  cxobj        *xe,           /* Request: <rpc><xn></rpc> */
+	  struct client_entry *ce,    /* Client session */
+	  cbuf         *cbret,        /* Reply eg <rpc-reply>... */
+	  void         *arg)          /* Argument given at register */
+{
+    cprintf(cbret, "<rpc-reply/>");
+    return 0;
+}
+
 /*! IETF Routing route-count rpc */
 static int 
 route_count(clicon_handle h, 
@@ -201,6 +213,11 @@ plugin_init(clicon_handle h)
 				"route-count"/* Xml tag when callback is made */
 				) < 0)
 	goto done;
+    if (backend_rpc_cb_register(h, empty, 
+				NULL, 
+				"empty"/* Xml tag when callback is made */
+				) < 0)
+	goto done;
     retval = 0;
  done:
     return retval;

include/clixon_config.h.in

@@ -1,11 +1,11 @@
 /* include/clixon_config.h.in.  Generated from configure.ac by autoheader.  */
 
-/* Backward compatible backend startup command-line options */
-#undef BACKEND_STARTUP_COMPAT
-
 /* Clixon data dir for system yang files etc */
 #undef CLIXON_DATADIR
 
+/* Location for apps to find default config file */
+#undef CLIXON_DEFAULT_CONFIG
+
 /* Clixon major release */
 #undef CLIXON_VERSION_MAJOR
 
@@ -18,9 +18,6 @@
 /* Clixon version string */
 #undef CLIXON_VERSION_STRING
 
-/* Backward compatible of .conf configuration files */
-#undef CONFIG_COMPAT
-
 /* Define to 1 if you have the `alphasort' function. */
 #undef HAVE_ALPHASORT
 
@@ -138,9 +135,6 @@
 /* Define to 1 if you have the ANSI C header files. */
 #undef STDC_HEADERS
 
-/* Backward compatible of XML API */
-#undef XML_COMPAT
-
 /* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
    `char[]'. */
 #undef YYTEXT_POINTER

lib/clixon/clixon.h.in

@@ -56,8 +56,6 @@
 #undef CLIXON_VERSION_MINOR
 #undef CLIXON_VERSION_PATCH
 
-#undef XML_COMPAT
-
 /*
  * Use this constant to disable some prototypes that should not be visible outside the lib.
  * This is an alternative to use separate internal include files.

lib/clixon/clixon_plugin.h

@@ -77,7 +77,10 @@ typedef int (plgexit_t)(clicon_handle);		       /* Plugin exit */
  * Returns 0 if credentials OK, -1 if failed
  */
 #define PLUGIN_CREDENTIALS      "plugin_credentials"
-typedef int (plgcredentials_t)(clicon_handle, void *); /* Plugin credentials */
+/* Plugin credentials 
+ * username should be freed after use
+ */
+typedef int (plgcredentials_t)(clicon_handle, void *, char **username);
 
 /* Find a function in global namespace or a plugin. XXX clicon internal */
 void *clicon_find_func(clicon_handle h, char *plugin, char *func);

lib/clixon/clixon_xml.h

@@ -161,16 +161,4 @@ int       xml_body_uint32(cxobj *xb, uint32_t *val);
 int       xml_operation(char *opstr, enum operation_type *op);
 char     *xml_operation2str(enum operation_type op);
 
-#ifdef XML_COMPAT /* See CHANGELOG */
-/* MANUAL CHANGE: xml_new(name, parent) --> xml_new(name, parent, NULL) */
-
-#define xml_new_spec(name, parent) xml_new(name, parent, NULL)
-#define clicon_xml_parse(xt, fmt, ...) xml_parse_va(xt, NULL, fmt, __VA_ARGS__)
-#define clicon_xml_parse_file(fd, xt, etag) xml_parse_file(fd, etag, NULL, xt)
-#define clicon_xml_parse_string(strp, xt) xml_parse_string(*strp, NULL, xt)
-#define clicon_xml_parse_str(str, xt) xml_parse_string(str, NULL, xt)
-#define xml_parse(str, xt) xml_parse_string(str, NULL, &xt)
-#endif
-
-
 #endif /* _CLIXON_XML_H */

lib/src/clixon_json.c

@@ -77,9 +77,10 @@
 
 enum array_element_type{
     NO_ARRAY=0,
-    FIRST_ARRAY,
+    FIRST_ARRAY,  /* [a, */
-    MIDDLE_ARRAY,
+    MIDDLE_ARRAY, /*  a, */
-    LAST_ARRAY,
+    LAST_ARRAY,   /*  a] */
+    SINGLE_ARRAY, /* [a] */
     BODY_ARRAY
 };
 
@@ -143,6 +144,9 @@ arraytype2str(enum array_element_type lt)
     case LAST_ARRAY:
 	return "last";
 	break;
+    case SINGLE_ARRAY:
+	return "single";
+	break;
     case BODY_ARRAY:
 	return "body";
 	break;
@@ -158,11 +162,13 @@ array_eval(cxobj *xprev,
     enum array_element_type array = NO_ARRAY;
     int                     eqprev=0;
     int                     eqnext=0;
+    yang_stmt              *ys;
 
     if (xml_type(x)!=CX_ELMNT){
 	array=BODY_ARRAY;
 	goto done;
     }
+    ys = xml_spec(x);
     if (xnext && 
 	xml_type(xnext)==CX_ELMNT &&
 	strcmp(xml_name(x),xml_name(xnext))==0)
@@ -177,14 +183,18 @@ array_eval(cxobj *xprev,
 	array = LAST_ARRAY;
     else if (eqnext)
 	array = FIRST_ARRAY;
+    else  if (ys && ys->ys_keyword == Y_LIST)
+	array = SINGLE_ARRAY;
     else
 	array = NO_ARRAY;
  done:
     return array;
 }
 
-char *
+/*! Escape a json string
-json_escape(char *str)
+ */
+static char *
+json_str_escape(char *str)
 {
     int   i, j;
     char *snew;
@@ -223,34 +233,35 @@ json_escape(char *str)
  * @param[in]    level     Indentation level
  * @param[in]    pretty    Pretty-print output (2 means debug)
  * @param[in]    flat      Dont print NO_ARRAY object name (for _vec call)
+ * @param[in]    bodystr   Set if value is string, 0 otherwise. Only if body
  *
  * @note Does not work with XML attributes
  * The following matrix explains how the mapping is done.
  * You need to understand what arraytype means (no/first/middle/last)
  * and what childtype is (null,body,any)
-  +---------+--------------+--------------+--------------+
+  +----------+--------------+--------------+--------------+
   |array,leaf| null         | body         | any          |
-  +---------+--------------+--------------+--------------+
+  +----------+--------------+--------------+--------------+
   |no        | <a/>         |<a>1</a>      |<a><b/></a>   |
   |          |              |              |              |
   |  json:   |\ta:null      |\ta:          |\ta:{\n       |
   |          |              |              |\n}           |
-  +---------+--------------+--------------+--------------+
+  +----------+--------------+--------------+--------------+
   |first     |<a/><a..      |<a>1</a><a..  |<a><b/></a><a.|
   |          |              |              |              |
   |  json:   |\ta:[\n\tnull |\ta:[\n\t     |\ta:[\n\t{\n  |
   |          |              |              |\n\t}         |
-  +---------+--------------+--------------+--------------+
+  +----------+--------------+--------------+--------------+
   |middle    |..a><a/><a..  |.a><a>1</a><a.|              |
   |          |              |              |              |
   |  json:   |\tnull        |\t            |\t{a          |
   |          |              |              |\n\t}         |
-  +---------+--------------+--------------+--------------+
+  +----------+--------------+--------------+--------------+
   |last      |..a></a>      |..a><a>1</a>  |              |
   |          |              |              |              |
   |  json:   |\tnull        |\t            |\t{a          |
   |          |\n\t]         |\n\t]         |\n\t}\t]      |
-  +---------+--------------+--------------+--------------+
+  +----------+--------------+--------------+--------------+
  */
 static int 
 xml2json1_cbuf(cbuf                  *cb,
@@ -258,26 +269,35 @@ xml2json1_cbuf(cbuf                  *cb,
 	       enum array_element_type arraytype,
 	       int                    level,
 	       int                    pretty,
-	       int                    flat)
+	       int                    flat,
+	       int                    bodystr)
 {
     int              retval = -1;
     int              i;
     cxobj           *xc;
     enum childtype   childt;
     enum array_element_type xc_arraytype;
+    yang_stmt       *ys;
+    int              bodystr0=1;
 
     childt = childtype(x);
+    ys = xml_spec(x);
     if (pretty==2)
 	cprintf(cb, "#%s_array, %s_child ", 
 		arraytype2str(arraytype),
 		childtype2str(childt));
     switch(arraytype){
     case BODY_ARRAY:{
+	if (bodystr){
 	    char      *str;
-	if ((str = json_escape(xml_value(x))) == NULL)
+	    if ((str = json_str_escape(xml_value(x))) == NULL)
 		goto done;
 	    cprintf(cb, "\"%s\"", str);
 	    free(str);
+	}
+	else
+	    cprintf(cb, "%s", xml_value(x));
+
 	break;
     }
     case NO_ARRAY:
@@ -299,6 +319,7 @@ xml2json1_cbuf(cbuf                  *cb,
 	}
 	break;
     case FIRST_ARRAY:
+    case SINGLE_ARRAY:
 	cprintf(cb, "%*s\"%s\": ", 
 		pretty?(level*JSON_INDENT):0, "", 
 		xml_name(x));
@@ -340,6 +361,30 @@ xml2json1_cbuf(cbuf                  *cb,
     default:
 	break;
     }
+    /* Check for typed sub-body if:
+     * arracytype=* but chilt-type is BODY_CHILD 
+     * This is code for writing <a>42</a> as "a":42 and not "a":"42"
+     */
+    if (childt == BODY_CHILD && ys!=NULL &&
+	(ys->ys_keyword == Y_LEAF || ys->ys_keyword == Y_LEAF_LIST))
+	switch (cv_type_get(ys->ys_cv)){
+	case CGV_INT8:
+	case CGV_INT16:
+	case CGV_INT32:
+	case CGV_INT64:
+	case CGV_UINT8:
+	case CGV_UINT16:
+	case CGV_UINT32:
+	case CGV_UINT64:
+	case CGV_DEC64:
+	case CGV_BOOL:
+	    bodystr0 = 0;
+	    break;
+	default:
+	    bodystr0 = 1;
+	    break;
+	}
+
     for (i=0; i<xml_child_nr(x); i++){
 	xc = xml_child_i(x, i);
 	xc_arraytype = array_eval(i?xml_child_i(x,i-1):NULL, 
@@ -348,7 +393,7 @@ xml2json1_cbuf(cbuf                  *cb,
 	if (xml2json1_cbuf(cb, 
 			   xc, 
 			   xc_arraytype,
-			   level+1, pretty,0) < 0)
+			   level+1, pretty, 0, bodystr0) < 0)
 	    goto done;
 	if (i<xml_child_nr(x)-1)
 	    cprintf(cb, ",%s", pretty?"\n":"");
@@ -387,6 +432,7 @@ xml2json1_cbuf(cbuf                  *cb,
 	    break;
 	}
 	break;
+    case SINGLE_ARRAY:
     case LAST_ARRAY:
 	switch (childt){
 	case NULL_CHILD:
@@ -446,7 +492,7 @@ xml2json_cbuf(cbuf  *cb,
     if (xml2json1_cbuf(cb, 
 		       x, 
 		       NO_ARRAY,
-		       level+1, pretty,0) < 0)
+		       level+1, pretty,0,1) < 0)
 	goto done;
     cprintf(cb, "%s%*s}%s", 
 	    pretty?"\n":"",
@@ -496,7 +542,7 @@ xml2json_cbuf_vec(cbuf   *cb,
     if (xml2json1_cbuf(cb, 
 		       xp, 
 		       NO_ARRAY,
-		       level+1, pretty,1) < 0)
+		       level+1, pretty,1, 1) < 0)
 	goto done;
 
     if (0){
@@ -519,6 +565,8 @@ xml2json_cbuf_vec(cbuf   *cb,
  * @retval     0      OK
  * @retval    -1      Error
  *
+ * @note yang is necessary to translate to one-member lists,
+ * eg if a is a yang LIST <a>0</a> -> {"a":["0"]} and not {"a":"0"}
  * @code
  * if (xml2json(stderr, xn, 0) < 0)
  *   goto err;
@@ -623,7 +671,7 @@ json_parse(char       *str,
  * @param[in]  str   String containing JSON
  * @param[out] xt    On success a top of XML parse tree is created with name 'top'
  * @retval  0  OK
- * @retval -1  Error with clicon_err called
+ * @retval -1  Error with clicon_err called. Includes parse errors
  *
  * @code
  *  cxobj *cx = NULL;
@@ -645,7 +693,7 @@ json_parse_str(char   *str,
 /*! Read a JSON definition from file and parse it into a parse-tree. 
  *
  * @param[in]  fd  A file descriptor containing the JSON file (as ASCII characters)
- * @param[in]  yspec   Yang specification, or NULL
+ * @param[in]  yspec   Yang specification, or NULL XXX Not yet used
  * @param[in,out] xt   Pointer to (XML) parse tree. If empty, create.
  * @retval        0  OK
  * @retval       -1  Error with clicon_err called
@@ -722,7 +770,7 @@ json_parse_file(int        fd,
 /*
  * Turn this on to get a json parse and pretty print test program
  * Usage: xpath
- * read xml from input
+ * read json from input
  * Example compile:
  gcc -g -o json -I. -I../clixon ./clixon_json.c -lclixon -lcligen
  * Example run:

lib/src/clixon_options.c

@@ -183,168 +183,6 @@ clicon_option_readfile_xml(clicon_hash_t *copt,
     return retval;
 }
 
-#ifdef CONFIG_COMPAT
-
-/*! Read filename and set values to global options registry
- * For legacy configuration file, ie not xml
- * @see clicon_option_readfile_xml
- */
-static int 
-clicon_option_readfile(clicon_hash_t *copt, 
-		       const char    *filename)
-{
-    struct stat st;
-    char        opt[1024];
-    char        val[1024];
-    char        line[1024];
-    char       *cp;
-    FILE       *f = NULL;
-    int         retval = -1;
-
-    if (filename == NULL || !strlen(filename)){
-	clicon_err(OE_UNIX, 0, "Not specified");
-	goto done;
-    }
-    if (stat(filename, &st) < 0){
-	clicon_err(OE_UNIX, errno, "%s", filename);
-	goto done;
-    }
-    if (!S_ISREG(st.st_mode)){
-	clicon_err(OE_UNIX, 0, "%s is not a regular file", filename);
-	goto done;
-    }
-    if ((f = fopen(filename, "r")) == NULL) {
-	clicon_err(OE_UNIX, errno, "configure file: %s", filename);
-	return -1;
-    }
-    clicon_debug(2, "Reading config file %s", __FUNCTION__, filename);
-    while (fgets(line, sizeof(line), f)) {
-	if ((cp = strchr(line, '\n')) != NULL) /* strip last \n */
-	    *cp = '\0';
-	/* Trim out comments, strip whitespace, and remove CR */
-	if ((cp = strchr(line, '#')) != NULL)
-	    memcpy(cp, "\n", 2);
-	if (sscanf(line, "%s %s", opt, val) < 2)
-	    continue;
-	if ((hash_add(copt, 
-		      opt,
-		      val,
-		      strlen(val)+1)) == NULL)
-	    goto done;
-    }
-    retval = 0;
-  done:
-    if (f)
-	fclose(f);
-    return retval;
-}
-
-/*! Set default values of some options that may not appear in config-file
- */
-static int
-clicon_option_default(clicon_hash_t  *copt)
-{
-    char           *val;
-    int             retval = 0;
-
-    if (!hash_lookup(copt, "CLICON_YANG_MODULE_MAIN")){
-	if (hash_add(copt, "CLICON_YANG_MODULE_MAIN", "clicon", strlen("clicon")+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_SOCK_GROUP")){
-	val = CLICON_SOCK_GROUP;
-	if (hash_add(copt, "CLICON_SOCK_GROUP", val, strlen(val)+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_CLI_MODE")){
-	if (hash_add(copt, "CLICON_CLI_MODE", "base", strlen("base")+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_MASTER_PLUGIN")){
-	val = CLICON_MASTER_PLUGIN;
-	if (hash_add(copt, "CLICON_MASTER_PLUGIN", val, strlen(val)+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_CLI_GENMODEL")){
-	if (hash_add(copt, "CLICON_CLI_GENMODEL", "1", strlen("1")+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_CLI_GENMODEL_TYPE")){
-	if (hash_add(copt, "CLICON_CLI_GENMODEL_TYPE", "VARS", strlen("VARS")+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_AUTOCOMMIT")){
-	if (hash_add(copt, "CLICON_AUTOCOMMIT", "0", strlen("0")+1) < 0)
-	    goto done;
-    }
-    /* Legacy is 1 but default should really be 0. New apps should use 0 */
-    if (!hash_lookup(copt, "CLICON_CLI_VARONLY")){
-	if (hash_add(copt, "CLICON_CLI_VARONLY", "1", strlen("1")+1) < 0)
-	    goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_CLI_GENMODEL_COMPLETION")){
-	if (hash_add(copt, "CLICON_CLI_GENMODEL_COMPLETION", "1", strlen("1")+1) < 0)
-	    goto done;
-    }
-    /* Default is to use line-scrolling */
-    if (!hash_lookup(copt, "CLICON_CLI_LINESCROLLING")){
-       if (hash_add(copt, "CLICON_CLI_LINESCROLLING", "1", strlen("1")+1) < 0)
-           goto done;
-    }
-    retval = 0;
-  done:
-    return retval;
-}
-
-/*! Check that options are set
- */
-static int 
-clicon_option_sanity(clicon_hash_t *copt)
-{
-    int retval = -1;
-
-    if (!hash_lookup(copt, "CLICON_CLI_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_CLI_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_CLISPEC_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_CLISPEC_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_BACKEND_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_BACKEND_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_NETCONF_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_NETCONF_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_RESTCONF_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_RESTCONF_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_YANG_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_YANG_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_XMLDB_DIR")){
-	clicon_err(OE_UNIX, 0, "CLICON_XMLDB_DIR not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_SOCK")){
-	clicon_err(OE_UNIX, 0, "CLICON_SOCK not defined in config file");
-	goto done;
-    }
-    if (!hash_lookup(copt, "CLICON_BACKEND_PIDFILE")){
-	clicon_err(OE_UNIX, 0, "CLICON_BACKEND_PIDFILE not defined in config file");
-	goto done;
-    }
-    retval = 0;
- done:
-    return retval;
-}
-#endif /* CONFIG_COMPAT */
-
 /*! Initialize option values
  *
  * Set default options, Read config-file, Check that all values are set.
@@ -364,8 +202,7 @@ clicon_options_main(clicon_handle h)
      * Set configure file if not set by command-line above
      */
     if (!hash_lookup(copt, "CLICON_CONFIGFILE")){ 
-	clicon_err(OE_CFG, 0, "CLICON_CONFIGFILE (-f) not set");
+	clicon_option_str_set(h, "CLICON_CONFIGFILE", CLIXON_DEFAULT_CONFIG);
-	goto done;
     }
     configfile = hash_value(copt, "CLICON_CONFIGFILE", NULL);
     clicon_debug(1, "CLICON_CONFIGFILE=%s", configfile);
@@ -389,19 +226,8 @@ clicon_options_main(clicon_handle h)
 	    xml_child_sort = 0;
     }
     else {
-#ifdef CONFIG_COMPAT
-	/* Set default options */
-	if (clicon_option_default(copt) < 0)  /* init registry from file */
-	    goto done;
-	/* Read configfile */
-	if (clicon_option_readfile(copt, configfile) < 0)
-	    goto done;
-	if (clicon_option_sanity(copt) < 0)
-	    goto done;
-#else /* CONFIG_COMPAT */
 	clicon_err(OE_CFG, 0, "%s: suffix %s not recognized (Run ./configure --with-config-compat?)", configfile, suffix);
 	goto done;
-#endif /* CONFIG_COMPAT */
     }
     retval = 0;
  done:

lib/src/clixon_proto_client.c

@@ -71,10 +71,11 @@
 /*! Send internal netconf rpc from client to backend
  * @param[in]    h      CLICON handle
  * @param[in]    msg    Encoded message. Deallocate woth free
- * @param[out]   xret   Return value from backend as netconf xml tree. Free w xml_free
+ * @param[out]   xret   Return value from backend as xml tree. Free w xml_free
  * @param[inout] sock0  If pointer exists, do not close socket to backend on success 
  *                      and return it here. For keeping a notify socket open
- * Note: sock0 is if connection should be persistent, like a notification/subscribe api
+ * @note sock0 is if connection should be persistent, like a notification/subscribe api
+ * @note xret is populated with yangspec according to standard handle yangspec
  */
 int
 clicon_rpc_msg(clicon_handle      h, 
@@ -87,6 +88,7 @@ clicon_rpc_msg(clicon_handle      h,
     int                port;
     char              *retdata = NULL;
     cxobj             *xret = NULL;
+    yang_spec         *yspec;
 
     if ((sock = clicon_sock(h)) == NULL){
 	clicon_err(OE_FATAL, 0, "CLICON_SOCK option not set");
@@ -119,9 +121,12 @@ clicon_rpc_msg(clicon_handle      h,
 	break;
     }
     clicon_debug(1, "%s retdata:%s", __FUNCTION__, retdata);
-    if (retdata &&
+
-	xml_parse_string(retdata, NULL, &xret) < 0)
+    if (retdata){
+ 	yspec = clicon_dbspec_yang(h);
+	if (xml_parse_string(retdata, yspec, &xret) < 0)
 	    goto done;
+    }
     if (xret0){
 	*xret0 = xret;
 	xret = NULL;

lib/src/clixon_xml.c

@@ -595,7 +595,7 @@ xml_find(cxobj *x_up,
 }
 
 /*! Append xc as child to xp. Remove xc from previous parent.
- * @param[in] xp  Parent xml node
+ * @param[in] xp  Parent xml node. If NULL just remove from old parent.
  * @param[in] xc  Child xml node to insert under xp
  * @retval    0   OK
  * @retval    -1  Error
@@ -618,10 +618,12 @@ xml_addsub(cxobj *xp,
 	    xml_child_rm(oldp, i);
     }
     /* Add xc to new parent */
+    if (xp){
 	if (xml_child_append(xp, xc) < 0)
 	    return -1;
 	/* Set new parent in child */
 	xml_parent_set(xc, xp);
+    }
     return 0;
 }
 
@@ -1356,7 +1358,7 @@ xml_parse_file(int        fd,
  * @param[in]     yspec Yang specification, or NULL
  * @param[in,out] xt    Pointer to XML parse tree. If empty will be created.
  * @retval        0     OK
- * @retval       -1  Error with clicon_err called
+ * @retval       -1     Error with clicon_err called. Includes parse error
  *
  * @code
  *  cxobj *xt = NULL;
@@ -1702,6 +1704,7 @@ xml_apply_ancestor(cxobj          *xn,
  * @param[out]  cvp      CLIgen variable containing the parsed value
  * @note free cv with cv_free after use.
  * @see xml_body_int32   etc, for type-specific parse functions
+ * @note range check failure returns 0
  */
 int
 xml_body_parse(cxobj       *xb,
@@ -1749,6 +1752,7 @@ xml_body_parse(cxobj       *xb,
  *                          alloc error.
  * @note extend to all other cligen var types and generalize
  * @note use yang type info?
+ * @note range check failure returns 0
  */
 int
 xml_body_int32(cxobj    *xb,
@@ -1772,6 +1776,7 @@ xml_body_int32(cxobj    *xb,
  *                          alloc error.
  * @note extend to all other cligen var types and generalize
  * @note use yang type info?
+ * @note range check failure returns 0
  */
 int
 xml_body_uint32(cxobj    *xb,

lib/src/clixon_xml_map.c

@@ -88,9 +88,6 @@
 #include "clixon_xml_sort.h"
 #include "clixon_xml_map.h"
 
-/* Something to do with reverse engineering of junos syntax? */
-#undef SPECIAL_TREATMENT_OF_NAME  
-
 /*
  * A node is a leaf if it contains a body.
  */
@@ -131,9 +128,6 @@ xml2txt(FILE  *f,
     char  *term = NULL;
     int    retval = -1;
     int    encr=0;
-#ifdef SPECIAL_TREATMENT_OF_NAME  
-    cxobj *xname;
-#endif
 
     xe = NULL;     /* count children */
     while ((xe = xml_child_each(x, xe, -1)) != NULL)
@@ -157,32 +151,11 @@ xml2txt(FILE  *f,
 	goto done;
     }
     fprintf(f, "%*s", 4*level, "");
-
-#ifdef SPECIAL_TREATMENT_OF_NAME  
-    if (strcmp(xml_name(x), "name") != 0)
-	fprintf(f, "%s ", xml_name(x));
-    if ((xname = xml_find(x, "name")) != NULL){
-	if (children > 1)
-	    fprintf(f, "%s ", xml_body(xname));
-	if (!tleaf(x))
-	    fprintf(f, "{\n");
-    }
-    else
-	if (!tleaf(x))
-	    fprintf(f, "{\n");
-#else
     fprintf(f, "%s ", xml_name(x));
     if (!tleaf(x))
 	fprintf(f, "{\n");
-#endif /* SPECIAL_TREATMENT_OF_NAME */
-
     xe = NULL;
     while ((xe = xml_child_each(x, xe, -1)) != NULL){
-#ifdef SPECIAL_TREATMENT_OF_NAME  
-	if (xml_type(xe) == CX_ELMNT &&  (strcmp(xml_name(xe), "name")==0) && 
-	    (children > 1)) /* skip if this is a name element (unless 0 children) */
-	    continue;
-#endif
 	if (xml2txt(f, xe, level+1) < 0)
 	    break;
     }
@@ -442,6 +415,8 @@ xml_yang_validate_all(cxobj   *xt,
  * @retval     0    Everything OK, cvv allocated and set
  * @retval    -1    Something wrong, clicon_err() called to set error. No cvv returned
  * 'Not recursive' means that only one level of XML bodies is translated to cvec:s.
+ * If range is wriong (eg 1000 for uint8) a warning is logged, the value is 
+ * skipped, and continues.
  * yang is needed to know which type an xml element has.
  * Example: 
     <a>
@@ -484,28 +459,30 @@ xml2cvec(cxobj      *xt,
 	    clicon_debug(0, "%s: yang sanity problem: %s in xml but not present in yang under %s",
 			 __FUNCTION__, name, yt->ys_argument);
 	    if ((body = xml_body(xc)) != NULL){
-		if ((cv = cvec_add(cvv, CGV_STRING)) == NULL){
+		if ((cv = cv_new(CGV_STRING)) == NULL){
-		    clicon_err(OE_PLUGIN, errno, "cvec_add");
+		    clicon_err(OE_PLUGIN, errno, "cv_new");
 		    goto err;
 		}
 		cv_name_set(cv, name);
 		if ((ret = cv_parse1(body, cv, &reason)) < 0){
-		    clicon_err(OE_PLUGIN, errno, "cv_parse");
+		    clicon_err(OE_PLUGIN, errno, "cv_parse %s",name);
 		    goto err;
 		}
+		/* If value is out-of-range, log and skip value, and continue */
 		if (ret == 0){
-		    clicon_err(OE_PLUGIN, errno, "cv_parse: %s", reason);
+		    clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason); 
 		    if (reason)
 			free(reason);
-		    goto err;
 		}
+		else
+		    cvec_append_var(cvv, cv); /* Add to variable vector */
+		cv_free(cv);
 	    }
 	}
 	else if ((ycv = ys->ys_cv) != NULL){
 	    if ((body = xml_body(xc)) != NULL){
-		/* XXX: cvec_add uses realloc, can we avoid that? */
+		if ((cv = cv_new(CGV_STRING)) == NULL){
-		if ((cv = cvec_add(cvv, CGV_STRING)) == NULL){
+		    clicon_err(OE_PLUGIN, errno, "cv_new");
-		    clicon_err(OE_PLUGIN, errno, "cvec_add");
 		    goto err;
 		}
 		if (cv_cp(cv, ycv) < 0){
@@ -513,15 +490,17 @@ xml2cvec(cxobj      *xt,
 		    goto err;
 		}
 		if ((ret = cv_parse1(body, cv, &reason)) < 0){
-		    clicon_err(OE_PLUGIN, errno, "cv_parse");
+		    clicon_err(OE_PLUGIN, errno, "cv_parse: %s", name);
 		    goto err;
 		}
 		if (ret == 0){
-		    clicon_err(OE_PLUGIN, errno, "cv_parse: %s", reason);
+		    clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason);
 		    if (reason)
 			free(reason);
-		    goto err;
 		}
+		else
+		    cvec_append_var(cvv, cv); /* Add to variable vector */
+		cv_free(cv);
 	    }
 	}
     }
@@ -870,9 +849,18 @@ yang2api_path_fmt(yang_stmt *ys,
  * @param[out] yang_arg      yang-stmt argument name. Free after use
  * @note first and last elements of cvv are not used,..
  * @see api_path_fmt2xpath
- * 
+ * @example
- * /interfaces/interface=%s/name            --> /interfaces/interface/name
+ *  api_path_fmt: /interfaces/interface=%s/name
- * /interfaces/interface=%s/ipv4/address=%s --> /interfaces/interface=e/ipv4/address
+ *  cvv:          -
+ *  api_path:     /interfaces/interface/name
+ * @example
+ *  api_path_fmt: /interfaces/interface=%s/name
+ *  cvv:          e0
+ *  api_path:     /interfaces/interface=e0/name
+ * @example
+ *  api_path_fmt: /subif-entry=%s,%s/subid
+ *  cvv:          foo
+ *  api_path:     /subif-entry=foo/subid
  */
 int
 api_path_fmt2api_path(char  *api_path_fmt, 
@@ -889,20 +877,6 @@ api_path_fmt2api_path(char  *api_path_fmt,
     char *strenc=NULL;
     cg_var *cv;
     
-#if 1
-    /* Sanity check */
-    j = 0; /* Count % */
-    for (i=0; i<strlen(api_path_fmt); i++)
-	if (api_path_fmt[i] == '%')
-	    j++;
-    if (j > cvec_len(cvv)) { //cvec_len can be longer
-	clicon_log(LOG_WARNING, "%s api_path_fmt number of %% is %d, does not match number of cvv entries %d", 
-		   api_path_fmt, 
-		   j,
-		   cvec_len(cvv));
-	goto done;
-    }
-#endif
     if ((cb = cbuf_new()) == NULL){
 	clicon_err(OE_UNIX, errno, "cbuf_new");
 	goto done;
@@ -953,18 +927,22 @@ api_path_fmt2api_path(char  *api_path_fmt,
 
 /*! Transform an xml key format and a vector of values to an XML path
  * Used to input xmldb_get() or xmldb_get_vec
- * Add .* in last %s position.
- * Example: 
- *   api_path_fmt:  /interface/%s/address/%s 
- *   cvv:        name=eth0
- *   xmlkey:     /interface/[name=eth0]/address
- * Example2:
- *   xmlkeyfmt:  /ip/me/%s (if key)
- *   cvv:        -
- *   xmlkey:     /ipv4/me/a
  * @param[in]  api_path_fmt  XML key format
  * @param[in]  cvv    cligen variable vector, one for every wildchar in api_path_fmt
  * @param[out] xpath     XPATH
+ * Add .* in last %s position.
+ * @example
+ *   api_path_fmt:  /interface/%s/address/%s 
+ *   cvv:           name=eth0
+ *   xpath:         /interface/[name=eth0]/address
+ * @example
+ *   api_path_fmt:  /ip/me/%s (if key)
+ *   cvv:           -
+ *   xpath:        /ipv4/me/a
+ * @example 
+ *   api_path_fmt: /subif-entry=%s,%s/subid
+ *   cvv:          foo
+ *   xpath:        /subif-entry[if-name=foo]/subid"
  */ 
 int
 api_path_fmt2xpath(char  *api_path_fmt, 
@@ -980,21 +958,6 @@ api_path_fmt2xpath(char  *api_path_fmt,
     char   *str;
     cg_var *cv;
 
-    /* Sanity check: count '%' */
-#if 1
-    j = 0; /* Count % */
-    for (i=0; i<strlen(api_path_fmt); i++)
-	if (api_path_fmt[i] == '%')
-	    j++;
-    if (j > cvec_len(cvv)) {
-	clicon_log(LOG_WARNING, "%s xmlkey format string mismatch(j=%d, cvec_len=%d): %s", 
-		   api_path_fmt, 
-		   j,
-		   cvec_len(cvv), 
-		   cv_string_get(cvec_i(cvv, 0)));
-	goto done;
-    }
-#endif
     if ((cb = cbuf_new()) == NULL){
 	clicon_err(OE_UNIX, errno, "cbuf_new");
 	goto done;
@@ -1537,11 +1500,9 @@ api_path2xml_vec(char             **vec,
 	name = local;
     }
     if (y0->yn_keyword == Y_SPEC){ /* top-node */
-	clicon_debug(1, "%s 1 %s", __FUNCTION__, name);
 	y = yang_find_topnode((yang_spec*)y0, name, schemanode);
     }
     else {
-	clicon_debug(1, "%s 2 %s", __FUNCTION__, name);
 	y = schemanode?yang_find_schemanode((yang_node*)y0, name):
 	    yang_find_datanode((yang_node*)y0, name);
     }
@@ -1578,7 +1539,7 @@ api_path2xml_vec(char             **vec,
 	else{
 	    if ((valvec = clicon_strsep(restval, ",", &nvalvec)) == NULL)
 		goto done;
-	    if (cvec_len(cvk) != nvalvec){ 	    
+	    if (nvalvec > cvec_len(cvk)){ 	    
 		clicon_err(OE_XML, errno, "List %s  key length mismatch", name);
 		goto done;
 	    }
@@ -1626,24 +1587,32 @@ api_path2xml_vec(char             **vec,
 /*! Create xml tree from api-path
  * @param[in]     api_path   API-path as defined in RFC 8040
  * @param[in]     yspec      Yang spec
+ * @param[in,out] xtop       Incoming XML tree
  * @param[in]     schemanode If set use schema nodes otherwise data nodes.
- * @param[out]  xpathp     Resulting xml tree 
+ * @param[out]    xbotp      Resulting xml tree (end of xpath)
- * @param[out]  ypathp     Yang spec matching xpathp
+ * @param[out]    ybotp      Yang spec matching xbotp
+ * @example
+ *   api_path: /subif-entry=foo/subid
+ *   xtop[in]  <config/> 
+ *   xtop[out]:<config/> <subif-entry>
+ *                            <if-name>foo<if-name><subid/>>
+ *                       </subif-entry></config>
+ *   xbotp:    <subid/>
+ *   ybotp:    Y_LEAF subid
  * @see api_path2xml_vec
  */
 int
 api_path2xml(char       *api_path,
 	     yang_spec  *yspec,
-	     cxobj      *xpath,
+	     cxobj      *xtop,
 	     int         schemanode,
-	     cxobj     **xpathp,
+	     cxobj     **xbotp,
-	     yang_node **ypathp)
+	     yang_node **ybotp)
 {
     int    retval = -1;
     char **vec = NULL;
     int    nvec;
 
-    clicon_debug(1, "%s 0", __FUNCTION__);
     if (*api_path!='/'){
 	clicon_err(OE_DB, 0, "Invalid key: %s", api_path);
 	goto done;
@@ -1659,8 +1628,8 @@ api_path2xml(char       *api_path,
     }
     nvec--; /* NULL-terminated */
     if (api_path2xml_vec(vec+1, nvec, 
-			 xpath, (yang_node*)yspec, schemanode,
+			 xtop, (yang_node*)yspec, schemanode,
-			 xpathp, ypathp) < 0)
+			 xbotp, ybotp) < 0)
 	goto done;
     retval = 0;
  done:
@@ -1669,7 +1638,6 @@ api_path2xml(char       *api_path,
     return retval;
 }
 
-
 /*! Merge a base tree x0 with x1 with yang spec y
  * @param[in]  x0  Base xml tree (can be NULL in add scenarios)
  * @param[in]  y0  Yang spec corresponding to xml-node x0. NULL if x0 is NULL
@@ -1750,6 +1718,7 @@ xml_merge1(cxobj              *x0,
 /*! Merge XML trees x1 into x0 according to yang spec yspec
  * @note both x0 and x1 need to be top-level trees
  * @see text_modify_top as more generic variant (in datastore text)
+ * @note returns -1 if YANG do not match, you may want to have a softer error
  */
 int
 xml_merge(cxobj     *x0,

lib/src/clixon_xml_sort.c

@@ -549,6 +549,7 @@ match_base_child(cxobj     *x0,
     char     **keyval = NULL;
     char     **keyvec = NULL;
     int        i;
+    int        yorder;
     
     *x0cp = NULL; /* return value */
     switch (yc->ys_keyword){
@@ -591,49 +592,21 @@ match_base_child(cxobj     *x0,
 	break;
     }
     /* Get match */
-    {
-	yang_node *y0;
-	int        yorder;
-	    
-	if ((y0 = yc->ys_parent) == NULL)
-	    goto done;
-	/* XXX: No we cant do this. on uppermost layer it can look like this:
-	 * config 
-	 * ximport-----ymod = ietf 
-	 * interfaces--ymod = example
-	 * Which means yang order can be different for different children in the
-	 * same childvec.
-	 */
     if (xml_child_sort==0)
 	*x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
     else{
-#if 1
 	if (xml_child_nr(x0)==0 || xml_spec(xml_child_i(x0,0))!=NULL){
 	    yorder = yang_order(yc);
 	    *x0cp = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval);
 	}
 	else{
-		clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__);
+#if 1 /* This is just a warning, but a catcher for when xml tree is not
-		*x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
+	 populated with yang spec. If you see this, a previous invacation of,
-	    }
+	 for example  xml_spec_populate() may be missing
-#else
+      */
-	    cxobj *xx;
-
-
-
-	    *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
-	    if (xml_child_nr(x0) && xml_spec(xml_child_i(x0,0))!=NULL){
-		xx = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval);
-		if (xx!=*x0cp){
-		    clicon_log(LOG_WARNING, "%s mismatch", __FUNCTION__);
-		    fprintf(stderr, "mismatch\n");
-		    xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval);
-		    assert(0);
-		}
-	    }
-	    else
 	    clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__);
 #endif
+	    *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval);
 	}
     }
  ok:

lib/src/clixon_yang.c

@@ -549,10 +549,8 @@ yang_find_myprefix(yang_stmt *ys)
 	clicon_err(OE_YANG, 0, "My yang module not found");
 	goto done;
     }
-    if ((yprefix = yang_find((yang_node*)ymod, Y_PREFIX, NULL)) == NULL){
+    if ((yprefix = yang_find((yang_node*)ymod, Y_PREFIX, NULL)) == NULL)
-	clicon_err(OE_YANG, 0, "No prefix in my module");
 	goto done;
-    }
     prefix = yprefix->ys_argument;
  done:
     return prefix;
@@ -729,7 +727,6 @@ yang_find_module_by_prefix(yang_stmt *ys,
 	clicon_err(OE_YANG, 0, "My yang spec not found");
 	goto done;
     }
-    myprefix = yang_find_myprefix(ys);
     if ((my_ymod = ys_module(ys)) == NULL){
 	clicon_err(OE_YANG, 0, "My yang module not found");
 	goto done;
@@ -741,7 +738,8 @@ yang_find_module_by_prefix(yang_stmt *ys,
 	goto done;
     }
 #endif
-    if (strcmp(myprefix, prefix) == 0){
+    myprefix = yang_find_myprefix(ys);
+    if (myprefix && strcmp(myprefix, prefix) == 0){
 	ymod = my_ymod;
 	goto done;
     }
@@ -1453,7 +1451,7 @@ yang_parse_file(clicon_handle h,
     FILE         *f = NULL;
     struct stat st;
 
-    clicon_debug(1, "Yang parse file: %s", filename);
+    clicon_log(LOG_DEBUG, "Parsing yang file: %s", filename);
     if (stat(filename, &st) < 0){
 	clicon_err(OE_YANG, errno, "%s not found", filename);
 	goto done;

test/lib.sh

@@ -1,7 +1,31 @@
 #!/bin/bash
+# Define test functions.
+# Create working dir as variable "dir"
 
 testnr=0
 testname=
+
+# For memcheck
+#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"
+clixon_cli=clixon_cli
+
+# For memcheck / performance
+#clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+# clixon_netconf="valgrind --tool=callgrind clixon_netconf 
+clixon_netconf=clixon_netconf
+
+# How to run restconf stand-alone and using valgrind
+#sudo su -c "/www-data/clixon_restconf -f $cfg -D" -s /bin/sh www-data
+#sudo su -c "valgrind --leak-check=full --show-leak-kinds=all /www-data/clixon_restconf -f $cfg -D" -s /bin/sh www-data
+
+clixon_backend=clixon_backend
+
+dir=/var/tmp/$0
+if [ ! -d $dir ]; then
+    mkdir $dir
+fi
+rm -rf $dir/*
+
 # error and exit, arg is optional extra errmsg
 err(){
   echo "Error in Test$testnr [$testname]:"

test/plot_perf.sh

@@ -0,0 +1,164 @@
+#!/bin/bash
+# Transactions per second for large lists read/write plotter using gnuplot
+# WORK IN PROGRESS
+. ./lib.sh
+max=2000 # Nr of db entries
+step=200
+reqs=500
+cfg=$dir/plot-conf.xml
+fyang=$dir/plot.yang
+fconfig=$dir/config
+
+# For memcheck
+# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+# clixon_netconf="valgrind --tool=callgrind clixon_netconf 
+clixon_netconf=clixon_netconf
+
+cat <<EOF > $fyang
+module ietf-ip{
+   container x {
+    list y {
+      key "a";
+      leaf a {
+        type string;
+      }
+      leaf b {
+        type string;
+      }
+    }
+    leaf-list c {
+       type string;
+    }
+  }
+}
+EOF
+
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>$fyang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>ietf-ip</CLICON_YANG_MODULE_MAIN>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
+
+run(){
+    nr=$1 # Number of entries in DB
+    reqs=$2
+    mode=$3
+    
+    echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x>" > $fconfig
+    for (( i=0; i<$nr; i++ )); do  
+	case $mode in
+	    readlist|writelist|restreadlist|restwritelist)
+		echo -n "<y><a>$i</a><b>$i</b></y>" >> $fconfig
+		;;
+	    writeleaflist)
+		echo -n "<c>$i</c>" >> $fconfig
+		;;
+	    esac
+    done
+
+    echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig    
+
+    expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+    case $mode in
+	readlist)
+	time -p for (( i=0; i<$reqs; i++ )); do
+    rnd=$(( ( RANDOM % $nr ) ))
+    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
+            ;;
+	writelist)
+	    time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ))
+	echo "<rpc><edit-config><target><candidate/></target><config><x><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
+    ;;
+    	restreadlist)
+	time -p for (( i=0; i<$reqs; i++ )); do
+    rnd=$(( ( RANDOM % $nr ) ))
+    curl -sSG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null
+done 
+            ;;
+    writeleaflist)
+	time -p for (( i=0; i<$reqs; i++ )); do
+	rnd=$(( ( RANDOM % $nr ) ))
+	echo "<rpc><edit-config><target><candidate/></target><config><x><c>$rnd</c></x></config></edit-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg -y $fyang > /dev/null
+	    ;;
+    esac
+    expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+
+}
+
+step(){
+    i=$1
+    mode=$2
+    echo -n "" > $fconfig
+    t=$(TEST=%e run $i $reqs $mode 2>&1 | awk '/real/ {print $2}')
+    #TEST=%e run $i $reqs $mode 2>&1 
+    #  t is time in secs of $reqs -> transactions per second. $reqs
+    p=$(echo "$reqs/$t" | bc -lq)
+    # p is transactions per second. 
+    echo "$i $p" >> $dir/$mode
+#    echo "m:$mode i:$i t=$t p=$p" 
+}
+
+once(){
+    # kill old backend (if any)
+    sudo clixon_backend -zf $cfg -y $fyang
+    if [ $? -ne 0 ]; then
+	err
+    fi
+
+    # start new backend
+    sudo clixon_backend -s init -f $cfg -y $fyang
+    if [ $? -ne 0 ]; then
+	err
+    fi
+
+    # Always as a start
+    for (( i=10; i<=$step; i=i+10 )); do  
+	step $i readlist
+	step $i writelist
+	step $i restreadlist
+	step $i writeleaflist
+    done
+    # Actual steps
+    for (( i=$step; i<=$max; i=i+$step )); do  
+	step $i readlist
+	step $i writelist
+	step $i restreadlist
+	step $i writeleaflist
+    done
+
+    # Check if still alive
+    pid=`pgrep clixon_backend`
+    if [ -z "$pid" ]; then
+	err "backend already dead"
+    fi
+    # kill backend
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+	err "kill backend"
+    fi
+}
+
+once
+
+gnuplot -persist <<EOF
+set title "Clixon transactions per second r/w large lists" font ",14" textcolor rgbcolor "royalblue"
+set xlabel "entries"
+set ylabel "transactions per second"
+set terminal wxt  enhanced title "Clixon transactions " persist raise
+plot "$dir/readlist" with linespoints title "read list", "$dir/writelist" with linespoints title "write list", "$dir/writeleaflist" with linespoints title "write leaf-list" , "$dir/restreadlist" with linespoints title "rest get list" 
+EOF
+
+rm -rf $dir
+

test/test_cli.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+
 # Test1: backend and cli basic functionality
 # Start backend server
 # Add an ethernet interface and an address
@@ -7,13 +8,25 @@
 # Set the mandatory type
 # Commit
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/usr/local/etc/routing.xml
+cfg=$dir/conf_yang.xml
 
-# For memcheck
+cat <<EOF > $cfg
-#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"
+<config>
-clixon_cli=clixon_cli
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
 
 # kill old backend (if any)
 new "kill old backend"
@@ -22,7 +35,7 @@ if [ $? -ne 0 ]; then
     err
 fi
 new "start backend -s init -f $cfg"
-sudo clixon_backend -s init -f $cfg 
+sudo $clixon_backend -s init -f $cfg 
 if [ $? -ne 0 ]; then
     err
 fi
@@ -104,3 +117,4 @@ if [ $? -ne 0 ]; then
     err "kill backend"
 fi
 
+rm -rf $dir

test/test_datastore.sh

@@ -2,12 +2,13 @@
 # Test5: datastore tests. 
 # Just run a binary direct to datastore. No clixon.
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
+fyang=$dir/ietf-ip.yang
 
 datastore=datastore_client
 
-cat <<EOF > /tmp/ietf-ip.yang
+cat <<EOF > $fyang
 module ietf-ip{
    container x {
     list y {
@@ -46,14 +47,14 @@ db='<config><x><y><a>1</a><b>2</b><c>first-entry</c></y><y><a>1</a><b>3</b><c>se
 
 run(){
     name=$1
-    dir=/tmp/$name
+    mydir=$dir/$name
 
-    if [ ! -d $dir ]; then
+    if [ ! -d $mydir ]; then
-	mkdir $dir
+	mkdir $mydir
     fi
-    rm -rf $dir/*
+    rm -rf $mydir/*
 
-    conf="-d candidate -b $dir -p ../datastore/$name/$name.so -y /tmp -m ietf-ip"
+    conf="-d candidate -b $mydir -p ../datastore/$name/$name.so -y $dir -m ietf-ip"
     echo "conf:$conf"
     new "datastore $name init"
     expectfn "$datastore $conf init" ""
@@ -140,21 +141,23 @@ run(){
     expectfn "$datastore $conf put create <config><x><y><a>1</a><b>3</b><c>newentry</c></y></x></config>"
 
     new "datastore other db init"
-    expectfn "$datastore -d kalle -b $dir -p ../datastore/$name/$name.so -y /tmp -m ietf-ip init"
+    expectfn "$datastore -d kalle -b $mydir -p ../datastore/$name/$name.so -y $dir -m ietf-ip init"
 
     new "datastore other db copy"
     expectfn "$datastore $conf copy kalle" ""
 
-    diff $dir/kalle_db $dir/candidate_db
+    diff $mydir/kalle_db $mydir/candidate_db
 
     new "datastore lock"
     expectfn "$datastore $conf lock 756" ""
 
 #leaf-list
 
-    rm -rf $dir
+    rm -rf $mydir
 }
 
 #run keyvalue # cant get the put to work
 run text
 
+rm -rf $dir
+

test/test_install.sh

@@ -1,67 +1,60 @@
 #!/bin/bash
 # Install test
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
 
-DIR=/tmp/clixoninstall
+new "Set up installdir $dir"
-
-new "Set up installdir $DIR"
-rm -rf $DIR
-mkdir $DIR
 
 new "Make DESTDIR install"
-(cd ..; make DESTDIR=$DIR install)
+(cd ..; make DESTDIR=$dir install)
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "Check installed files"
-if [ ! -d $DIR/usr ]; then
+if [ ! -d $dir/usr ]; then
-    err $DIR/usr
+    err $dir/usr
 fi
-if [ ! -d $DIR/www-data ]; then
+if [ ! -d $dir/www-data ]; then
-    err $DIR/www-data
+    err $dir/www-data
 fi
-if [ ! -f $DIR/usr/local/share/clixon/clixon.mk ]; then
+if [ ! -f $dir/usr/local/share/clixon/clixon.mk ]; then
-    err $DIR/usr/local/share/clixon/clixon.mk
+    err $dir/usr/local/share/clixon/clixon.mk
 fi
-if [ ! -f $DIR/usr/local/share/clixon/clixon.conf.cpp ]; then
+if [ ! -f $dir/usr/local/share/clixon/clixon-config* ]; then
-    err $DIR/usr/local/share/clixon/clixon.conf.cpp
+    err $dir/usr/local/share/clixon/clixon-config*
 fi
-if [ ! -f $DIR/usr/local/share/clixon/clixon-config* ]; then
+if [ ! -h $dir/usr/local/lib/libclixon.so ]; then
-    err $DIR/usr/local/share/clixon/clixon-config*
+    err $dir/usr/local/lib/libclixon.so
 fi
-if [ ! -h $DIR/usr/local/lib/libclixon.so ]; then
+if [ ! -h $dir/usr/local/lib/libclixon_backend.so ]; then
-    err $DIR/usr/local/lib/libclixon.so
+    err $dir/usr/local/lib/libclixon_backend.so
-fi
-if [ ! -h $DIR/usr/local/lib/libclixon_backend.so ]; then
-    err $DIR/usr/local/lib/libclixon_backend.so
 fi
 
 new "Make DESTDIR install include"
-(cd ..; make DESTDIR=$DIR install-include)
+(cd ..; make DESTDIR=$dir install-include)
 if [ $? -ne 0 ]; then
     err
 fi
 new "Check installed includes"
-if [ ! -f $DIR/usr/local/include/clixon/clixon.h ]; then
+if [ ! -f $dir/usr/local/include/clixon/clixon.h ]; then
-    err $DIR/usr/local/include/clixon/clixon.h
+    err $dir/usr/local/include/clixon/clixon.h
 fi
 new "Make DESTDIR uninstall"
-(cd ..; make DESTDIR=$DIR uninstall)
+(cd ..; make DESTDIR=$dir uninstall)
 if [ $? -ne 0 ]; then
     err
 fi
 
 new "Check remaining files"
-f=$(find $DIR -type f)
+f=$(find $dir -type f)
 if [ -n "$f" ]; then
     err "$f"
 fi
 
 new "Check remaining symlinks"
-l=$(find $DIR -type l)
+l=$(find $dir -type l)
 if [ -n "$l" ]; then
     err "$l"
 fi

test/test_leafref.sh

@@ -1,15 +1,26 @@
 #!/bin/bash
 # Test7: Yang specifics: leafref
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/usr/local/etc/routing.xml
+cfg=$dir/conf_yang.xml
-fyang=/tmp/leafref.yang
+fyang=$dir/leafref.yang
 
-# For memcheck
+cat <<EOF > $cfg
-# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+<config>
-clixon_netconf=clixon_netconf
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
-clixon_cli=clixon_cli
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
 
 cat <<EOF > $fyang
 module example{
@@ -55,7 +66,7 @@ if [ $? -ne 0 ]; then
     err
 fi
 
-new "start backend"
+new "start backend  -s init -f $cfg -y $fyang"
 # start new backend
 sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
@@ -102,7 +113,7 @@ expecteof "$clixon_netconf -qf $cfg -y $fyang" "<rpc><validate><source><candidat
 
 new "leafref discard-changes"
 expecteof "$clixon_netconf -qf $cfg" "<rpc><discard-changes/></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
-exit
+
 new "cli leafref lo"
 expectfn "$clixon_cli -1f $cfg -y $fyang -l o set default-address absname lo" "^$"
 
@@ -126,3 +137,5 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi
+
+rm -rf $dir

test/test_netconf.sh

@@ -1,13 +1,29 @@
 #!/bin/bash
 # Test2: backend and netconf basic functionality
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/usr/local/etc/routing.xml
 
-# For memcheck
+cfg=$dir/conf_yang.xml
-#clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+
-clixon_netconf=clixon_netconf
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/routing/backend</CLICON_BACKEND_DIR>
+  <CLICON_NETCONF_DIR>/usr/local/lib/routing/netconf</CLICON_NETCONF_DIR>
+  <CLICON_RESTCONF_DIR>/usr/local/lib/routing/restconf</CLICON_RESTCONF_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
 
 echo "clixon_backend -zf $cfg"
 # kill old backend (if any)
@@ -16,7 +32,7 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err
 fi
-new "start backend"
+new "start backend  -s init -f $cfg"
 # start new backend
 sudo clixon_backend -s init -f $cfg
 if [ $? -ne 0 ]; then
@@ -146,3 +162,5 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi
+
+rm -rf $dir

test/test_order.sh

@@ -6,20 +6,21 @@
 # No test of ordered-by system is done yet
 # (we may want to sort them alphabetically for better performance).
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/tmp/conf_yang.xml
+cfg=$dir/conf_yang.xml
-fyang=/tmp/order.yang
+fyang=$dir/order.yang
 
 # For memcheck
 # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
-clixon_netconf=clixon_netconf
+
-clixon_cli=clixon_cli
+dbdir=$dir/order
-dbdir=/tmp/order
 
 new "Set up $dbdir"
 rm -rf $dbdir
+if [ ! -d $dbdir ]; then
     mkdir $dbdir
+fi
 
 cat <<EOF > $cfg
 <config>
@@ -175,3 +176,5 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi
+
+rm -rf $dir

test/test_perf.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Scaling test
 
-number=1000
+number=5000
 req=100
 if [ $# = 0 ]; then
     number=1000
@@ -14,18 +14,13 @@ else
     echo "Usage: $0 [<number> [<requests>]]"
     exit 1
 fi
-cfg=/tmp/scaling-conf.xml
-fyang=/tmp/scaling.yang
-fconfig=/tmp/config
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
 
-
+cfg=$dir/scaling-conf.xml
-# For memcheck
+fyang=$dir/scaling.yang
-# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+fconfig=$dir/config
-# clixon_netconf="valgrind --tool=callgrind clixon_netconf 
-clixon_netconf=clixon_netconf
 
 cat <<EOF > $fyang
 module ietf-ip{
@@ -53,12 +48,12 @@ cat <<EOF > $cfg
   <CLICON_YANG_MODULE_MAIN>ietf-ip</CLICON_YANG_MODULE_MAIN>
   <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+<CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
   <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
   <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
 </config>
 EOF
 
-
 # kill old backend (if any)
 new "kill old backend"
 sudo clixon_backend -zf $cfg -y $fyang
@@ -73,7 +68,15 @@ if [ $? -ne 0 ]; then
     err
 fi
 
-new "generate large list config"
+new "kill old restconf daemon"
+sudo pkill -u www-data clixon_restconf
+
+new "start restconf daemon"
+sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg -y $fyang 
+
+sleep 1
+
+new "generate 'large' config with $number list entries"
 echo -n "<rpc><edit-config><target><candidate/></target><config><x>" > $fconfig
 for (( i=0; i<$number; i++ )); do  
     echo -n "<y><a>$i</a><b>$i</b></y>" >> $fconfig
@@ -83,26 +86,38 @@ echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
 # Just for manual dbg
 echo "$clixon_netconf -qf $cfg  -y $fyang"
 
-new "netconf edit large config"
+new "netconf write large config"
-expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof_file "time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 #echo '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' | $clixon_netconf -qf $cfg  -y $fyang 
 
-new "netconf edit large config again"
+new "netconf write large config again"
-expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
+expecteof_file "time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$"
 
 #echo '<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>' | $clixon_netconf -qf $cfg  -y $fyang 
 
 rm $fconfig
 
 new "netconf commit large config"
-expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
-new "netconf commit same config again"
+new "netconf commit large config again"
-expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
-exit
+
-new "netconf add one small config"
+new "netconf add small (1 entry) config"
-expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><y><a>x</a><b>y</b></y></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><y><a>x</a><b>y</b></y></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+
+new "netconf get $req small config"
+time -p for (( i=0; i<$req; i++ )); do
+    rnd=$(( ( RANDOM % $number ) ))
+    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
+
+new "netconf get $req restconf small config"
+time -p for (( i=0; i<$req; i++ )); do
+    rnd=$(( ( RANDOM % $number ) ))
+#XXX    curl -sX PUT -d {"y":{"a":"$rnd","b":"$rnd"}} http://localhost/restconf/data/x/y=$rnd,$rnd 
+done 
 
 new "netconf add $req small config"
 time -p for (( i=0; i<$req; i++ )); do
@@ -110,14 +125,14 @@ time -p for (( i=0; i<$req; i++ )); do
     echo "<rpc><edit-config><target><candidate/></target><config><x><y><a>$rnd</a><b>$rnd</b></y></x></config></edit-config></rpc>]]>]]>"
 done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 
-new "netconf get large config"
+new "netconf add $req restconf small config"
-expecteof "time -p $clixon_netconf -qf $cfg  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>0</a><b>0</b></y><y><a>1</a><b>1</b>"
-
-new "netconf get $req small config"
 time -p for (( i=0; i<$req; i++ )); do
     rnd=$(( ( RANDOM % $number ) ))
-    echo "<rpc><get-config><source><candidate/></source><filter type=\"xpath\" select=\"/x/y[a=$rnd][b=$rnd]\" /></get-config></rpc>]]>]]>"
+    curl -sG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null
-done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
+done 
+
+new "netconf get large config"
+expecteof "time -f %e $clixon_netconf -qf $cfg  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><y><a>0</a><b>0</b></y><y><a>1</a><b>1</b>"
 
 new "generate large leaf-list config"
 echo -n "<rpc><edit-config><target><candidate/></target><default-operation>replace</default-operation><config><x>" > $fconfig
@@ -127,21 +142,27 @@ done
 echo "</x></config></edit-config></rpc>]]>]]>" >> $fconfig
 
 new "netconf replace large list-leaf config"
-expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof_file "time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 rm $fconfig
 
 new "netconf commit large leaf-list config"
-expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+
+new "netconf add $req small leaf-list config"
+time -p for (( i=0; i<$req; i++ )); do
+    rnd=$(( ( RANDOM % $number ) ))
+    echo "<rpc><edit-config><target><candidate/></target><config><x><c>$rnd</c></x></config></edit-config></rpc>]]>]]>"
+done | $clixon_netconf -qf $cfg  -y $fyang > /dev/null
 
 new "netconf add small leaf-list config"
-expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><c>x</c></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "<rpc><edit-config><target><candidate/></target><config><x><c>x</c></x></config></edit-config></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 new "netconf commit small leaf-list config"
-expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
+expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "<rpc><commit><source><candidate/></source></commit></rpc>]]>]]>" "^<rpc-reply><ok/></rpc-reply>]]>]]>$" 
 
 new "netconf get large leaf-list config"
-expecteof "time -p $clixon_netconf -qf $cfg  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><c>0</c><c>1</c>"
+expecteof "time -f %e $clixon_netconf -qf $cfg  -y $fyang" "<rpc><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply><data><x><c>0</c><c>1</c>"
 
 new "Kill backend"
 # Check if still alive
@@ -154,3 +175,5 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi
+
+rm -rf $dir

test/test_restconf.sh

@@ -2,12 +2,59 @@
 # Restconf basic functionality
 # Assume http server setup, such as nginx described in apps/restconf/README.md
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/usr/local/etc/routing.xml
+cfg=$dir/conf.xml
+fyang=$dir/restconf.yang
 
-# This is a fixed 'state' implemented in routing_backend. It is always there
+#  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
-state='{"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}'
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>$fyang</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_BACKEND_DIR>/usr/local/lib/routing/backend</CLICON_BACKEND_DIR>
+  <CLICON_RESTCONF_DIR>/usr/local/lib/routing/restconf</CLICON_RESTCONF_DIR>
+  <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
+
+cat <<EOF > $fyang
+module example{
+    prefix ex;
+    import ietf-ip {
+      prefix ip;
+    }
+    import ietf-routing {
+      prefix rt;
+    }
+    import ietf-inet-types {
+       prefix "inet";
+       revision-date "2013-07-15";
+    }
+    rpc empty {
+    }
+    rpc input {
+       input {
+       }
+    }
+    rpc output {
+       output {
+       }
+    }
+}
+EOF
+
+# This is a fixed 'state' implemented in routing_backend. It is assumed to be always there
+state='{"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": 42}\]}}'
 
 # kill old backend (if any)
 new "kill old backend"
@@ -15,8 +62,8 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err
 fi
-new "start backend"
+new "start backend -s init -f $cfg -y $fyang"
-sudo clixon_backend -s init -f $cfg
+sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
 fi
@@ -25,7 +72,7 @@ new "kill old restconf daemon"
 sudo pkill -u www-data clixon_restconf
 
 new "start restconf daemon"
-sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df /usr/local/etc/routing.xml # -D
+sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -f $cfg -D
 
 sleep 1
 
@@ -35,70 +82,117 @@ new "restconf options"
 expectfn "curl -i -sS -X OPTIONS http://localhost/restconf/data" "Allow: OPTIONS,HEAD,GET,POST,PUT,DELETE"
 
 new "restconf head"
-expectfn "curl -sS -I http://localhost/restconf/data" "HTTP/1.1 200 OK"
+expectfn "curl -s -I http://localhost/restconf/data" "HTTP/1.1 200 OK"
 #Content-Type: application/yang-data+json"
 
-new "restconf get empty config + state"
+new "restconf root discovery"
-expectfn "curl -sSG http://localhost/restconf/data" $state
+expectfn "curl  -s -X GET http://localhost/.well-known/host-meta" "<Link rel='restconf' href='/restconf'/>"
 
-new "restconf get state operation"
+new "restconf get restconf json"
-expectfn "curl -sS -G http://localhost/restconf/data/interfaces-state" $state
+expectfn "curl -sG http://localhost/restconf" '{"data": null,"operations": null,"yang-library-version": "2016-06-21"}}'
+
+new "restconf get restconf/yang-library-version json"
+expectfn "curl -sG http://localhost/restconf/yang-library-version" '{"yang-library-version": "2016-06-21"}'
+
+new "restconf empty rpc"
+expectfn 'curl -s -X POST -d {"input":{"name":""}} http://localhost/restconf/operations/ex:empty' '{"output": null}'
+
+new "restconf get empty config + state json"
+expectfn "curl -sSG http://localhost/restconf/data" "{\"data\": $state}"
+
+new "restconf get empty config + state xml"
+ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data)
+expect="<data><interfaces-state><interface><name>eth0</name><type>eth</type><if-index>42</if-index></interface></interfaces-state></data>"
+match=`echo $ret | grep -EZo "$expect"`
+if [ -z "$match" ]; then
+    err "$expect" "$ret"
+fi
+
+new "restconf get data/interfaces-state/interface=eth0 json"
+expectfn "curl -s -G http://localhost/restconf/data/interfaces-state/interface=eth0" '{"interface": \[{"name": "eth0","type": "eth","if-index": 42}\]}'
+
+new "restconf get state operation eth0 xml"
+# Cant get shell macros to work, inline matching from lib.sh
+ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0)
+expect="<interface><name>eth0</name><type>eth</type><if-index>42</if-index></interface>"
+match=`echo $ret | grep -EZo "$expect"`
+if [ -z "$match" ]; then
+    err "$expect" "$ret"
+fi
+
+new "restconf get state operation eth0 type json"
+expectfn "curl -s -G http://localhost/restconf/data/interfaces-state/interface=eth0/type" '{"type": "eth"}
+$'
+
+new "restconf get state operation eth0 type xml"
+# Cant get shell macros to work, inline matching from lib.sh
+ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0/type)
+expect="<type>eth</type>"
+match=`echo $ret | grep -EZo "$expect"`
+if [ -z "$match" ]; then
+    err "$expect" "$ret"
+fi
 
 new "restconf Add subtree  to datastore using POST"
-expectfn 'curl -sS -X POST -d {"interfaces":{"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}}} http://localhost/restconf/data' ""
+expectfn 'curl -s -X POST -d {"interfaces":{"interface":{"name":"eth/0/0","type":"eth","enabled":true}}} http://localhost/restconf/data' ""
 
 new "restconf Check interfaces eth/0/0 added"
-expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}
+expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","type": "eth","enabled": true}\]},"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": 42}\]}}
 $'
 
 new "restconf delete interfaces"
-expectfn 'curl -sS -X DELETE  http://localhost/restconf/data/interfaces' ""
+expectfn 'curl -s -X DELETE  http://localhost/restconf/data/interfaces' ""
 
 new "restconf Check empty config"
-expectfn "curl -sSG http://localhost/restconf/data" $state
+expectfn "curl -sG http://localhost/restconf/data" $state
 
 new "restconf Add interfaces subtree eth/0/0 using POST"
-expectfn 'curl -sS -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' ""
+expectfn 'curl -s -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":true}} http://localhost/restconf/data/interfaces' ""
 
 new "restconf Check eth/0/0 added"
-expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}
+expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","type": "eth","enabled": true}\]},"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": 42}\]}}
 $'
 
 new "restconf Re-post eth/0/0 which should generate error"
-expectfn 'curl -sS -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' "Data resource already exists"
+expectfn 'curl -s -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":true}} http://localhost/restconf/data/interfaces' "Data resource already exists"
 
 new "Add leaf description using POST"
-expectfn 'curl -sS -X POST -d {"description":"The-first-interface"} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' ""
+expectfn 'curl -s -X POST -d {"description":"The-first-interface"} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' ""
+
+new "Add nothing using POST"
+expectfn 'curl -s -X POST http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "data is in some way badly formed"
 
 new "restconf Check description added"
-expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": "true"}}
+expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": true}\]}
 $'
 
 new "restconf delete eth/0/0"
-expectfn 'curl -sS -X DELETE  http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' ""
+expectfn 'curl -s -X DELETE  http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' ""
 
 new "Check deleted eth/0/0"
-expectfn 'curl -sS -G http://localhost/restconf/data' $state
+expectfn 'curl -s -G http://localhost/restconf/data' $state
 
 new "restconf Re-Delete eth/0/0 using none should generate error"
-expectfn 'curl -sS -X DELETE  http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "Not Found"
+expectfn 'curl -s -X DELETE  http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "Not Found"
 
 new "restconf Add subtree eth/0/0 using PUT"
-expectfn 'curl -sS -X PUT -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' ""
+expectfn 'curl -s -X PUT -d {"interface":{"name":"eth/0/0","type":"eth","enabled":true}} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' ""
 
 new "restconf get subtree"
-expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}
+expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","type": "eth","enabled": true}\]},"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": 42}\]}}
 $'
 
-new "restconf operation rpc using POST json"
+new "restconf rpc using POST json"
-expectfn 'curl -sS -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{ "output": { "route": { "address-family": "ipv4", "next-hop": { "next-hop-list": "2.3.4.5" } } } } '
+expectfn 'curl -s -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{"output": {"route": {"address-family": "ipv4","next-hop": {"next-hop-list": "2.3.4.5"}}}}'
 
 new "restconf rpc using POST xml"
-ret=$(curl -sS -X POST -H "Accept: application/yang-data+xml" -d '{"input":{"routing-instance-name":"ipv4"}}' http://localhost/restconf/operations/rt:fib-route)
+# Cant get shell macros to work, inline matching from lib.sh
-expect="<output> <route> <address-family>ipv4</address-family> <next-hop> <next-hop-list>2.3.4.5</next-hop-li t> </next-hop> </route> </output> "
+ret=$(curl -s -X POST -H "Accept: application/yang-data+xml" -d '{"input":{"routing-instance-name":"ipv4"}}' http://localhost/restconf/operations/rt:fib-route)
+expect="<output><route><address-family>ipv4</address-family><next-hop><next-hop-list>2.3.4.5</next-hop-list></next-hop></route></output>"
 match=`echo $ret | grep -EZo "$expect"`
-echo -n "ret:   "
+if [ -z "$match" ]; then
-echo $ret
+    err "$expect" "$ret"
+fi
 
 new "Kill restconf daemon"
 sudo pkill -u www-data clixon_restconf
@@ -114,3 +208,5 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi
+
+rm -rf $dir

test/test_restconf2.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+# Restconf basic functionality
+# Assume http server setup, such as nginx described in apps/restconf/README.md
+
+# include err() and new() functions and creates $dir
+. ./lib.sh
+cfg=$dir/conf.xml
+fyang=$dir/restconf.yang
+
+#  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+cat <<EOF > $cfg
+<config>
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
+  <CLICON_YANG_DIR>/usr/local/var</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>$fyang</CLICON_YANG_MODULE_MAIN>
+
+  <CLICON_RESTCONF_PRETTY>false</CLICON_RESTCONF_PRETTY>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
+
+cat <<EOF > $fyang
+module example{
+   container interfaces-config{
+      list interface{
+        key name;
+        leaf name{
+          type string;
+        }
+        leaf type{
+          mandatory true;
+          type string;
+        }
+      }
+   }
+}
+EOF
+
+# kill old backend (if any)
+new "kill old backend"
+sudo clixon_backend -zf $cfg
+if [ $? -ne 0 ]; then
+    err
+fi
+new "start backend -s init -f $cfg -y $fyang"
+sudo clixon_backend -s init -f $cfg -y $fyang
+if [ $? -ne 0 ]; then
+    err
+fi
+
+new "kill old restconf daemon"
+sudo pkill -u www-data clixon_restconf
+
+new "start restconf daemon"
+sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg
+
+sleep 1
+
+new "restconf tests"
+
+new "restconf POST initial tree"
+expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' ""
+
+new "restconf GET datastore"
+expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": \[{"name": "local0","type": "regular"}\]}}}'
+
+new "restconf GET interface"
+expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0" '{"interface": \[{"name": "local0","type": "regular"}\]}'
+
+new "restconf GET if-type"
+expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0/type" '{"type": "regular"}'
+
+new "restconf POST interface"
+expectfn 'curl -s -X POST -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config' ""
+
+new "restconf POST again"
+expectfn 'curl -s -X POST -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config' "Data resource already exis"
+
+new "restconf POST from top"
+expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"TEST","type":"eth0"}}} http://localhost/restconf/data' "Data resource already exists"
+
+new "restconf DELETE"
+expectfn 'curl -s -X DELETE http://localhost/restconf/data/interfaces-config' ""
+
+new "restconf GET null datastore"
+expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": null}'
+
+new "restconf POST initial tree"
+expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' ""
+
+new "restconf PUT initial datastore" 
+expectfn 'curl -s -X PUT -d {"data":{"interfaces-config":{"interface":{"name":"local0","type":"regular"}}}} http://localhost/restconf/data' ""
+
+new "restconf GET datastore"
+expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": \[{"name": "local0","type": "regular"}\]}}}'
+
+new "restconf PUT change interface"
+expectfn 'curl -s -X PUT -d {"interface":{"name":"local0","type":"atm0"}} http://localhost/restconf/data/interfaces-config/interface=local0' ""
+
+new "restconf GET datastore atm"
+expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": \[{"name": "local0","type": "atm0"}\]}}}'
+
+new "restconf PUT add interface"
+expectfn 'curl -s -X PUT -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' ""
+
+new "restconf PUT change key error"
+expectfn 'curl -is -X PUT -d {"interface":{"name":"ALPHA","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "Bad request"
+
+new "restconf POST invalid no type"
+expectfn 'curl -s -X POST -d {"interface":{"name":"ALPHA"}} http://localhost/restconf/data/interfaces-config' "Bad request"
+
+new "Kill restconf daemon"
+sudo pkill -u www-data clixon_restconf
+
+new "Kill backend"
+# Check if still alive
+pid=`pgrep clixon_backend`
+if [ -z "$pid" ]; then
+    err "backend already dead"
+fi
+# kill backend
+sudo clixon_backend -zf $cfg
+if [ $? -ne 0 ]; then
+    err "kill backend"
+fi
+
+rm -rf $dir

test/test_startup.sh

@@ -6,14 +6,9 @@
 # - running db starts with a "run" interface
 # - startup db starts with a "start" interface
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/tmp/conf_startup.xml
+cfg=$dir/conf_startup.xml
-
-# For memcheck
-# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
-clixon_netconf=clixon_netconf
-clixon_cli=clixon_cli
 
 cat <<EOF > $cfg
 <config>
@@ -42,7 +37,8 @@ run(){
     mode=$1
     expect=$2
 
-    cat <<EOF > /tmp/db
+    dbdir=$dir/db
+    cat <<EOF > $dbdir
 <config>
    <interfaces>
       <interface>
@@ -52,9 +48,9 @@ run(){
     </interfaces>
 </config>
 EOF
-    sudo mv /tmp/db /usr/local/var/routing/running_db
+    sudo mv $dbdir /usr/local/var/routing/running_db
 
-    cat <<EOF > /tmp/db
+    cat <<EOF > $dbdir
 <config>
    <interfaces>
       <interface>
@@ -64,9 +60,9 @@ EOF
     </interfaces>
 </config>
 EOF
-    sudo mv /tmp/db /usr/local/var/routing/startup_db
+    sudo mv $dbdir /usr/local/var/routing/startup_db
 
-    cat <<EOF > /tmp/config
+    cat <<EOF > $dir/config
 <config>
    <interfaces>
       <interface>
@@ -84,9 +80,8 @@ EOF
 	err
     fi
 
-    new "start backend"
+    new "start backend  -f $cfg -s $mode -c $dir/config"
-    # start new backend
+    sudo clixon_backend -f $cfg -s $mode -c $dir/config
-    sudo clixon_backend -f $cfg -s $mode -c /tmp/config
     if [ $? -ne 0 ]; then
 	err
     fi
@@ -112,3 +107,4 @@ run none    '<data><interfaces><interface><name>run</name><type>eth</type><enabl
 run running '<data><interfaces><interface><name>extra</name><type>eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>local</type><enabled>true</enabled></interface><interface><name>run</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
 run startup '<data><interfaces><interface><name>extra</name><type>eth</type><enabled>true</enabled></interface><interface><name>lo</name><type>local</type><enabled>true</enabled></interface><interface><name>startup</name><type>eth</type><enabled>true</enabled></interface></interfaces></data>'
 
+rm -rf $dir

test/test_type.sh

@@ -2,15 +2,26 @@
 # Advanced union types and generated code
 # and enum w values
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/usr/local/etc/routing.xml
+fyang=$dir/type.yang
-fyang=/tmp/type.yang
+cfg=$dir/conf_yang.xml
 
-# For memcheck
+cat <<EOF > $cfg
-#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"
+<config>
-clixon_cli=clixon_cli
+  <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
-clixon_netconf=clixon_netconf
+  <CLICON_YANG_DIR>/usr/local/share/routing/yang</CLICON_YANG_DIR>
+  <CLICON_YANG_MODULE_MAIN>example</CLICON_YANG_MODULE_MAIN>
+  <CLICON_CLISPEC_DIR>/usr/local/lib/routing/clispec</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_DIR>/usr/local/lib/routing/cli</CLICON_CLI_DIR>
+  <CLICON_CLI_MODE>routing</CLICON_CLI_MODE>
+  <CLICON_SOCK>/usr/local/var/routing/routing.sock</CLICON_SOCK>
+  <CLICON_BACKEND_PIDFILE>/usr/local/var/routing/routing.pidfile</CLICON_BACKEND_PIDFILE>
+  <CLICON_CLI_GENMODEL_COMPLETION>1</CLICON_CLI_GENMODEL_COMPLETION>
+  <CLICON_XMLDB_DIR>/usr/local/var/routing</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_PLUGIN>/usr/local/lib/xmldb/text.so</CLICON_XMLDB_PLUGIN>
+</config>
+EOF
 
 cat <<EOF > $fyang
 module example{
@@ -69,8 +80,7 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err
 fi
-new "start backend"
+new "start backend -s init -f $cfg -y $fyang"
-# start new backend
 sudo clixon_backend -s init -f $cfg -y $fyang
 if [ $? -ne 0 ]; then
     err
@@ -127,3 +137,4 @@ if [ $? -ne 0 ]; then
     err "kill backend"
 fi
 
+rm -rf $dir

test/test_yang.sh

@@ -1,15 +1,11 @@
 #!/bin/bash
 # Test4: Yang specifics: multi-keys and empty type
 
-# include err() and new() functions
+# include err() and new() functions and creates $dir
 . ./lib.sh
-cfg=/tmp/conf_yang.xml
-fyang=/tmp/test.yang
 
-# For memcheck
+cfg=$dir/conf_yang.xml
-# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf"
+fyang=$dir/test.yang
-clixon_netconf=clixon_netconf
-clixon_cli=clixon_cli
 
 cat <<EOF > $cfg
 <config>
@@ -151,3 +147,5 @@ sudo clixon_backend -zf $cfg
 if [ $? -ne 0 ]; then
     err "kill backend"
 fi
+
+rm -rf $dir

yang/Makefile.in

@@ -38,8 +38,9 @@ bindir 		= @bindir@
 includedir	= @includedir@
 datarootdir     = @datarootdir@
 
-YANGSPECS	 = clixon-config@2017-12-27.yang
+YANGSPECS	 = clixon-config@2018-02-12.yang
 YANGSPECS	+= ietf-netconf@2011-06-01.yang
+YANGSPECS	+= ietf-netconf-acm@2012-02-22.yang
 YANGSPECS	+= ietf-inet-types@2013-07-15.yang
 
 APPNAME	        = clixon  # subdir ehere these files are installed

yang/clixon-config@2018-02-12.yang

@@ -0,0 +1,304 @@
+module clixon-config {
+
+    prefix cc;
+
+    organization
+	"Clicon / Clixon";
+
+    contact
+	"Olof Hagsand <olof@hagsand.se>";
+
+    description
+      "Clixon configuration file
+       ***** BEGIN LICENSE BLOCK *****
+       Copyright (C) 2009-2018 Olof Hagsand and Benny Holmgren
+       
+       This file is part of CLIXON
+
+       Licensed under the Apache License, Version 2.0 (the \"License\");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+            http://www.apache.org/licenses/LICENSE-2.0
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an \"AS IS\" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
+
+       Alternatively, the contents of this file may be used under the terms of
+       the GNU General Public License Version 3 or later (the \"GPL\"),
+       in which case the provisions of the GPL are applicable instead
+       of those above. If you wish to allow use of your version of this file only
+       under the terms of the GPL, and not to allow others to
+       use your version of this file under the terms of Apache License version 2, 
+       indicate your decision by deleting the provisions above and replace them with
+       the notice and other provisions required by the GPL. If you do not delete
+       the provisions above, a recipient may use your version of this file under
+       the terms of any one of the Apache License version 2 or the GPL.
+
+       ***** END LICENSE BLOCK *****";
+
+    revision 2018-02-12 {
+	description
+	    "Added pretty print for datastore";
+    }
+    typedef startup_mode{
+	description
+	    "Which method to boot/start clicon backend.
+             The methods differ in how they reach a running state
+             Which source database to commit from, if any.";
+	type enumeration{
+	    enum none{
+		description
+		"Do not touch running state
+                 Typically after crash when running state and db are synched";
+	    }
+	    enum init{
+		description
+		"Initialize running state.
+                 Start with a completely clean running state";
+	    }
+	    enum running{
+		description
+		"Commit running db configuration into running state
+                 After reboot if a persistent running db exists";
+	    }
+	    enum startup{
+		description
+		"Commit startup configuration into running state
+                 After reboot when no persistent running db exists";
+	    }
+	}
+    }
+    typedef xmldb_format{
+	description
+	    "Format of TEXT xml database format.";
+	type enumeration{
+	    enum xml{
+		description "Save and load xmldb as XML";
+	    }
+	    enum json{
+		description "Save and load xmldb as JSON";
+	    }
+	}
+    }
+    container config {
+	leaf CLICON_CONFIGFILE{
+	    type string;
+	    description
+		"Location of configuration-file for default values (this file)";
+	}
+	leaf CLICON_YANG_DIR {
+	    type string;
+	    mandatory true;
+	    description
+		"Location of YANG module and submodule files.";
+	}
+	leaf CLICON_YANG_MODULE_MAIN {
+	    type string;
+            default "clicon";
+	    description
+		"Option used to construct initial yang file: 
+                 <module>[@<revision>]";
+	}
+	leaf CLICON_YANG_MODULE_REVISION {
+	    type string;
+	    description
+		"Option used to construct initial yang file: 
+                 <module>[@<revision>]";
+	}
+	leaf CLICON_BACKEND_DIR {
+	    type string;
+	    description
+		"Location of backend .so plugins. Load all .so 
+       	         plugins in this dir as backend plugins";
+	}
+	leaf CLICON_NETCONF_DIR {
+	    type string;
+	    description "Location of netconf (frontend) .so plugins";
+	}
+	leaf CLICON_RESTCONF_DIR {
+	    type string;
+	    description
+		"Location of restconf (frontend) .so plugins. Load all .so
+       	         plugins in this dir as restconf code plugins";
+	}
+	leaf CLICON_RESTCONF_PATH {
+	    type string;
+	    default "/www-data/fastcgi_restconf.sock";
+	    description
+		"FastCGI unix socket. Should be specified in webserver
+         	 Eg in nginx: fastcgi_pass unix:/www-data/clicon_restconf.sock";
+	}
+	leaf CLICON_RESTCONF_PRETTY {
+	    type boolean;
+	    default true;
+	    description
+		"Restconf return value pretty print. 
+                 Restconf clients may add HTTP header:
+                      Accept: application/yang-data+json, or
+                      Accept: application/yang-data+xml
+                 to get return value in XML or JSON. 
+                 RFC 8040 examples print XML and JSON in pretty-printed form.
+                 Setting this value to false makes restconf return not pretty-printed
+                 which may be desirable for performance or tests";
+	}
+	leaf CLICON_CLI_DIR {
+	    type string;
+	    description
+		"Location of cli frontend .so plugins. Load all .so 
+                 plugins in this dir as CLI object plugins";
+	}
+	leaf CLICON_CLISPEC_DIR {
+	    type string;
+	    description
+		"Location of frontend .cli cligen spec files. Load all .cli 
+       	         files in this dir as CLI specification files";
+	}
+	leaf CLICON_CLISPEC_FILE {
+	    type string;
+	    description "Specific frontend .cli cligen spec file.";
+	}
+	leaf CLICON_CLI_MODE {
+	    type string;
+	    default "base";
+	    description
+		"Startup CLI mode. This should match a CLICON_MODE set in
+                 one of the clispec files";
+	}
+	leaf CLICON_CLI_GENMODEL {
+	    type int32;
+	    default 1;
+	    description
+		"Generate code for CLI completion of existing db symbols. 
+      	         Example: Add name=\"myspec\" in datamodel spec and reference
+	         as @myspec";
+	}
+	leaf CLICON_CLI_GENMODEL_COMPLETION {
+	    type int32;
+	    default 1;
+	    description "Generate code for CLI completion of existing db symbols";
+	}
+	leaf CLICON_CLI_GENMODEL_TYPE {
+	    type string;
+	    default "VARS";
+	    description "How to generate and show CLI syntax: VARS|ALL";
+	}
+	leaf CLICON_CLI_VARONLY {
+	    type int32;
+	    default 1;
+	    description
+		"Dont include keys in cvec in cli vars callbacks, 
+          	 ie a & k in 'a <b> k <c>' ignored";
+	}
+	leaf CLICON_CLI_LINESCROLLING {
+	    type int32;
+	    default 1;
+	    description
+		"Set to 0 if you want CLI to wrap to next line.
+                 Set to 1 if you  want CLI to scroll sideways when approaching 
+                      right margin";
+	}
+	leaf CLICON_SOCK_FAMILY {
+	    type string;
+	    default "UNIX";
+	    description
+		"Address family for communicating with clixon_backend 
+                 (UNIX|IPv4|IPv6)";
+	}
+	leaf CLICON_SOCK {
+	    type string;
+	    mandatory true;
+	    description
+		"If family above is AF_UNIX: Unix socket for communicating 
+       	         with clixon_backend. If family is AF_INET: IPv4 address";
+	}
+	leaf CLICON_SOCK_PORT {
+	    type int32;
+	    default 4535;
+	    description
+		"Inet socket port for communicating with clixon_backend 
+                 (only IPv4|IPv6)";
+	}
+	leaf CLICON_SOCK_GROUP {
+	    type string;
+	    default "clicon";
+	    description "Group membership to access clixon_backend unix socket";
+	}
+	leaf CLICON_BACKEND_PIDFILE {
+	    type string;
+	    mandatory true;
+	    description "Process-id file of backend daemon";
+	}
+	leaf CLICON_AUTOCOMMIT {
+	    type int32;
+	    default 0;
+	    description
+		"Set if all configuration changes are committed automatically 
+                 on every edit change. Explicit commit commands unnecessary";
+	}
+	leaf CLICON_MASTER_PLUGIN {
+	    type string;
+	    default "master";
+	    description
+		"Name of master plugin (both frontend and backend). 
+                 Master plugin has special callbacks for frontends. 
+	         See clicon user manual for more info. (Obsolete?)";
+	}
+	leaf CLICON_XMLDB_DIR {
+	    type string;
+	    mandatory true;
+	    description
+		"Directory where \"running\", \"candidate\" and \"startup\" are placed";
+	}
+	leaf CLICON_XMLDB_PLUGIN {
+	    type string;
+	    mandatory true;
+	    description
+		"XMLDB datastore plugin filename 
+	         (see datastore/ and clixon_xml_db.[ch])";
+	}
+	leaf CLICON_XMLDB_CACHE {
+	    type boolean;
+	    default true;
+	    description
+		"XMLDB datastore cache. 
+                 If set, XML candidate/running parsed tree is stored in memory
+                 If not set, candidate/running is always accessed via disk.";
+	}
+	leaf CLICON_XMLDB_FORMAT {
+	    type xmldb_format;
+	    default xml;
+	    description	"XMLDB datastore format.";
+	}
+	leaf CLICON_XMLDB_PRETTY {
+	    type boolean;
+	    default true;
+	    description
+		"XMLDB datastore pretty print. 
+                 If set, insert spaces and line-feeds making the XML/JSON human
+                 readable. If not set, make the XML/JSON more compact.";
+	}
+	leaf CLICON_XML_SORT {
+	    type boolean;
+	    default true;
+	    description
+		"If set, sort XML lists and leaf-lists alphabetically and uses binary
+                 search. Unless ordered-by user is used. 
+                 Only works for Yang specified XML.
+                 If not set, all lists accessed via linear search.";
+	}
+	leaf CLICON_USE_STARTUP_CONFIG {
+	    type int32;
+	    default 0;
+	    description
+		"Enabled uses \"startup\" configuration on boot. It is called
+                 startup_db and exists in XMLDB_DIR. 
+	         NOTE: Obsolete with 1.3.3 and CLICON_STARTUP_MODE";
+	}
+	leaf CLICON_STARTUP_MODE {
+	    type startup_mode;
+	    description "Which method to boot/start clicon backend";
+	}
+    }
+}

yang/ietf-netconf-acm@2012-02-22.yang

@@ -0,0 +1,445 @@
+module ietf-netconf-acm {
+    namespace "urn:ietf:params:xml:ns:yang:ietf-netconf-acm";
+    prefix "nacm";
+    import ietf-yang-types {
+	prefix yang;
+    }
+    organization
+	"IETF NETCONF (Network Configuration) Working Group";
+
+    contact
+	"WG Web:   <http://tools.ietf.org/wg/netconf/>
+        WG List:  <mailto:netconf@ietf.org>
+
+        WG Chair: Mehmet Ersue
+                  <mailto:mehmet.ersue@nsn.com>
+
+        WG Chair: Bert Wijnen
+                  <mailto:bertietf@bwijnen.net>
+
+        Editor:   Andy Bierman
+                  <mailto:andy@yumaworks.com>
+
+        Editor:   Martin Bjorklund
+                  <mailto:mbj@tail-f.com>";
+
+    description
+	"NETCONF Access Control Model.
+
+        Copyright (c) 2012 IETF Trust and the persons identified as
+        authors of the code.  All rights reserved.
+
+        Redistribution and use in source and binary forms, with or
+        without modification, is permitted pursuant to, and subject
+        to the license terms contained in, the Simplified BSD
+        License set forth in Section 4.c of the IETF Trust's
+        Legal Provisions Relating to IETF Documents
+        (http://trustee.ietf.org/license-info).
+
+        This version of this YANG module is part of RFC 6536; see
+        the RFC itself for full legal notices.";
+
+    revision "2012-02-22" {
+	description
+	    "Initial version";
+	reference
+	    "RFC 6536: Network Configuration Protocol (NETCONF)
+                    Access Control Model";
+    }
+
+    /*
+     * Extension statements
+     */
+
+    extension default-deny-write {
+	description
+	    "Used to indicate that the data model node
+          represents a sensitive security system parameter.
+
+          If present, and the NACM module is enabled (i.e.,
+          /nacm/enable-nacm object equals 'true'), the NETCONF server
+          will only allow the designated 'recovery session' to have
+          write access to the node.  An explicit access control rule is
+          required for all other users.
+
+          The 'default-deny-write' extension MAY appear within a data
+          definition statement.  It is ignored otherwise.";
+    }
+
+    extension default-deny-all {
+	description
+	    "Used to indicate that the data model node
+          controls a very sensitive security system parameter.
+
+          If present, and the NACM module is enabled (i.e.,
+          /nacm/enable-nacm object equals 'true'), the NETCONF server
+          will only allow the designated 'recovery session' to have
+          read, write, or execute access to the node.  An explicit
+          access control rule is required for all other users.
+
+          The 'default-deny-all' extension MAY appear within a data
+          definition statement, 'rpc' statement, or 'notification'
+          statement.  It is ignored otherwise.";
+    }
+
+    /*
+     * Derived types
+     */
+
+    typedef user-name-type {
+	type string {
+	    length "1..max";
+	}
+	description
+	    "General Purpose Username string.";
+    }
+
+    typedef matchall-string-type {
+	type string {
+	    pattern "\*";
+	}
+	description
+	    "The string containing a single asterisk '*' is used
+          to conceptually represent all possible values
+          for the particular leaf using this data type.";
+    }
+
+    typedef access-operations-type {
+	type bits {
+	    bit create {
+		description
+		    "Any protocol operation that creates a
+              new data node.";
+	    }
+	    bit read {
+		description
+		    "Any protocol operation or notification that
+              returns the value of a data node.";
+	    }
+	    bit update {
+		description
+		    "Any protocol operation that alters an existing
+              data node.";
+	    }
+	    bit delete {
+		description
+		    "Any protocol operation that removes a data node.";
+	    }
+	    bit exec {
+		description
+		    "Execution access to the specified protocol operation.";
+	    }
+	}
+	description
+	    "NETCONF Access Operation.";
+    }
+
+    typedef group-name-type {
+	type string {
+	    length "1..max";
+	    pattern "[^\*].*";
+	}
+	description
+	    "Name of administrative group to which
+          users can be assigned.";
+    }
+
+    typedef action-type {
+	type enumeration {
+	    enum permit {
+		description
+		"Requested action is permitted.";
+	    }
+	    enum deny {
+		description
+		"Requested action is denied.";
+	    }
+	}
+	description
+	    "Action taken by the server when a particular
+          rule matches.";
+    }
+
+    typedef node-instance-identifier {
+	type yang:xpath1.0;
+	description
+	    "Path expression used to represent a special
+          data node instance identifier string.
+
+          A node-instance-identifier value is an
+          unrestricted YANG instance-identifier expression.
+          All the same rules as an instance-identifier apply
+          except predicates for keys are optional.  If a key
+          predicate is missing, then the node-instance-identifier
+          represents all possible server instances for that key.
+
+          This XPath expression is evaluated in the following context:
+
+           o  The set of namespace declarations are those in scope on
+              the leaf element where this type is used.
+
+           o  The set of variable bindings contains one variable,
+              'USER', which contains the name of the user of the current
+               session.
+
+           o  The function library is the core function library, but
+              note that due to the syntax restrictions of an
+              instance-identifier, no functions are allowed.
+
+           o  The context node is the root node in the data tree.";
+    }
+
+    /*
+     * Data definition statements
+     */
+
+    container nacm {
+	/*	nacm:default-deny-all; XXX How is this parsed ?? */
+
+	description
+	    "Parameters for NETCONF Access Control Model.";
+
+	leaf enable-nacm {
+	    type boolean;
+	    default true;
+	    description
+		"Enables or disables all NETCONF access control
+            enforcement.  If 'true', then enforcement
+            is enabled.  If 'false', then enforcement
+            is disabled.";
+	}
+
+	leaf read-default {
+	    type action-type;
+	    default "permit";
+	    description
+		"Controls whether read access is granted if
+            no appropriate rule is found for a
+            particular read request.";
+	}
+
+	leaf write-default {
+	    type action-type;
+	    default "deny";
+	    description
+		"Controls whether create, update, or delete access
+            is granted if no appropriate rule is found for a
+            particular write request.";
+	}
+
+	leaf exec-default {
+	    type action-type;
+	    default "permit";
+	    description
+		"Controls whether exec access is granted if no appropriate
+            rule is found for a particular protocol operation request.";
+	}
+
+	leaf enable-external-groups {
+	    type boolean;
+	    default true;
+	    description
+		"Controls whether the server uses the groups reported by the
+            NETCONF transport layer when it assigns the user to a set of
+            NACM groups.  If this leaf has the value 'false', any group
+            names reported by the transport layer are ignored by the
+            server.";
+	}
+
+	leaf denied-operations {
+	    type yang:zero-based-counter32;
+	    config false;
+	    mandatory true;
+	    description
+		"Number of times since the server last restarted that a
+            protocol operation request was denied.";
+	}
+
+	leaf denied-data-writes {
+	    type yang:zero-based-counter32;
+	    config false;
+	    mandatory true;
+	    description
+		"Number of times since the server last restarted that a
+            protocol operation request to alter
+            a configuration datastore was denied.";
+	}
+
+	leaf denied-notifications {
+	    type yang:zero-based-counter32;
+	    config false;
+	    mandatory true;
+	    description
+		"Number of times since the server last restarted that
+            a notification was dropped for a subscription because
+            access to the event type was denied.";
+	}
+
+	container groups {
+	    description
+		"NETCONF Access Control Groups.";
+
+	    list group {
+		key name;
+
+		description
+		    "One NACM Group Entry.  This list will only contain
+              configured entries, not any entries learned from
+              any transport protocols.";
+
+		leaf name {
+		    type group-name-type;
+		    description
+			"Group name associated with this entry.";
+		}
+
+		leaf-list user-name {
+		    type user-name-type;
+		    description
+			"Each entry identifies the username of
+                a member of the group associated with
+                this entry.";
+		}
+	    }
+	}
+
+	list rule-list {
+	    key "name";
+	    ordered-by user;
+	    description
+		"An ordered collection of access control rules.";
+
+	    leaf name {
+		type string {
+		    length "1..max";
+		}
+		description
+		    "Arbitrary name assigned to the rule-list.";
+	    }
+	    leaf-list group {
+		type union {
+		    type matchall-string-type;
+		    type group-name-type;
+		}
+		description
+		    "List of administrative groups that will be
+              assigned the associated access rights
+              defined by the 'rule' list.
+
+              The string '*' indicates that all groups apply to the
+              entry.";
+	    }
+
+	    list rule {
+		key "name";
+		ordered-by user;
+		description
+		    "One access control rule.
+
+              Rules are processed in user-defined order until a match is
+              found.  A rule matches if 'module-name', 'rule-type', and
+              'access-operations' match the request.  If a rule
+              matches, the 'action' leaf determines if access is granted
+              or not.";
+
+		leaf name {
+		    type string {
+			length "1..max";
+		    }
+		    description
+			"Arbitrary name assigned to the rule.";
+		}
+
+		leaf module-name {
+		    type union {
+			type matchall-string-type;
+			type string;
+		    }
+		    default "*";
+		    description
+			"Name of the module associated with this rule.
+
+                This leaf matches if it has the value '*' or if the
+                object being accessed is defined in the module with the
+                specified module name.";
+		}
+		choice rule-type {
+		    description
+			"This choice matches if all leafs present in the rule
+                match the request.  If no leafs are present, the
+                choice matches all requests.";
+		    case protocol-operation {
+			leaf rpc-name {
+			    type union {
+				type matchall-string-type;
+				type string;
+			    }
+			    description
+				"This leaf matches if it has the value '*' or if
+                    its value equals the requested protocol operation
+                    name.";
+			}
+		    }
+		    case notification {
+			leaf notification-name {
+			    type union {
+				type matchall-string-type;
+				type string;
+			    }
+			    description
+				"This leaf matches if it has the value '*' or if its
+                    value equals the requested notification name.";
+			}
+		    }
+		    case data-node {
+			leaf path {
+			    type node-instance-identifier;
+			    mandatory true;
+			    description
+				"Data Node Instance Identifier associated with the
+                    data node controlled by this rule.
+
+                    Configuration data or state data instance
+                    identifiers start with a top-level data node.  A
+                    complete instance identifier is required for this
+                    type of path value.
+
+                    The special value '/' refers to all possible
+                    datastore contents.";
+			}
+		    }
+		}
+
+		leaf access-operations {
+		    type union {
+			type matchall-string-type;
+			type access-operations-type;
+		    }
+		    default "*";
+		    description
+			"Access operations associated with this rule.
+
+                This leaf matches if it has the value '*' or if the
+                bit corresponding to the requested operation is set.";
+		}
+
+		leaf action {
+		    type action-type;
+		    mandatory true;
+		    description
+			"The access control action associated with the
+                rule.  If a rule is determined to match a
+                particular request, then this object is used
+                to determine whether to permit or deny the
+                request.";
+		}
+
+		leaf comment {
+		    type string;
+		    description
+			"A textual description of the access rule.";
+		}
+	    }
+	}
+    }
+}

yang/ietf-yang-library@2016-06-21.yang

@@ -0,0 +1,242 @@
+module ietf-yang-library {
+    namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library";
+    prefix "yanglib";
+
+    import ietf-yang-types {
+	prefix yang;
+    }
+    import ietf-inet-types {
+	prefix inet;
+    }
+    organization
+	"IETF NETCONF (Network Configuration) Working Group";
+
+    contact
+	"WG Web:   <https://datatracker.ietf.org/wg/netconf/>
+        WG List:  <mailto:netconf@ietf.org>
+
+        WG Chair: Mehmet Ersue
+                  <mailto:mehmet.ersue@nsn.com>
+
+        WG Chair: Mahesh Jethanandani
+                  <mailto:mjethanandani@gmail.com>
+
+        Editor:   Andy Bierman
+                  <mailto:andy@yumaworks.com>
+
+        Editor:   Martin Bjorklund
+                  <mailto:mbj@tail-f.com>
+
+        Editor:   Kent Watsen
+                  <mailto:kwatsen@juniper.net>";
+
+    description
+	"This module contains monitoring information about the YANG
+        modules and submodules that are used within a YANG-based
+        server.
+
+        Copyright (c) 2016 IETF Trust and the persons identified as
+        authors of the code.  All rights reserved.
+
+        Redistribution and use in source and binary forms, with or
+        without modification, is permitted pursuant to, and subject
+        to the license terms contained in, the Simplified BSD License
+        set forth in Section 4.c of the IETF Trust's Legal Provisions
+        Relating to IETF Documents
+        (http://trustee.ietf.org/license-info).
+
+        This version of this YANG module is part of RFC 7895; see
+        the RFC itself for full legal notices.";
+
+    revision 2016-06-21 {
+	description
+	    "Initial revision.";
+	reference
+	    "RFC 7895: YANG Module Library.";
+    }
+
+    /*
+     * Typedefs
+     */
+
+    typedef revision-identifier {
+	type string {
+	    pattern '\d{4}-\d{2}-\d{2}';
+	}
+	description
+	    "Represents a specific date in YYYY-MM-DD format.";
+    }
+
+    /*
+     * Groupings
+     */
+
+    grouping module-list {
+	description
+	    "The module data structure is represented as a grouping
+          so it can be reused in configuration or another monitoring
+          data structure.";
+
+	grouping common-leafs {
+	    description
+		"Common parameters for YANG modules and submodules.";
+
+	    leaf name {
+		type yang:yang-identifier;
+		description
+		    "The YANG module or submodule name.";
+	    }
+	    leaf revision {
+		type union {
+		    type revision-identifier;
+		    type string { length 0; }
+		}
+		description
+		    "The YANG module or submodule revision date.
+              A zero-length string is used if no revision statement
+              is present in the YANG module or submodule.";
+	    }
+	}
+
+	grouping schema-leaf {
+	    description
+		"Common schema leaf parameter for modules and submodules.";
+	    leaf schema {
+		type inet:uri;
+		description
+		    "Contains a URL that represents the YANG schema
+              resource for this module or submodule.
+
+              This leaf will only be present if there is a URL
+              available for retrieval of the schema for this entry.";
+	    }
+	}
+
+	list module {
+	    key "name revision";
+	    description
+		"Each entry represents one revision of one module
+            currently supported by the server.";
+
+	    uses common-leafs;
+	    uses schema-leaf;
+
+	    leaf namespace {
+		type inet:uri;
+		mandatory true;
+		description
+		    "The XML namespace identifier for this module.";
+	    }
+	    leaf-list feature {
+		type yang:yang-identifier;
+		description
+		    "List of YANG feature names from this module that are
+              supported by the server, regardless of whether they are
+              defined in the module or any included submodule.";
+	    }
+	    list deviation {
+		key "name revision";
+		description
+		    "List of YANG deviation module names and revisions
+              used by this server to modify the conformance of
+              the module associated with this entry.  Note that
+              the same module can be used for deviations for
+              multiple modules, so the same entry MAY appear
+              within multiple 'module' entries.
+
+              The deviation module MUST be present in the 'module'
+              list, with the same name and revision values.
+              The 'conformance-type' value will be 'implement' for
+              the deviation module.";
+		uses common-leafs;
+	    }
+	    leaf conformance-type {
+		type enumeration {
+		    enum implement {
+			description
+			"Indicates that the server implements one or more
+                  protocol-accessible objects defined in the YANG module
+                  identified in this entry.  This includes deviation
+                  statements defined in the module.
+
+                  For YANG version 1.1 modules, there is at most one
+                  module entry with conformance type 'implement' for a
+                  particular module name, since YANG 1.1 requires that,
+                  at most, one revision of a module is implemented.
+
+                  For YANG version 1 modules, there SHOULD NOT be more
+                  than one module entry for a particular module name.";
+		    }
+		    enum import {
+			description
+			    "Indicates that the server imports reusable definitions
+                  from the specified revision of the module but does
+                  not implement any protocol-accessible objects from
+                  this revision.
+
+                  Multiple module entries for the same module name MAY
+                  exist.  This can occur if multiple modules import the
+                  same module but specify different revision dates in
+                  the import statements.";
+		    }
+		}
+		mandatory true;
+		description
+		    "Indicates the type of conformance the server is claiming
+              for the YANG module identified by this entry.";
+	    }
+	    list submodule {
+		key "name revision";
+		description
+		    "Each entry represents one submodule within the
+              parent module.";
+		uses common-leafs;
+		uses schema-leaf;
+	    }
+	}
+    }
+
+
+
+    /*
+     * Operational state data nodes
+     */
+
+    container modules-state {
+	config false;
+	description
+	    "Contains YANG module monitoring information.";
+
+	leaf module-set-id {
+	    type string;
+	    mandatory true;
+	    description
+		"Contains a server-specific identifier representing
+            the current set of modules and submodules.  The
+            server MUST change the value of this leaf if the
+            information represented by the 'module' list instances
+            has changed.";
+	}
+
+	uses module-list;
+    }
+
+    /*
+     * Notifications
+     */
+    notification yang-library-change {
+	description
+	    "Generated when the set of modules and submodules supported
+          by the server has changed.";
+	leaf module-set-id {
+	    type leafref {
+		path "/yanglib:modules-state/yanglib:module-set-id";
+	    }
+	    mandatory true;
+	    description
+		"Contains the module-set-id value representing the
+            set of modules and submodules supported at the server at
+            the time the notification is generated.";
+	}
+    }
+}