From 75848a8e30b21e927bf413a4b9f8c6f0c02b57e5 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 1 Jan 2018 12:34:45 +0100 Subject: [PATCH 01/28] 3.5.0.PRE --- CHANGELOG.md | 7 +++++++ configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2f19a40..5f877d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Clixon Changelog +## 3.5.0 (Upcoming) + +### Major changes: +### Minor changes: +### Corrected Bugs +### Known issues + ## 3.4.0 (1 January 2018) ### Major changes: diff --git a/configure b/configure index 3d63adb5..86fc64c3 100755 --- a/configure +++ b/configure @@ -2157,9 +2157,9 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu : ${CFLAGS="-O2"} CLIXON_VERSION_MAJOR="3" -CLIXON_VERSION_MINOR="4" +CLIXON_VERSION_MINOR="5" CLIXON_VERSION_PATCH="0" -CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\"" +CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\"" # Fix to specific version (eg 3.5) or head (3) CLIGEN_VERSION="3" if test "$prefix" = "NONE"; then diff --git a/configure.ac b/configure.ac index 294d269e..9855666a 100644 --- a/configure.ac +++ b/configure.ac @@ -42,9 +42,9 @@ AC_INIT(lib/clixon/clixon.h.in) : ${CFLAGS="-O2"} CLIXON_VERSION_MAJOR="3" -CLIXON_VERSION_MINOR="4" +CLIXON_VERSION_MINOR="5" CLIXON_VERSION_PATCH="0" -CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\"" +CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\"" # Fix to specific version (eg 3.5) or head (3) CLIGEN_VERSION="3" if test "$prefix" = "NONE"; then From 3758c8dab49bc095d623685dedceea9a7e4ca337 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 1 Jan 2018 14:48:57 +0100 Subject: [PATCH 02/28] * The following backward compatible options to configure have been obsoleted. If you havent already migrated this code you must do this now. * Backend startup modes prior to 3.3.3. As enabled with `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted. * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` --- CHANGELOG.md | 6 ++ Makefile.in | 13 +-- apps/backend/backend_main.c | 197 +----------------------------------- clixon.mk.cpp | 16 +-- configure | 71 ------------- configure.ac | 38 ------- include/clixon_config.h.in | 9 -- lib/clixon/clixon.h.in | 2 - lib/clixon/clixon_xml.h | 12 --- lib/src/clixon_options.c | 173 ------------------------------- test/test_install.sh | 3 - 11 files changed, 11 insertions(+), 529 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f877d2b..bcb394ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ### Major changes: ### Minor changes: + +* The following backward compatible options to configure have been obsoleted. If you havent already migrated this code you must do this now. + * Backend startup modes prior to 3.3.3. As enabled with `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted. + * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. + * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` + ### Corrected Bugs ### Known issues diff --git a/Makefile.in b/Makefile.in index b9cdb616..3041750f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -56,7 +56,7 @@ SUBDIRS = lib apps include etc datastore yang .PHONY: doc all clean depend $(SUBDIRS) install loc TAGS .config.status docker -all: $(SUBDIRS) clixon.conf.cpp clixon.mk +all: $(SUBDIRS) clixon.mk $(SUBDIRS): (cd $@ && $(MAKE) $(MFLAGS) all) @@ -65,16 +65,11 @@ depend: for i in $(SUBDIRS) doc example docker; \ do (cd $$i && $(MAKE) $(MFLAGS) depend); done -# template clixon.conf file -clixon.conf.cpp: clixon.conf.cpp.cpp - $(CPP) -P -x assembler-with-cpp -Dprefix=$(prefix) -Dlocalstatedir=$(localstatedir) -Dsysconfdir=$(sysconfdir) -Ddatadir=$(datadir) -Dlibdir=$(libdir) $< > $@ - clixon.mk: clixon.mk.cpp $(CPP) -P -traditional-cpp -x assembler-with-cpp -Dprefix=$(prefix) -Dlocalstatedir=$(localstatedir) -Dsysconfdir=$(sysconfdir) -Ddatadir=$(datadir) -Dlibdir=$(libdir) $< > $@ -install: clixon.conf.cpp clixon.mk +install: clixon.mk install -d -m 755 $(DESTDIR)$(datadir)/clixon - install -m 755 clixon.conf.cpp $(DESTDIR)$(datadir)/clixon install -m 755 clixon.mk $(DESTDIR)$(datadir)/clixon for i in $(SUBDIRS) doc; \ do (cd $$i; $(MAKE) $(MFLAGS) $@)||exit 1; done; \ @@ -88,7 +83,6 @@ install-include: uninstall: for i in $(SUBDIRS) doc example docker; \ do (cd $$i && $(MAKE) $(MFLAGS) $@)||exit 1; done; - rm -f $(DESTDIR)$(datadir)/clixon/clixon.conf.cpp rm -f $(DESTDIR)$(datadir)/clixon/clixon.mk doc: @@ -106,8 +100,7 @@ clean: distclean: rm -f Makefile TAGS config.status config.log *~ .depend - rm -rf autom4te.cache - rm -rf clixon.conf.cpp clixon.mk build-root/rpmbuild + rm -rf autom4te.cache clixon.mk build-root/rpmbuild rm -f build-root/*.tar.xz build-root/*.rpm extras/rpm/Makefile for i in $(SUBDIRS) doc example docker; \ do (cd $$i && $(MAKE) $(MFLAGS) $@); done diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index 1b52b629..31cb39c2 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -73,11 +73,7 @@ #include "backend_handle.h" /* Command line options to be passed to getopt(3) */ -#ifdef BACKEND_STARTUP_COMPAT -#define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:IRCrg:y:x:" /* substitute s: for IRCc:r */ -#else #define BACKEND_OPTS "hD:f:d:b:Fzu:P:1s:c:g:y:x:" /* substitute s: for IRCc:r */ -#endif /*! Terminate. Cannot use h after this */ static int @@ -142,12 +138,6 @@ usage(char *argv0, clicon_handle h) " -P \tPid filename (default: %s)\n" " -s \tSpecify backend startup mode: none|startup|running|init (replaces -IRCr\n" " -c \tLoad extra xml configuration, but don't commit.\n" -#ifdef BACKEND_STARTUP_COMPAT - " -I\t\tInitialize running state database\n" - " -R\t\tCall plugin_reset() in plugins to reset system state in running db (use with -I)\n" - " -C\t\tCall plugin_reset() in plugins to reset system state in candidate db (use with -I)\n" - " -r\t\tReload running database\n" -#endif /* BACKEND_STARTUP_COMPAT */ " -g \tClient membership required to this group (default: %s)\n" " -y \tOverride yang spec file (dont include .yang suffix)\n" " -x \tXMLDB plugin\n", @@ -270,161 +260,6 @@ plugin_start_useroptions(clicon_handle h, return 0; } -#ifdef BACKEND_STARTUP_COMPAT -/*! Initialize running-config from file application configuration - * - * @param[in] h clicon handle - * @param[in] extraxml_file clicon application configuration file - * @param[in] running_db Name of running db - * @retval 0 OK - * @retval -1 Error. clicon_err set - */ -static int -rundb_main(clicon_handle h, - char *extraxml_file) -{ - int retval = -1; - int fd = -1; - cxobj *xt = NULL; - cxobj *xn; - - if (xmldb_create(h, "tmp") < 0) - goto done; - if (xmldb_copy(h, "running", "tmp") < 0) - goto done; - if ((fd = open(extraxml_file, O_RDONLY)) < 0){ - clicon_err(OE_UNIX, errno, "open(%s)", extraxml_file); - goto done; - } - if (xml_parse_file(fd, &xt, "") < 0) - goto done; - if ((xn = xml_child_i(xt, 0)) != NULL) - if (xmldb_put(h, "tmp", OP_MERGE, xn) < 0) - goto done; - if (candidate_commit(h, "tmp") < 0) - goto done; - if (xmldb_delete(h, "tmp") < 0) - goto done; - retval = 0; -done: - if (xt) - xml_free(xt); - if (fd != -1) - close(fd); - return retval; -} - -static int -candb_reset(clicon_handle h) -{ - int retval = -1; - - if (xmldb_copy(h, "running", "tmp") < 0) - goto done; - /* Request plugins to reset system state, eg initiate running from system - * -R - */ - if (plugin_reset_state(h, "tmp") < 0) - goto done; - if (candidate_commit(h, "tmp") < 0) - goto done; - retval = 0; - done: - return retval; -} - -/*! Legacy (old-style) startup mode where flags -IRCcr was used - */ -static int -fragmented_startup_mode(clicon_handle h, - char *argv0, - int argc, - char *argv[], - int reload_running, - int init_rundb, - int reset_state_candidate, - int reset_state_running, - char *extraxml_file) -{ - int retval = -1; - - /* First check for startup config */ - if (clicon_option_int(h, "CLICON_USE_STARTUP_CONFIG") > 0){ - if (xmldb_exists(h, "startup") == 1){ - /* copy startup config -> running */ - if (xmldb_copy(h, "startup", "running") < 0) - goto done; - } - else - if (db_reset(h, "running") < 0) - goto done; - if (xmldb_create(h, "candidate") < 0) - goto done; - if (xmldb_copy(h, "running", "candidate") < 0) - goto done; - } - /* If running exists and reload_running set, make a copy to candidate */ - if (reload_running){ - if (xmldb_exists(h, "running") != 1){ - clicon_log(LOG_NOTICE, "%s: -r (reload running) option given but no running_db found, proceeding without", __PROGRAM__); - reload_running = 0; /* void it, so we dont commit candidate below */ - } - else - if (xmldb_copy(h, "running", "candidate") < 0) - goto done; - } - /* Init running db - * -I or if it isnt there - */ - if (init_rundb || xmldb_exists(h, "running") != 1){ - if (db_reset(h, "running") < 0) - goto done; - } - /* If candidate does not exist, create it from running */ - if (xmldb_exists(h, "candidate") != 1){ - if (xmldb_create(h, "candidate") < 0) - goto done; - if (xmldb_copy(h, "running", "candidate") < 0) - goto done; - } - - /* Load plugins and call plugin_init() */ - if (plugin_initiate(h) != 0) - goto done; - - if (reset_state_candidate){ - if (candb_reset(h) < 0) - goto done; - } - else - if (reset_state_running){ - if (plugin_reset_state(h, "running") < 0) - goto done; - } - - if (plugin_start_useroptions(h, argv0, argc, argv) <0) - goto done; - - if (reload_running){ - /* This could be a failed validation, and we should not fail for that */ - (void)candidate_commit(h, "candidate"); - } - - /* Have we specified a config file to load? eg - * -c [] - */ - if (extraxml_file) - if (rundb_main(h, extraxml_file) < 0) - goto done; - /* Initiate the shared candidate. */ - if (xmldb_copy(h, "running", "candidate") < 0) - goto done; - retval = 0; - done: - return retval; -} -#endif /* BACKEND_STARTUP_COMPAT */ - /*! Merge xml in filename into database */ static int @@ -648,12 +483,6 @@ main(int argc, char **argv) int foreground; int once; enum startup_mode_t startup_mode; -#ifdef BACKEND_STARTUP_COMPAT - int init_rundb = 0; - int reset_state_running = 0; - int reset_state_candidate = 0; - int reload_running = 0; -#endif char *extraxml_file; char *config_group; char *argv0 = argv[0]; @@ -769,20 +598,6 @@ main(int argc, char **argv) case 'c': /* Load application config */ extraxml_file = optarg; break; -#ifdef BACKEND_STARTUP_COMPAT - case 'I': /* Initiate running db */ - init_rundb++; - break; - case 'R': /* Reset state directly into running */ - reset_state_running++; - break; - case 'C': /* Reset state into candidate and then commit it */ - reset_state_candidate++; - break; - case 'r': /* Reload running */ - reload_running++; - break; -#endif /* BACKEND_STARTUP_COMPAT */ case 'g': /* config socket group */ clicon_option_str_set(h, "CLICON_SOCK_GROUP", optarg); break; @@ -890,19 +705,9 @@ main(int argc, char **argv) goto done; /* If startup mode is not defined, eg via OPTION or -s, assume old method */ startup_mode = clicon_startup_mode(h); - if (startup_mode == -1){ /* Old style, fragmented mode, phase out */ -#ifdef BACKEND_STARTUP_COMPAT - if (fragmented_startup_mode(h, - argv0, argc, argv, - reload_running, init_rundb, - reset_state_candidate, reset_state_running, - extraxml_file - ) < 0) - goto done; -#else + if (startup_mode == -1){ clicon_log(LOG_ERR, "Startup mode undefined. Specify option CLICON_STARTUP_MODE or specify -s option to clicon_backend.\n"); goto done; -#endif } else { /* Init running db if it is not there diff --git a/clixon.mk.cpp b/clixon.mk.cpp index 966fae05..2f27c669 100644 --- a/clixon.mk.cpp +++ b/clixon.mk.cpp @@ -42,19 +42,5 @@ clixon_DBSPECDIR=prefix/share/$(APPNAME) clixon_SYSCONFDIR=sysconfdir clixon_LOCALSTATEDIR=localstatedir/$(APPNAME) clixon_LIBDIR=libdir/$(APPNAME) -clixon_DATADIR=datadir/clixon +clixon_DATADIR=datadir/clixon # for system yang files -# Rules for the clixon application configuration file. -# The clixon applications should be started with this fileas its -f argument. -# Typically installed in sysconfdir -# Example: APPNAME=myapp --> clixon_cli -f /usr/local/etc/myapp.conf -# The two variants are if there is a .conf.local file or not -.PHONY: $(APPNAME).conf -ifneq (,$(wildcard ${APPNAME}.conf.local)) -${APPNAME}.conf: ${clixon_DATADIR}/clixon.conf.cpp ${APPNAME}.conf.local - $(CPP) -P -x assembler-with-cpp -DAPPNAME=$(APPNAME) $< > $@ - cat ${APPNAME}.conf.local >> $@ -else -${APPNAME}.conf: ${clixon_DATADIR}/clixon.conf.cpp - $(CPP) -P -x assembler-with-cpp -DAPPNAME=$(APPNAME) $< > $@ -endif diff --git a/configure b/configure index 86fc64c3..80d4af73 100755 --- a/configure +++ b/configure @@ -632,9 +632,6 @@ CPP OBJEXT EXEEXT ac_ct_CC -with_xml_compat -with_config_compat -with_startup_compat with_keyvalue with_restconf RANLIB @@ -712,9 +709,6 @@ with_cligen with_restconf with_keyvalue with_qdbm -with_startup_compat -with_config_compat -with_xml_compat ' ac_precious_vars='build_alias host_alias @@ -1353,9 +1347,6 @@ Optional Packages: --without-restconf disable support for restconf --with-keyvalue enable support for key-value xmldb datastore --with-qdbm=dir Use QDBM here, if keyvalue - --with-startup-compat Backward compatibility of backend startup commands - --with-config-compat Backward compatibility of configuration file - --with-xml-compat Backward compatibility of XML API Some influential environment variables: CC C compiler command @@ -2357,13 +2348,6 @@ test -n "$target_alias" && # If yes, compile apps/restconf # If yes, compile datastore/keyvalue -# If yes, backward compatible with 3.3.2 backend startup - -# If yes, backward compatible with 3.3.2 .conf configuration - -# If yes, backward compatible with 3.3.3 XML api new and parse functions - - # ac_ext=c ac_cpp='$CPP $CPPFLAGS' @@ -4061,61 +4045,6 @@ fi fi -# This is for backward compatibility of backend startup commands in 3.3.3 -# Will be removed in 3.4.0 - -# Check whether --with-startup_compat was given. -if test "${with_startup_compat+set}" = set; then : - withval=$with_startup_compat; -else - with_startup_compat=no -fi - -if test "x${with_startup_compat}" == xyes; then - -cat >>confdefs.h <<_ACEOF -#define BACKEND_STARTUP_COMPAT $with_startup_compat -_ACEOF - -fi - -# This is for backward compatibility of .conf configuration file in 3.3.3 -# Will be removed in 3.4.0 - -# Check whether --with-config_compat was given. -if test "${with_config_compat+set}" = set; then : - withval=$with_config_compat; -else - with_config_compat=no -fi - -if test "x${with_config_compat}" == xyes; then - -cat >>confdefs.h <<_ACEOF -#define CONFIG_COMPAT $with_config_compat -_ACEOF - -fi - -# Clixon 3.4.0 changes XML creation and parse API -# Set this for backward compat and migration. -# Will be removed in 3.5.0 - -# Check whether --with-xml_compat was given. -if test "${with_xml_compat+set}" = set; then : - withval=$with_xml_compat; -else - with_xml_compat=no -fi - -if test "x${with_xml_compat}" == xyes; then - -cat >>confdefs.h <<_ACEOF -#define XML_COMPAT $with_xml_compat -_ACEOF - -fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for crypt in -lcrypt" >&5 $as_echo_n "checking for crypt in -lcrypt... " >&6; } if ${ac_cv_lib_crypt_crypt+:} false; then : diff --git a/configure.ac b/configure.ac index 9855666a..54bc1c4b 100644 --- a/configure.ac +++ b/configure.ac @@ -86,13 +86,6 @@ AC_SUBST(AR) AC_SUBST(RANLIB) AC_SUBST(with_restconf) # If yes, compile apps/restconf AC_SUBST(with_keyvalue) # If yes, compile datastore/keyvalue -# If yes, backward compatible with 3.3.2 backend startup -AC_SUBST(with_startup_compat) -# If yes, backward compatible with 3.3.2 .conf configuration -AC_SUBST(with_config_compat) -# If yes, backward compatible with 3.3.3 XML api new and parse functions -AC_SUBST(with_xml_compat) - # AC_PROG_CC() AC_PROG_CPP @@ -168,37 +161,6 @@ if test "x${with_keyvalue}" == xyes; then AC_CONFIG_FILES(datastore/keyvalue/Makefile) fi -# This is for backward compatibility of backend startup commands in 3.3.3 -# Will be removed in 3.4.0 -AC_ARG_WITH([startup_compat], - [AS_HELP_STRING([--with-startup-compat],[Backward compatibility of backend startup commands])], - [], - [with_startup_compat=no]) -if test "x${with_startup_compat}" == xyes; then - AC_DEFINE_UNQUOTED(BACKEND_STARTUP_COMPAT, $with_startup_compat, [Backward compatible backend startup command-line options]) -fi - -# This is for backward compatibility of .conf configuration file in 3.3.3 -# Will be removed in 3.4.0 -AC_ARG_WITH([config_compat], - [AS_HELP_STRING([--with-config-compat],[Backward compatibility of configuration file])], - [], - [with_config_compat=no]) -if test "x${with_config_compat}" == xyes; then - AC_DEFINE_UNQUOTED(CONFIG_COMPAT, $with_config_compat, [Backward compatible of .conf configuration files]) -fi - -# Clixon 3.4.0 changes XML creation and parse API -# Set this for backward compat and migration. -# Will be removed in 3.5.0 -AC_ARG_WITH([xml_compat], - [AS_HELP_STRING([--with-xml-compat],[Backward compatibility of XML API])], - [], - [with_xml_compat=no]) -if test "x${with_xml_compat}" == xyes; then - AC_DEFINE_UNQUOTED(XML_COMPAT, $with_xml_compat, [Backward compatible of XML API]) -fi - AC_CHECK_LIB(crypt, crypt) AC_CHECK_HEADERS(crypt.h) diff --git a/include/clixon_config.h.in b/include/clixon_config.h.in index 2c710c39..8d7f194e 100644 --- a/include/clixon_config.h.in +++ b/include/clixon_config.h.in @@ -1,8 +1,5 @@ /* include/clixon_config.h.in. Generated from configure.ac by autoheader. */ -/* Backward compatible backend startup command-line options */ -#undef BACKEND_STARTUP_COMPAT - /* Clixon data dir for system yang files etc */ #undef CLIXON_DATADIR @@ -18,9 +15,6 @@ /* Clixon version string */ #undef CLIXON_VERSION_STRING -/* Backward compatible of .conf configuration files */ -#undef CONFIG_COMPAT - /* Define to 1 if you have the `alphasort' function. */ #undef HAVE_ALPHASORT @@ -138,9 +132,6 @@ /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS -/* Backward compatible of XML API */ -#undef XML_COMPAT - /* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a `char[]'. */ #undef YYTEXT_POINTER diff --git a/lib/clixon/clixon.h.in b/lib/clixon/clixon.h.in index 5b75e91c..9422c501 100644 --- a/lib/clixon/clixon.h.in +++ b/lib/clixon/clixon.h.in @@ -56,8 +56,6 @@ #undef CLIXON_VERSION_MINOR #undef CLIXON_VERSION_PATCH -#undef XML_COMPAT - /* * Use this constant to disable some prototypes that should not be visible outside the lib. * This is an alternative to use separate internal include files. diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 5b3d8765..78f043f8 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -161,16 +161,4 @@ int xml_body_uint32(cxobj *xb, uint32_t *val); int xml_operation(char *opstr, enum operation_type *op); char *xml_operation2str(enum operation_type op); -#ifdef XML_COMPAT /* See CHANGELOG */ -/* MANUAL CHANGE: xml_new(name, parent) --> xml_new(name, parent, NULL) */ - -#define xml_new_spec(name, parent) xml_new(name, parent, NULL) -#define clicon_xml_parse(xt, fmt, ...) xml_parse_va(xt, NULL, fmt, __VA_ARGS__) -#define clicon_xml_parse_file(fd, xt, etag) xml_parse_file(fd, etag, NULL, xt) -#define clicon_xml_parse_string(strp, xt) xml_parse_string(*strp, NULL, xt) -#define clicon_xml_parse_str(str, xt) xml_parse_string(str, NULL, xt) -#define xml_parse(str, xt) xml_parse_string(str, NULL, &xt) -#endif - - #endif /* _CLIXON_XML_H */ diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index 769237c9..f874cb3d 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -183,168 +183,6 @@ clicon_option_readfile_xml(clicon_hash_t *copt, return retval; } -#ifdef CONFIG_COMPAT - -/*! Read filename and set values to global options registry - * For legacy configuration file, ie not xml - * @see clicon_option_readfile_xml - */ -static int -clicon_option_readfile(clicon_hash_t *copt, - const char *filename) -{ - struct stat st; - char opt[1024]; - char val[1024]; - char line[1024]; - char *cp; - FILE *f = NULL; - int retval = -1; - - if (filename == NULL || !strlen(filename)){ - clicon_err(OE_UNIX, 0, "Not specified"); - goto done; - } - if (stat(filename, &st) < 0){ - clicon_err(OE_UNIX, errno, "%s", filename); - goto done; - } - if (!S_ISREG(st.st_mode)){ - clicon_err(OE_UNIX, 0, "%s is not a regular file", filename); - goto done; - } - if ((f = fopen(filename, "r")) == NULL) { - clicon_err(OE_UNIX, errno, "configure file: %s", filename); - return -1; - } - clicon_debug(2, "Reading config file %s", __FUNCTION__, filename); - while (fgets(line, sizeof(line), f)) { - if ((cp = strchr(line, '\n')) != NULL) /* strip last \n */ - *cp = '\0'; - /* Trim out comments, strip whitespace, and remove CR */ - if ((cp = strchr(line, '#')) != NULL) - memcpy(cp, "\n", 2); - if (sscanf(line, "%s %s", opt, val) < 2) - continue; - if ((hash_add(copt, - opt, - val, - strlen(val)+1)) == NULL) - goto done; - } - retval = 0; - done: - if (f) - fclose(f); - return retval; -} - -/*! Set default values of some options that may not appear in config-file - */ -static int -clicon_option_default(clicon_hash_t *copt) -{ - char *val; - int retval = 0; - - if (!hash_lookup(copt, "CLICON_YANG_MODULE_MAIN")){ - if (hash_add(copt, "CLICON_YANG_MODULE_MAIN", "clicon", strlen("clicon")+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_SOCK_GROUP")){ - val = CLICON_SOCK_GROUP; - if (hash_add(copt, "CLICON_SOCK_GROUP", val, strlen(val)+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_CLI_MODE")){ - if (hash_add(copt, "CLICON_CLI_MODE", "base", strlen("base")+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_MASTER_PLUGIN")){ - val = CLICON_MASTER_PLUGIN; - if (hash_add(copt, "CLICON_MASTER_PLUGIN", val, strlen(val)+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_CLI_GENMODEL")){ - if (hash_add(copt, "CLICON_CLI_GENMODEL", "1", strlen("1")+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_CLI_GENMODEL_TYPE")){ - if (hash_add(copt, "CLICON_CLI_GENMODEL_TYPE", "VARS", strlen("VARS")+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_AUTOCOMMIT")){ - if (hash_add(copt, "CLICON_AUTOCOMMIT", "0", strlen("0")+1) < 0) - goto done; - } - /* Legacy is 1 but default should really be 0. New apps should use 0 */ - if (!hash_lookup(copt, "CLICON_CLI_VARONLY")){ - if (hash_add(copt, "CLICON_CLI_VARONLY", "1", strlen("1")+1) < 0) - goto done; - } - if (!hash_lookup(copt, "CLICON_CLI_GENMODEL_COMPLETION")){ - if (hash_add(copt, "CLICON_CLI_GENMODEL_COMPLETION", "1", strlen("1")+1) < 0) - goto done; - } - /* Default is to use line-scrolling */ - if (!hash_lookup(copt, "CLICON_CLI_LINESCROLLING")){ - if (hash_add(copt, "CLICON_CLI_LINESCROLLING", "1", strlen("1")+1) < 0) - goto done; - } - retval = 0; - done: - return retval; -} - -/*! Check that options are set - */ -static int -clicon_option_sanity(clicon_hash_t *copt) -{ - int retval = -1; - - if (!hash_lookup(copt, "CLICON_CLI_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_CLI_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_CLISPEC_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_CLISPEC_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_BACKEND_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_BACKEND_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_NETCONF_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_NETCONF_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_RESTCONF_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_RESTCONF_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_YANG_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_YANG_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_XMLDB_DIR")){ - clicon_err(OE_UNIX, 0, "CLICON_XMLDB_DIR not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_SOCK")){ - clicon_err(OE_UNIX, 0, "CLICON_SOCK not defined in config file"); - goto done; - } - if (!hash_lookup(copt, "CLICON_BACKEND_PIDFILE")){ - clicon_err(OE_UNIX, 0, "CLICON_BACKEND_PIDFILE not defined in config file"); - goto done; - } - retval = 0; - done: - return retval; -} -#endif /* CONFIG_COMPAT */ - /*! Initialize option values * * Set default options, Read config-file, Check that all values are set. @@ -389,19 +227,8 @@ clicon_options_main(clicon_handle h) xml_child_sort = 0; } else { -#ifdef CONFIG_COMPAT - /* Set default options */ - if (clicon_option_default(copt) < 0) /* init registry from file */ - goto done; - /* Read configfile */ - if (clicon_option_readfile(copt, configfile) < 0) - goto done; - if (clicon_option_sanity(copt) < 0) - goto done; -#else /* CONFIG_COMPAT */ clicon_err(OE_CFG, 0, "%s: suffix %s not recognized (Run ./configure --with-config-compat?)", configfile, suffix); goto done; -#endif /* CONFIG_COMPAT */ } retval = 0; done: diff --git a/test/test_install.sh b/test/test_install.sh index 851eeae7..171b14b7 100755 --- a/test/test_install.sh +++ b/test/test_install.sh @@ -26,9 +26,6 @@ fi if [ ! -f $DIR/usr/local/share/clixon/clixon.mk ]; then err $DIR/usr/local/share/clixon/clixon.mk fi -if [ ! -f $DIR/usr/local/share/clixon/clixon.conf.cpp ]; then - err $DIR/usr/local/share/clixon/clixon.conf.cpp -fi if [ ! -f $DIR/usr/local/share/clixon/clixon-config* ]; then err $DIR/usr/local/share/clixon/clixon-config* fi From cc9111c8ee04dcc8daa37f1b70c79e0cf4640434 Mon Sep 17 00:00:00 2001 From: Renato Botelho Date: Fri, 5 Jan 2018 09:56:10 -0200 Subject: [PATCH 03/28] Update Makefile.in --- etc/Makefile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/Makefile.in b/etc/Makefile.in index 37e2e30d..f124dc52 100644 --- a/etc/Makefile.in +++ b/etc/Makefile.in @@ -52,10 +52,10 @@ install: clixonrc install -m 755 clixonrc $(DESTDIR)$(sysconfdir) install-include: - rm -f $(DESTDIR)$(sysconfdir)/clixonrc + uninstall: - + rm -f $(DESTDIR)$(sysconfdir)/clixonrc depend: From 83cb696d0e4a6f15016a599b195fb3fe86c1b372 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 6 Jan 2018 13:40:14 +0100 Subject: [PATCH 04/28] Corrected "No yang spec" printed on tty on leafref CLI usage --- CHANGELOG.md | 2 ++ lib/src/clixon_proto_client.c | 15 ++++++++++----- lib/src/clixon_xml_sort.c | 8 +++++--- lib/src/clixon_yang.c | 8 +++----- test/test_leafref.sh | 4 ++-- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcb394ee..2b58a9e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` ### Corrected Bugs +* Corrected "No yang spec" printed on tty on leafref CLI usage + ### Known issues ## 3.4.0 (1 January 2018) diff --git a/lib/src/clixon_proto_client.c b/lib/src/clixon_proto_client.c index 83fc792a..4a5808cf 100644 --- a/lib/src/clixon_proto_client.c +++ b/lib/src/clixon_proto_client.c @@ -71,10 +71,11 @@ /*! Send internal netconf rpc from client to backend * @param[in] h CLICON handle * @param[in] msg Encoded message. Deallocate woth free - * @param[out] xret Return value from backend as netconf xml tree. Free w xml_free + * @param[out] xret Return value from backend as xml tree. Free w xml_free * @param[inout] sock0 If pointer exists, do not close socket to backend on success * and return it here. For keeping a notify socket open - * Note: sock0 is if connection should be persistent, like a notification/subscribe api + * @note sock0 is if connection should be persistent, like a notification/subscribe api + * @note xret is populated with yangspec according to standard handle yangspec */ int clicon_rpc_msg(clicon_handle h, @@ -87,6 +88,7 @@ clicon_rpc_msg(clicon_handle h, int port; char *retdata = NULL; cxobj *xret = NULL; + yang_spec *yspec; if ((sock = clicon_sock(h)) == NULL){ clicon_err(OE_FATAL, 0, "CLICON_SOCK option not set"); @@ -119,9 +121,12 @@ clicon_rpc_msg(clicon_handle h, break; } clicon_debug(1, "%s retdata:%s", __FUNCTION__, retdata); - if (retdata && - xml_parse_string(retdata, NULL, &xret) < 0) - goto done; + + if (retdata){ + yspec = clicon_dbspec_yang(h); + if (xml_parse_string(retdata, yspec, &xret) < 0) + goto done; + } if (xret0){ *xret0 = xret; xret = NULL; diff --git a/lib/src/clixon_xml_sort.c b/lib/src/clixon_xml_sort.c index 5d4c7eb9..ef7e27c8 100644 --- a/lib/src/clixon_xml_sort.c +++ b/lib/src/clixon_xml_sort.c @@ -592,11 +592,8 @@ match_base_child(cxobj *x0, } /* Get match */ { - yang_node *y0; int yorder; - if ((y0 = yc->ys_parent) == NULL) - goto done; /* XXX: No we cant do this. on uppermost layer it can look like this: * config * ximport-----ymod = ietf @@ -613,7 +610,12 @@ match_base_child(cxobj *x0, *x0cp = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval); } else{ +#if 1 /* This is just a warning, but a catcher for when xml tree is not + populated with yang spec. If you see this, a previous inovation of, + for example xml_spec_populate() may be missing + */ clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__); +#endif *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval); } #else diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 7e435feb..2dcb578d 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -549,10 +549,8 @@ yang_find_myprefix(yang_stmt *ys) clicon_err(OE_YANG, 0, "My yang module not found"); goto done; } - if ((yprefix = yang_find((yang_node*)ymod, Y_PREFIX, NULL)) == NULL){ - clicon_err(OE_YANG, 0, "No prefix in my module"); + if ((yprefix = yang_find((yang_node*)ymod, Y_PREFIX, NULL)) == NULL) goto done; - } prefix = yprefix->ys_argument; done: return prefix; @@ -729,7 +727,6 @@ yang_find_module_by_prefix(yang_stmt *ys, clicon_err(OE_YANG, 0, "My yang spec not found"); goto done; } - myprefix = yang_find_myprefix(ys); if ((my_ymod = ys_module(ys)) == NULL){ clicon_err(OE_YANG, 0, "My yang module not found"); goto done; @@ -741,7 +738,8 @@ yang_find_module_by_prefix(yang_stmt *ys, goto done; } #endif - if (strcmp(myprefix, prefix) == 0){ + myprefix = yang_find_myprefix(ys); + if (myprefix && strcmp(myprefix, prefix) == 0){ ymod = my_ymod; goto done; } diff --git a/test/test_leafref.sh b/test/test_leafref.sh index e4986695..715bb89b 100755 --- a/test/test_leafref.sh +++ b/test/test_leafref.sh @@ -55,7 +55,7 @@ if [ $? -ne 0 ]; then err fi -new "start backend" +new "start backend -s init -f $cfg -y $fyang" # start new backend sudo clixon_backend -s init -f $cfg -y $fyang if [ $? -ne 0 ]; then @@ -102,7 +102,7 @@ expecteof "$clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" -exit + new "cli leafref lo" expectfn "$clixon_cli -1f $cfg -y $fyang -l o set default-address absname lo" "^$" From b5b96cde0c8eb7bb69690304104c89282639cd4e Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 6 Jan 2018 13:42:53 +0100 Subject: [PATCH 05/28] test: removed exit:s and printed backend start options --- test/test_netconf.sh | 2 +- test/test_perf.sh | 5 ++--- test/test_restconf.sh | 2 +- test/test_startup.sh | 3 +-- test/test_type.sh | 3 +-- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/test/test_netconf.sh b/test/test_netconf.sh index 12d55b5f..828464db 100755 --- a/test/test_netconf.sh +++ b/test/test_netconf.sh @@ -16,7 +16,7 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi -new "start backend" +new "start backend -s init -f $cfg" # start new backend sudo clixon_backend -s init -f $cfg if [ $? -ne 0 ]; then diff --git a/test/test_perf.sh b/test/test_perf.sh index 580b2d16..483dace7 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -1,7 +1,7 @@ #!/bin/bash # Scaling test -number=1000 +number=5000 req=100 if [ $# = 0 ]; then number=1000 @@ -58,7 +58,6 @@ cat < $cfg EOF - # kill old backend (if any) new "kill old backend" sudo clixon_backend -zf $cfg -y $fyang @@ -100,7 +99,7 @@ expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" -exit + new "netconf add one small config" expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "xy]]>]]>" "^]]>]]>$" diff --git a/test/test_restconf.sh b/test/test_restconf.sh index b63109d0..eccd4035 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -15,7 +15,7 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi -new "start backend" +new "start backend -s init -f $cfg" sudo clixon_backend -s init -f $cfg if [ $? -ne 0 ]; then err diff --git a/test/test_startup.sh b/test/test_startup.sh index 1793b89c..93541b2a 100755 --- a/test/test_startup.sh +++ b/test/test_startup.sh @@ -84,8 +84,7 @@ EOF err fi - new "start backend" - # start new backend + new "start backend -f $cfg -s $mode -c /tmp/config" sudo clixon_backend -f $cfg -s $mode -c /tmp/config if [ $? -ne 0 ]; then err diff --git a/test/test_type.sh b/test/test_type.sh index 3c01e271..0c6afbef 100755 --- a/test/test_type.sh +++ b/test/test_type.sh @@ -69,8 +69,7 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi -new "start backend" -# start new backend +new "start backend -s init -f $cfg -y $fyang" sudo clixon_backend -s init -f $cfg -y $fyang if [ $? -ne 0 ]; then err From cefaf4717f5c7ea24c958717580c9f77fba4f629 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sun, 7 Jan 2018 17:56:33 +0100 Subject: [PATCH 06/28] Making the tests self-contained wrt config file and lib.sh creating test scratch dir. --- test/lib.sh | 9 ++++ test/test_cli.sh | 22 ++++++++- test/test_datastore.sh | 23 +++++---- test/test_install.sh | 46 +++++++++--------- test/test_leafref.sh | 24 ++++++++-- test/test_netconf.sh | 27 ++++++++++- test/test_order.sh | 14 ++++-- test/test_perf.sh | 11 +++-- test/test_restconf.sh | 105 ++++++++++++++++++++++++++++++++++++----- test/test_startup.sh | 20 ++++---- test/test_type.sh | 24 ++++++++-- test/test_yang.sh | 9 ++-- 12 files changed, 256 insertions(+), 78 deletions(-) diff --git a/test/lib.sh b/test/lib.sh index 467fccd8..00eec2d7 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -1,7 +1,16 @@ #!/bin/bash +# Define test functions. +# Create working dir as variable "dir" testnr=0 testname= + +dir=/var/tmp/$0 +if [ ! -d $dir ]; then + mkdir $dir +fi +rm -rf $dir/* + # error and exit, arg is optional extra errmsg err(){ echo "Error in Test$testnr [$testname]:" diff --git a/test/test_cli.sh b/test/test_cli.sh index a77329ca..75f596eb 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -1,4 +1,5 @@ #!/bin/bash + # Test1: backend and cli basic functionality # Start backend server # Add an ethernet interface and an address @@ -7,9 +8,25 @@ # Set the mandatory type # Commit -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/usr/local/etc/routing.xml +cfg=$dir/conf_yang.xml + +cat < $cfg + + $cfg + /usr/local/share/routing/yang + example + /usr/local/lib/routing/clispec + /usr/local/lib/routing/cli + routing + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + 1 + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF # For memcheck #clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli" @@ -104,3 +121,4 @@ if [ $? -ne 0 ]; then err "kill backend" fi +rm -rf $dir diff --git a/test/test_datastore.sh b/test/test_datastore.sh index acf6af59..6f7ae875 100755 --- a/test/test_datastore.sh +++ b/test/test_datastore.sh @@ -2,12 +2,13 @@ # Test5: datastore tests. # Just run a binary direct to datastore. No clixon. -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh +fyang=$dir/ietf-ip.yang datastore=datastore_client -cat < /tmp/ietf-ip.yang +cat < $fyang module ietf-ip{ container x { list y { @@ -46,14 +47,14 @@ db='12first-entry13se run(){ name=$1 - dir=/tmp/$name + mydir=$dir/$name - if [ ! -d $dir ]; then - mkdir $dir + if [ ! -d $mydir ]; then + mkdir $mydir fi - rm -rf $dir/* + rm -rf $mydir/* - conf="-d candidate -b $dir -p ../datastore/$name/$name.so -y /tmp -m ietf-ip" + conf="-d candidate -b $mydir -p ../datastore/$name/$name.so -y $dir -m ietf-ip" echo "conf:$conf" new "datastore $name init" expectfn "$datastore $conf init" "" @@ -140,21 +141,23 @@ run(){ expectfn "$datastore $conf put create 13newentry" new "datastore other db init" - expectfn "$datastore -d kalle -b $dir -p ../datastore/$name/$name.so -y /tmp -m ietf-ip init" + expectfn "$datastore -d kalle -b $mydir -p ../datastore/$name/$name.so -y $dir -m ietf-ip init" new "datastore other db copy" expectfn "$datastore $conf copy kalle" "" - diff $dir/kalle_db $dir/candidate_db + diff $mydir/kalle_db $mydir/candidate_db new "datastore lock" expectfn "$datastore $conf lock 756" "" #leaf-list - rm -rf $dir + rm -rf $mydir } #run keyvalue # cant get the put to work run text +rm -rf $dir + diff --git a/test/test_install.sh b/test/test_install.sh index 171b14b7..5e88336b 100755 --- a/test/test_install.sh +++ b/test/test_install.sh @@ -1,64 +1,60 @@ #!/bin/bash # Install test -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -DIR=/tmp/clixoninstall - -new "Set up installdir $DIR" -rm -rf $DIR -mkdir $DIR +new "Set up installdir $dir" new "Make DESTDIR install" -(cd ..; make DESTDIR=$DIR install) +(cd ..; make DESTDIR=$dir install) if [ $? -ne 0 ]; then err fi new "Check installed files" -if [ ! -d $DIR/usr ]; then - err $DIR/usr +if [ ! -d $dir/usr ]; then + err $dir/usr fi -if [ ! -d $DIR/www-data ]; then - err $DIR/www-data +if [ ! -d $dir/www-data ]; then + err $dir/www-data fi -if [ ! -f $DIR/usr/local/share/clixon/clixon.mk ]; then - err $DIR/usr/local/share/clixon/clixon.mk +if [ ! -f $dir/usr/local/share/clixon/clixon.mk ]; then + err $dir/usr/local/share/clixon/clixon.mk fi -if [ ! -f $DIR/usr/local/share/clixon/clixon-config* ]; then - err $DIR/usr/local/share/clixon/clixon-config* +if [ ! -f $dir/usr/local/share/clixon/clixon-config* ]; then + err $dir/usr/local/share/clixon/clixon-config* fi -if [ ! -h $DIR/usr/local/lib/libclixon.so ]; then - err $DIR/usr/local/lib/libclixon.so +if [ ! -h $dir/usr/local/lib/libclixon.so ]; then + err $dir/usr/local/lib/libclixon.so fi -if [ ! -h $DIR/usr/local/lib/libclixon_backend.so ]; then - err $DIR/usr/local/lib/libclixon_backend.so +if [ ! -h $dir/usr/local/lib/libclixon_backend.so ]; then + err $dir/usr/local/lib/libclixon_backend.so fi new "Make DESTDIR install include" -(cd ..; make DESTDIR=$DIR install-include) +(cd ..; make DESTDIR=$dir install-include) if [ $? -ne 0 ]; then err fi new "Check installed includes" -if [ ! -f $DIR/usr/local/include/clixon/clixon.h ]; then - err $DIR/usr/local/include/clixon/clixon.h +if [ ! -f $dir/usr/local/include/clixon/clixon.h ]; then + err $dir/usr/local/include/clixon/clixon.h fi new "Make DESTDIR uninstall" -(cd ..; make DESTDIR=$DIR uninstall) +(cd ..; make DESTDIR=$dir uninstall) if [ $? -ne 0 ]; then err fi new "Check remaining files" -f=$(find $DIR -type f) +f=$(find $dir -type f) if [ -n "$f" ]; then err "$f" fi new "Check remaining symlinks" -l=$(find $DIR -type l) +l=$(find $dir -type l) if [ -n "$l" ]; then err "$l" fi diff --git a/test/test_leafref.sh b/test/test_leafref.sh index 715bb89b..f31893af 100755 --- a/test/test_leafref.sh +++ b/test/test_leafref.sh @@ -1,10 +1,26 @@ #!/bin/bash # Test7: Yang specifics: leafref -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/usr/local/etc/routing.xml -fyang=/tmp/leafref.yang +cfg=$dir/conf_yang.xml +fyang=$dir/leafref.yang + +cat < $cfg + + $cfg + /usr/local/share/routing/yang + example + /usr/local/lib/routing/clispec + /usr/local/lib/routing/cli + routing + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + 1 + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -126,3 +142,5 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err "kill backend" fi + +rm -rf $dir diff --git a/test/test_netconf.sh b/test/test_netconf.sh index 828464db..5b6798ec 100755 --- a/test/test_netconf.sh +++ b/test/test_netconf.sh @@ -1,9 +1,30 @@ #!/bin/bash # Test2: backend and netconf basic functionality -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/usr/local/etc/routing.xml + +cfg=$dir/conf_yang.xml + +cat < $cfg + + $cfg + /usr/local/share/routing/yang + example + /usr/local/lib/routing/clispec + /usr/local/lib/routing/backend + /usr/local/lib/routing/netconf + /usr/local/lib/routing/restconf + /usr/local/lib/routing/cli + routing + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + 1 + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + # For memcheck #clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -146,3 +167,5 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err "kill backend" fi + +rm -rf $dir diff --git a/test/test_order.sh b/test/test_order.sh index 16ea036f..433ee9bd 100755 --- a/test/test_order.sh +++ b/test/test_order.sh @@ -6,20 +6,22 @@ # No test of ordered-by system is done yet # (we may want to sort them alphabetically for better performance). -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/tmp/conf_yang.xml -fyang=/tmp/order.yang +cfg=$dir/conf_yang.xml +fyang=$dir/order.yang # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" clixon_netconf=clixon_netconf clixon_cli=clixon_cli -dbdir=/tmp/order +dbdir=$dir/order new "Set up $dbdir" rm -rf $dbdir -mkdir $dbdir +if [ ! -d $dbdir ]; then + mkdir $dbdir +fi cat < $cfg @@ -175,3 +177,5 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err "kill backend" fi + +rm -rf $dir diff --git a/test/test_perf.sh b/test/test_perf.sh index 483dace7..a91e6bb4 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -14,13 +14,14 @@ else echo "Usage: $0 [ []]" exit 1 fi -cfg=/tmp/scaling-conf.xml -fyang=/tmp/scaling.yang -fconfig=/tmp/config -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh +cfg=$dir/scaling-conf.xml +fyang=$dir/scaling.yang +fconfig=$dir/config + # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -153,3 +154,5 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err "kill backend" fi + +rm -rf $dir diff --git a/test/test_restconf.sh b/test/test_restconf.sh index eccd4035..395e2827 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -2,11 +2,46 @@ # Restconf basic functionality # Assume http server setup, such as nginx described in apps/restconf/README.md -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/usr/local/etc/routing.xml +cfg=$dir/conf.xml +fyang=$dir/restconf.yang -# This is a fixed 'state' implemented in routing_backend. It is always there +cat < $cfg + + $cfg + /usr/local/share/routing/yang + example + /usr/local/lib/routing/clispec + /usr/local/lib/routing/backend + /usr/local/lib/routing/restconf + false + /usr/local/lib/routing/cli + routing + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + 1 + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + +cat < $fyang +module example{ + import ietf-ip { + prefix ip; + } + import ietf-routing { + prefix rt; + } + import ietf-inet-types { + prefix "inet"; + revision-date "2013-07-15"; + } +} +EOF + +# This is a fixed 'state' implemented in routing_backend. It is assumed to be always there state='{"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}' # kill old backend (if any) @@ -15,8 +50,8 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err fi -new "start backend -s init -f $cfg" -sudo clixon_backend -s init -f $cfg +new "start backend -s init -f $cfg -y $fyang" +sudo clixon_backend -s init -f $cfg -y $fyang if [ $? -ne 0 ]; then err fi @@ -25,7 +60,7 @@ new "kill old restconf daemon" sudo pkill -u www-data clixon_restconf new "start restconf daemon" -sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df /usr/local/etc/routing.xml # -D +sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg # -D sleep 1 @@ -38,11 +73,51 @@ new "restconf head" expectfn "curl -sS -I http://localhost/restconf/data" "HTTP/1.1 200 OK" #Content-Type: application/yang-data+json" -new "restconf get empty config + state" -expectfn "curl -sSG http://localhost/restconf/data" $state +new "restconf root discovery" +expectfn "curl -sS -X GET http://localhost/.well-known/host-meta" "" -new "restconf get state operation" -expectfn "curl -sS -G http://localhost/restconf/data/interfaces-state" $state +#new "restconf get restconf json XXX" +#expectfn "curl -sSG http://localhost/restconf" "{\"restconf\" : $state }" + +#new "restconf get restconf/yang-library-version json XXX" +#expectfn "curl -sSG http://localhost/restconf/yang-library-version" "{\"restconf\" : $state }" + + +new "restconf get empty config + state json" +expectfn "curl -sSG http://localhost/restconf/data" "{\"data\": $state}" + +new "restconf get empty config + state xml" +ret=$(curl -sS -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data) +expect="eth0eth42" +match=`echo $ret | grep -EZo "$expect"` +if [ -z "$match" ]; then + err "$expect" "$ret" +fi + +new "restconf get data/interfaces-state/interface=eth0 json" +expectfn "curl -sS -G http://localhost/restconf/data/interfaces-state/interface=eth0" '{"interface": {"name": "eth0","type": "eth","if-index": "42"}}' + +new "restconf get state operation eth0 xml" +# Cant get shell macros to work, inline matching from lib.sh +ret=$(curl -sS -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0) +expect="eth0eth42" +match=`echo $ret | grep -EZo "$expect"` +if [ -z "$match" ]; then + err "$expect" "$ret" +fi + +new "restconf get state operation eth0 type json" +expectfn "curl -sS -G http://localhost/restconf/data/interfaces-state/interface=eth0/type" '{"type": "eth"} +$' + +new "restconf get state operation eth0 type xml" +# Cant get shell macros to work, inline matching from lib.sh +ret=$(curl -sS -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0/type) +expect="eth" +match=`echo $ret | grep -EZo "$expect"` +if [ -z "$match" ]; then + err "$expect" "$ret" +fi new "restconf Add subtree to datastore using POST" expectfn 'curl -sS -X POST -d {"interfaces":{"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}}} http://localhost/restconf/data' "" @@ -93,12 +168,16 @@ $' new "restconf operation rpc using POST json" expectfn 'curl -sS -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{ "output": { "route": { "address-family": "ipv4", "next-hop": { "next-hop-list": "2.3.4.5" } } } } ' +exit # XXX + new "restconf rpc using POST xml" +# Cant get shell macros to work, inline matching from lib.sh ret=$(curl -sS -X POST -H "Accept: application/yang-data+xml" -d '{"input":{"routing-instance-name":"ipv4"}}' http://localhost/restconf/operations/rt:fib-route) expect=" ipv4 2.3.4.5 " match=`echo $ret | grep -EZo "$expect"` -echo -n "ret: " -echo $ret +if [ -z "$match" ]; then + err "$expect" "$ret" +fi new "Kill restconf daemon" sudo pkill -u www-data clixon_restconf @@ -114,3 +193,5 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err "kill backend" fi + +rm -rf $dir diff --git a/test/test_startup.sh b/test/test_startup.sh index 93541b2a..53f7c257 100755 --- a/test/test_startup.sh +++ b/test/test_startup.sh @@ -6,9 +6,9 @@ # - running db starts with a "run" interface # - startup db starts with a "start" interface -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/tmp/conf_startup.xml +cfg=$dir/conf_startup.xml # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -42,7 +42,8 @@ run(){ mode=$1 expect=$2 - cat < /tmp/db + dbdir=$dir/db + cat < $dbdir @@ -52,9 +53,9 @@ run(){ EOF - sudo mv /tmp/db /usr/local/var/routing/running_db + sudo mv $dbdir /usr/local/var/routing/running_db - cat < /tmp/db + cat < $dbdir @@ -64,9 +65,9 @@ EOF EOF - sudo mv /tmp/db /usr/local/var/routing/startup_db + sudo mv $dbdir /usr/local/var/routing/startup_db - cat < /tmp/config + cat < $dir/config @@ -84,8 +85,8 @@ EOF err fi - new "start backend -f $cfg -s $mode -c /tmp/config" - sudo clixon_backend -f $cfg -s $mode -c /tmp/config + new "start backend -f $cfg -s $mode -c $dir/config" + sudo clixon_backend -f $cfg -s $mode -c $dir/config if [ $? -ne 0 ]; then err fi @@ -111,3 +112,4 @@ run none 'runethextraethtruelolocaltruerunethtrue' run startup 'extraethtruelolocaltruestartupethtrue' +rm -rf $dir diff --git a/test/test_type.sh b/test/test_type.sh index 0c6afbef..432d30cb 100755 --- a/test/test_type.sh +++ b/test/test_type.sh @@ -2,10 +2,27 @@ # Advanced union types and generated code # and enum w values -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/usr/local/etc/routing.xml -fyang=/tmp/type.yang +fyang=$dir/type.yang +cfg=$dir/conf_yang.xml + +cat < $cfg + + $cfg + /usr/local/share/routing/yang + example + /usr/local/lib/routing/clispec + /usr/local/lib/routing/cli + routing + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + 1 + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + # For memcheck #clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli" @@ -126,3 +143,4 @@ if [ $? -ne 0 ]; then err "kill backend" fi +rm -rf $dir diff --git a/test/test_yang.sh b/test/test_yang.sh index 506109b3..c2cdead5 100755 --- a/test/test_yang.sh +++ b/test/test_yang.sh @@ -1,10 +1,11 @@ #!/bin/bash # Test4: Yang specifics: multi-keys and empty type -# include err() and new() functions +# include err() and new() functions and creates $dir . ./lib.sh -cfg=/tmp/conf_yang.xml -fyang=/tmp/test.yang + +cfg=$dir/conf_yang.xml +fyang=$dir/test.yang # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -151,3 +152,5 @@ sudo clixon_backend -zf $cfg if [ $? -ne 0 ]; then err "kill backend" fi + +rm -rf $dir From 363bd5d19dbfa830d1b2daa73a3c219274e8dd04 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sun, 7 Jan 2018 18:01:42 +0100 Subject: [PATCH 07/28] New configuration option: CLICON_RESTCONF_PRETTY Changed RESTCONF GET to return object referenced. ie, GET /restconf/data/X returns X. --- CHANGELOG.md | 3 + apps/cli/cli_show.c | 1 - apps/restconf/README.md | 2 + apps/restconf/restconf_methods.c | 191 ++++++++++++++++++++--------- lib/src/clixon_xml_sort.c | 51 ++------ lib/src/clixon_xsl.c | 8 +- yang/clixon-config@2017-12-27.yang | 15 ++- 7 files changed, 169 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b58a9e5..6d7a1eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` +* new configuration option: CLICON_RESTCONF_PRETTY +* Changed RESTCONF GET to return object referenced. ie, GET /restconf/data/X returns X. + ### Corrected Bugs * Corrected "No yang spec" printed on tty on leafref CLI usage diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index 10bfd062..0948f46b 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -88,7 +88,6 @@ * @param[out] commands vector of function pointers to callback functions * @param[out] helptxt vector of pointers to helptexts * @see cli_expand_var_generate This is where arg is generated - * XXX: helptexts? */ int expand_dbvar(void *h, diff --git a/apps/restconf/README.md b/apps/restconf/README.md index 807483d3..a7952a9c 100644 --- a/apps/restconf/README.md +++ b/apps/restconf/README.md @@ -9,6 +9,8 @@ There is currently (2017) a [RFC 8040: RESTCONF Protocol](https://tools.ietf.org including: - query parameters (section 4.9) - notifications (sec 6) +- GET /restconf/ (sec 3.3) +- GET /restconf/yang-library-version (sec 3.3.3) - only rudimentary error reporting exists (sec 7) ### Installation using Nginx diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 21331273..549cda02 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -140,82 +140,139 @@ api_data_options(clicon_handle h, return 0; } -/*! Generic GET (both HEAD and GET) +/*! Return error on get/head request + * @param[in] h Clixon handle + * @param[in] r Fastcgi request handle + * @param[in] xerr XML error message from backend */ static int -api_data_get_gen(clicon_handle h, +api_data_get_err(clicon_handle h, FCGX_Request *r, - cvec *pcvec, - int pi, - cvec *qvec, - int head) + cxobj *xerr) { int retval = -1; - cbuf *path = NULL; + cbuf *cbj = NULL; + cxobj *xtag; + int code; + const char *reason_phrase; + + if ((cbj = cbuf_new()) == NULL) + goto done; + if ((xtag = xpath_first(xerr, "/error-tag")) == NULL){ + notfound(r); /* bad reply? */ + goto done; + } + code = restconf_err2code(xml_body(xtag)); + if ((reason_phrase = restconf_code2reason(code)) == NULL) + reason_phrase=""; + clicon_debug(1, "%s code:%d reason phrase:%s", + __FUNCTION__, code, reason_phrase); + + if (xml_name_set(xerr, "error") < 0) + goto done; + if (xml2json_cbuf(cbj, xerr, 1) < 0) + goto done; + FCGX_FPrintF(r->out, "Status: %d %s\r\n", code, reason_phrase); + FCGX_FPrintF(r->out, "Content-Type: application/yang-data+json\r\n\r\n"); + FCGX_FPrintF(r->out, "\r\n"); + FCGX_FPrintF(r->out, "{\r\n"); + FCGX_FPrintF(r->out, " \"ietf-restconf:errors\" : {\r\n"); + FCGX_FPrintF(r->out, " %s", cbuf_get(cbj)); + FCGX_FPrintF(r->out, " }\r\n"); + FCGX_FPrintF(r->out, "}\r\n"); + retval = 0; + done: + if (cbj) + cbuf_free(cbj); + return retval; +} + +/*! Generic GET (both HEAD and GET) + * According to restconf + * @param[in] h Clixon handle + * @param[in] r Fastcgi request handle + * @param[in] pcvec Vector of path ie DOCUMENT_URI element + * @param[in] pi Offset, where path starts + * @param[in] qvec Vector of query string (QUERY_STRING) + * @param[in] head If 1 is HEAD, otherwise GET + * @code + * curl -G http://localhost/restconf/data/interfaces/interface=eth0 + * @endcode + * XXX: cant find a way to use Accept request field to choose Content-Type + * I would like to support both xml and json. + * Request may contain + * Accept: application/yang.data+json,application/yang.data+xml + * Response contains one of: + * Content-Type: application/yang-data+xml + * Content-Type: application/yang-data+json + * NOTE: If a retrieval request for a data resource representing a YANG leaf- + * list or list object identifies more than one instance, and XML + * encoding is used in the response, then an error response containing a + * "400 Bad Request" status-line MUST be returned by the server. + * Netconf: , + */ +static int +api_data_get2(clicon_handle h, + FCGX_Request *r, + cvec *pcvec, + int pi, + cvec *qvec, + int head) +{ + int retval = -1; + cbuf *cbpath = NULL; + char *path; cbuf *cbx = NULL; - cxobj **vec = NULL; yang_spec *yspec; cxobj *xret = NULL; cxobj *xerr; - cxobj *xtag; - cbuf *cbj = NULL;; - int code; - const char *reason_phrase; char *media_accept; int use_xml = 0; /* By default use JSON */ + cxobj **xvec = NULL; + size_t xlen; + int pretty; + int i; + cxobj *x; clicon_debug(1, "%s", __FUNCTION__); + pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY"); media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp); if (strcmp(media_accept, "application/yang-data+xml")==0) use_xml++; yspec = clicon_dbspec_yang(h); - if ((path = cbuf_new()) == NULL) + if ((cbpath = cbuf_new()) == NULL) goto done; - cprintf(path, "/"); - if (api_path2xpath_cvv(yspec, pcvec, pi, path) < 0){ + cprintf(cbpath, "/"); + clicon_debug(1, "%s pi:%d", __FUNCTION__, pi); + /* We know "data" is element pi-1 */ + if (api_path2xpath_cvv(yspec, pcvec, pi, cbpath) < 0){ notfound(r); goto done; } - clicon_debug(1, "%s path:%s", __FUNCTION__, cbuf_get(path)); - if (clicon_rpc_get(h, cbuf_get(path), &xret) < 0){ + path = cbuf_get(cbpath); + clicon_debug(1, "%s path:%s", __FUNCTION__, path); + if (clicon_rpc_get(h, path, &xret) < 0){ notfound(r); goto done; } -#if 0 /* DEBUG */ + /* We get return via netconf which is complete tree from root + * We need to cut that tree to only the object. + */ +#if 1 /* DEBUG */ { cbuf *cb = cbuf_new(); - xml2json_cbuf(cb, xret, 1); + clicon_xml2cbuf(cb, xret, 0, 0); clicon_debug(1, "%s xret:%s", __FUNCTION__, cbuf_get(cb)); cbuf_free(cb); } #endif + /* Check if error return */ if ((xerr = xpath_first(xret, "/rpc-error")) != NULL){ - if ((cbj = cbuf_new()) == NULL) + if (api_data_get_err(h, r, xerr) < 0) goto done; - if ((xtag = xpath_first(xerr, "/error-tag")) == NULL){ - notfound(r); /* bad reply? */ - goto done; - } - code = restconf_err2code(xml_body(xtag)); - if ((reason_phrase = restconf_code2reason(code)) == NULL) - reason_phrase=""; - clicon_debug(1, "%s code:%d reason phrase:%s", - __FUNCTION__, code, reason_phrase); - - if (xml_name_set(xerr, "error") < 0) - goto done; - if (xml2json_cbuf(cbj, xerr, 1) < 0) - goto done; - FCGX_FPrintF(r->out, "Status: %d %s\r\n", code, reason_phrase); - FCGX_FPrintF(r->out, "Content-Type: application/yang-data+json\r\n\r\n"); - FCGX_FPrintF(r->out, "\r\n"); - FCGX_FPrintF(r->out, "{\r\n"); - FCGX_FPrintF(r->out, " \"ietf-restconf:errors\" : {\r\n"); - FCGX_FPrintF(r->out, " %s", cbuf_get(cbj)); - FCGX_FPrintF(r->out, " }\r\n"); - FCGX_FPrintF(r->out, "}\r\n"); goto ok; } + /* Normal return, no error */ if ((cbx = cbuf_new()) == NULL) goto done; FCGX_SetExitStatus(200, r->out); /* OK */ @@ -223,17 +280,39 @@ api_data_get_gen(clicon_handle h, FCGX_FPrintF(r->out, "\r\n"); if (head) goto ok; - clicon_debug(1, "%s name:%s child:%d", __FUNCTION__, xml_name(xret), xml_child_nr(xret)); - - clicon_debug(1, "%s xretnr:%d", __FUNCTION__, xml_child_nr(xret)); - if (use_xml){ - if (clicon_xml2cbuf(cbx, xret, 0, 1) < 0) /* Dont print top object? */ - goto done; + if (path==NULL || strcmp(path,"/")==0){ /* Special case: data root */ + if (use_xml){ + if (clicon_xml2cbuf(cbx, xret, 0, pretty) < 0) /* Dont print top object? */ + goto done; + } + else{ + if (xml2json_cbuf(cbx, xret, pretty) < 0) + goto done; + } } else{ - vec = xml_childvec_get(xret); - if (xml2json_cbuf_vec(cbx, vec, xml_child_nr(xret), 0) < 0) + if (xpath_vec(xret, path, &xvec, &xlen) < 0) goto done; + clicon_debug(1, "%s: xpath:%s xlen:%d", __FUNCTION__, path, xlen); + for (i=0; iout, "%s", cbx?cbuf_get(cbx):""); @@ -244,12 +323,12 @@ api_data_get_gen(clicon_handle h, clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); if (cbx) cbuf_free(cbx); - if (cbj) - cbuf_free(cbj); - if (path) - cbuf_free(path); + if (cbpath) + cbuf_free(cbpath); if (xret) xml_free(xret); + if (xvec) + free(xvec); return retval; } @@ -271,7 +350,7 @@ api_data_head(clicon_handle h, int pi, cvec *qvec) { - return api_data_get_gen(h, r, pcvec, pi, qvec, 1); + return api_data_get2(h, r, pcvec, pi, qvec, 1); } /*! REST GET method @@ -304,7 +383,7 @@ api_data_get(clicon_handle h, int pi, cvec *qvec) { - return api_data_get_gen(h, r, pcvec, pi, qvec, 0); + return api_data_get2(h, r, pcvec, pi, qvec, 0); } /*! REST POST method diff --git a/lib/src/clixon_xml_sort.c b/lib/src/clixon_xml_sort.c index ef7e27c8..ed2e987e 100644 --- a/lib/src/clixon_xml_sort.c +++ b/lib/src/clixon_xml_sort.c @@ -549,7 +549,8 @@ match_base_child(cxobj *x0, char **keyval = NULL; char **keyvec = NULL; int i; - + int yorder; + *x0cp = NULL; /* return value */ switch (yc->ys_keyword){ case Y_CONTAINER: /* Equal regardless */ @@ -591,51 +592,21 @@ match_base_child(cxobj *x0, break; } /* Get match */ - { - int yorder; - - /* XXX: No we cant do this. on uppermost layer it can look like this: - * config - * ximport-----ymod = ietf - * interfaces--ymod = example - * Which means yang order can be different for different children in the - * same childvec. - */ - if (xml_child_sort==0) - *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval); + if (xml_child_sort==0) + *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval); + else{ + if (xml_child_nr(x0)==0 || xml_spec(xml_child_i(x0,0))!=NULL){ + yorder = yang_order(yc); + *x0cp = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval); + } else{ -#if 1 - if (xml_child_nr(x0)==0 || xml_spec(xml_child_i(x0,0))!=NULL){ - yorder = yang_order(yc); - *x0cp = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval); - } - else{ #if 1 /* This is just a warning, but a catcher for when xml tree is not - populated with yang spec. If you see this, a previous inovation of, + populated with yang spec. If you see this, a previous invacation of, for example xml_spec_populate() may be missing */ - clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__); + clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__); #endif - *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval); - } -#else - cxobj *xx; - - - *x0cp = xml_match(x0, xml_name(x1c), yc->ys_keyword, keynr, keyvec, keyval); - if (xml_child_nr(x0) && xml_spec(xml_child_i(x0,0))!=NULL){ - xx = xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval); - if (xx!=*x0cp){ - clicon_log(LOG_WARNING, "%s mismatch", __FUNCTION__); - fprintf(stderr, "mismatch\n"); - xml_search(x0, xml_name(x1c), yorder, yc->ys_keyword, keynr, keyvec, keyval); - assert(0); - } - } - else - clicon_log(LOG_WARNING, "%s No yspec", __FUNCTION__); -#endif } } ok: diff --git a/lib/src/clixon_xsl.c b/lib/src/clixon_xsl.c index a44f28a1..22be2b63 100644 --- a/lib/src/clixon_xsl.c +++ b/lib/src/clixon_xsl.c @@ -1046,10 +1046,10 @@ xpath_each(cxobj *xcur, */ int xpath_vec(cxobj *xcur, - char *format, - cxobj ***vec, - size_t *veclen, - ...) + char *format, + cxobj ***vec, + size_t *veclen, + ...) { int retval = -1; va_list ap; diff --git a/yang/clixon-config@2017-12-27.yang b/yang/clixon-config@2017-12-27.yang index 1bbc1dcb..d9190da7 100644 --- a/yang/clixon-config@2017-12-27.yang +++ b/yang/clixon-config@2017-12-27.yang @@ -11,7 +11,7 @@ module clixon-config { description "Clixon configuration file ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2017 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2018 Olof Hagsand and Benny Holmgren This file is part of CLIXON @@ -130,6 +130,19 @@ module clixon-config { "FastCGI unix socket. Should be specified in webserver Eg in nginx: fastcgi_pass unix:/www-data/clicon_restconf.sock"; } + leaf CLICON_RESTCONF_PRETTY { + type boolean; + default true; + description + "Restconf return value pretty print. + Restconf clients may add HTTP header: + Accept: application/yang-data+json, or + Accept: application/yang-data+xml + to get return value in XML or JSON. + RFC 8040 examples print XML and JSON in pretty-printed form. + Setting this value to false makes restconf return not pretty-printed + which may be desirable for performance or tests"; + } leaf CLICON_CLI_DIR { type string; description From 6a9697c2ffd4f04cebfca3be1de7bad3e6b2fb90 Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Sun, 14 Jan 2018 21:25:17 -0600 Subject: [PATCH 08/28] Use default config file when none is provided Add config option to set a default config file path. If none given, the default config file path is /etc/clixon.xml. --- configure | 19 +++++++++++++++++++ configure.ac | 9 +++++++++ include/clixon_config.h.in | 3 +++ lib/src/clixon_options.c | 3 +-- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 80d4af73..0f897b4e 100755 --- a/configure +++ b/configure @@ -709,6 +709,7 @@ with_cligen with_restconf with_keyvalue with_qdbm +with_configfile ' ac_precious_vars='build_alias host_alias @@ -1347,6 +1348,7 @@ Optional Packages: --without-restconf disable support for restconf --with-keyvalue enable support for key-value xmldb datastore --with-qdbm=dir Use QDBM here, if keyvalue + --with-configfile set default path to config file Some influential environment variables: CC C compiler command @@ -4045,6 +4047,16 @@ fi fi +# Set default config file location + +# Check whether --with-configfile was given. +if test "${with_configfile+set}" = set; then : + withval=$with_configfile; DEFAULT_CONFIG="$withval" +else + DEFAULT_CONFIG="/etc/clixon.xml" +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for crypt in -lcrypt" >&5 $as_echo_n "checking for crypt in -lcrypt... " >&6; } if ${ac_cv_lib_crypt_crypt+:} false; then : @@ -4290,6 +4302,13 @@ cat >>confdefs.h <<_ACEOF _ACEOF +# Default location for config file + +cat >>confdefs.h <<_ACEOF +#define CLIXON_DEFAULT_CONFIG "${DEFAULT_CONFIG}" +_ACEOF + + # See also datastore/keyvalue/Makefile in with_keyvalue clause above diff --git a/configure.ac b/configure.ac index 54bc1c4b..73c6f07b 100644 --- a/configure.ac +++ b/configure.ac @@ -161,6 +161,12 @@ if test "x${with_keyvalue}" == xyes; then AC_CONFIG_FILES(datastore/keyvalue/Makefile) fi +# Set default config file location +AC_ARG_WITH([configfile], + [AS_HELP_STRING([--with-configfile],[set default path to config file])], + [DEFAULT_CONFIG="$withval"], + [DEFAULT_CONFIG="/etc/clixon.xml"]) + AC_CHECK_LIB(crypt, crypt) AC_CHECK_HEADERS(crypt.h) @@ -182,6 +188,9 @@ AC_CHECK_FUNCS(inet_aton sigaction sigvec strlcpy strsep strndup alphasort versi # same as clixon_DATADIR defined in clixon.mk.cpp for Makefiles AC_DEFINE_UNQUOTED(CLIXON_DATADIR, "${prefix}/share/clixon", [Clixon data dir for system yang files etc]) +# Default location for config file +AC_DEFINE_UNQUOTED(CLIXON_DEFAULT_CONFIG,"${DEFAULT_CONFIG}",[Location for apps to find default config file]) + AH_BOTTOM([#include ]) # See also datastore/keyvalue/Makefile in with_keyvalue clause above diff --git a/include/clixon_config.h.in b/include/clixon_config.h.in index 8d7f194e..d356c165 100644 --- a/include/clixon_config.h.in +++ b/include/clixon_config.h.in @@ -3,6 +3,9 @@ /* Clixon data dir for system yang files etc */ #undef CLIXON_DATADIR +/* Location for apps to find default config file */ +#undef CLIXON_DEFAULT_CONFIG + /* Clixon major release */ #undef CLIXON_VERSION_MAJOR diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index f874cb3d..6c528662 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -202,8 +202,7 @@ clicon_options_main(clicon_handle h) * Set configure file if not set by command-line above */ if (!hash_lookup(copt, "CLICON_CONFIGFILE")){ - clicon_err(OE_CFG, 0, "CLICON_CONFIGFILE (-f) not set"); - goto done; + clicon_option_str_set(h, "CLICON_CONFIGFILE", CLIXON_DEFAULT_CONFIG); } configfile = hash_value(copt, "CLICON_CONFIGFILE", NULL); clicon_debug(1, "CLICON_CONFIGFILE=%s", configfile); From 73074bf7a885dea538600e4d130f65dd51584734 Mon Sep 17 00:00:00 2001 From: Olof Hagsand Date: Mon, 15 Jan 2018 21:36:42 +0000 Subject: [PATCH 09/28] json vector --- apps/restconf/restconf_methods.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 549cda02..eaa88b33 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -294,24 +294,16 @@ api_data_get2(clicon_handle h, if (xpath_vec(xret, path, &xvec, &xlen) < 0) goto done; clicon_debug(1, "%s: xpath:%s xlen:%d", __FUNCTION__, path, xlen); - for (i=0; i Date: Mon, 15 Jan 2018 22:49:44 +0100 Subject: [PATCH 10/28] syntax error --- apps/restconf/restconf_methods.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index eaa88b33..f7aa7c0b 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -304,8 +304,8 @@ api_data_get2(clicon_handle h, else if (xml2json_cbuf_vec(cbx, xvec, xlen, pretty) < 0) goto done; - } } + clicon_debug(1, "%s cbuf:%s", __FUNCTION__, cbuf_get(cbx)); FCGX_FPrintF(r->out, "%s", cbx?cbuf_get(cbx):""); FCGX_FPrintF(r->out, "\r\n\r\n"); From cd333803687e7e21f288dcd9f04844d56452157d Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Tue, 16 Jan 2018 22:25:55 +0100 Subject: [PATCH 11/28] Fixed empty restconf data segv; error restconf quit; etc --- CHANGELOG.md | 7 +++- apps/restconf/restconf_methods.c | 71 ++++++++++++++++++-------------- example/routing_backend.c | 17 ++++++++ lib/src/clixon_json.c | 2 +- lib/src/clixon_xml.c | 4 +- lib/src/clixon_xml_map.c | 1 + test/test_restconf.sh | 31 ++++++++++---- 7 files changed, 90 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7a1eaa..59f45c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,13 @@ * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` * new configuration option: CLICON_RESTCONF_PRETTY -* Changed RESTCONF GET to return object referenced. ie, GET /restconf/data/X returns X. +* Changed RESTCONF GET to return object referenced. ie, GET /restconf/data/X returns X. Thanks Stephen Jones for getting this right. +* Default configure file added by Matt Smith. Config file is selected in the following priority order: + * Provide -f option when starting a program. + * Provide --with-configfile=FILE when configuring + * /etc/clixon.xml + ### Corrected Bugs * Corrected "No yang spec" printed on tty on leafref CLI usage diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index f7aa7c0b..f796d8c3 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -160,7 +160,7 @@ api_data_get_err(clicon_handle h, goto done; if ((xtag = xpath_first(xerr, "/error-tag")) == NULL){ notfound(r); /* bad reply? */ - goto done; + goto ok; } code = restconf_err2code(xml_body(xtag)); if ((reason_phrase = restconf_code2reason(code)) == NULL) @@ -180,6 +180,7 @@ api_data_get_err(clicon_handle h, FCGX_FPrintF(r->out, " %s", cbuf_get(cbj)); FCGX_FPrintF(r->out, " }\r\n"); FCGX_FPrintF(r->out, "}\r\n"); + ok: retval = 0; done: if (cbj) @@ -247,13 +248,13 @@ api_data_get2(clicon_handle h, /* We know "data" is element pi-1 */ if (api_path2xpath_cvv(yspec, pcvec, pi, cbpath) < 0){ notfound(r); - goto done; + goto ok; } path = cbuf_get(cbpath); clicon_debug(1, "%s path:%s", __FUNCTION__, path); if (clicon_rpc_get(h, path, &xret) < 0){ notfound(r); - goto done; + goto ok; } /* We get return via netconf which is complete tree from root * We need to cut that tree to only the object. @@ -431,7 +432,8 @@ api_data_post(clicon_handle h, __FUNCTION__, api_path, data); media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp); - if (strcmp(media_content_type, "application/yang-data+xml")==0) + if (media_content_type && + strcmp(media_content_type, "application/yang-data+xml")==0) parse_xml++; if ((yspec = clicon_dbspec_yang(h)) == NULL){ clicon_err(OE_FATAL, 0, "No DB_SPEC"); @@ -443,18 +445,19 @@ api_data_post(clicon_handle h, if ((xtop = xml_new("config", NULL, NULL)) == NULL) goto done; xbot = xtop; + /* xbot is resulting xml tree on exit */ if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0) goto done; /* Parse input data as json or xml into xml */ if (parse_xml){ if (xml_parse_string(data, NULL, &xdata) < 0){ - clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); - goto done; + badrequest(r); + goto ok; } } else if (json_parse_str(data, &xdata) < 0){ - clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); - goto done; + badrequest(r); + goto ok; } /* Add xdata to xbot */ x = NULL; @@ -477,14 +480,13 @@ api_data_post(clicon_handle h, cbuf_get(cbx)) < 0){ // notfound(r); /* XXX */ conflict(r); - goto done; + goto ok; } if (clicon_rpc_validate(h, "candidate") < 0){ if (clicon_rpc_discard_changes(h) < 0) goto done; badrequest(r); - retval = 0; - goto done; + goto ok; } if (clicon_rpc_commit(h) < 0) goto done; @@ -492,6 +494,7 @@ api_data_post(clicon_handle h, FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); // XXX api_path can be null FCGX_FPrintF(r->out, "Location: %s\r\n", api_path); FCGX_FPrintF(r->out, "\r\n"); + ok: retval = 0; done: clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); @@ -550,7 +553,8 @@ api_data_put(clicon_handle h, __FUNCTION__, api_path, data); media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp); - if (strcmp(media_content_type, "application/yang-data+xml")==0) + if (media_content_type && + strcmp(media_content_type, "application/yang-data+xml")==0) parse_xml++; if ((yspec = clicon_dbspec_yang(h)) == NULL){ clicon_err(OE_FATAL, 0, "No DB_SPEC"); @@ -567,18 +571,17 @@ api_data_put(clicon_handle h, /* Parse input data as json or xml into xml */ if (parse_xml){ if (xml_parse_string(data, NULL, &xdata) < 0){ - clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); - goto done; + badrequest(r); + goto ok; } } else if (json_parse_str(data, &xdata) < 0){ - clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); - goto done; + badrequest(r); + goto ok; } if (xml_child_nr(xdata) != 1){ badrequest(r); - retval = 0; - goto done; + goto ok; } x = xml_child_i(xdata,0); if ((xa = xml_new("operation", x, NULL)) == NULL) @@ -600,21 +603,21 @@ api_data_put(clicon_handle h, OP_NONE, cbuf_get(cbx)) < 0){ notfound(r); - goto done; + goto ok; } if (clicon_rpc_validate(h, "candidate") < 0){ if (clicon_rpc_discard_changes(h) < 0) goto done; badrequest(r); - retval = 0; - goto done; + goto ok; } if (clicon_rpc_commit(h) < 0) goto done; FCGX_SetExitStatus(201, r->out); /* Created */ FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); FCGX_FPrintF(r->out, "\r\n"); + ok: retval = 0; done: clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); @@ -703,13 +706,14 @@ api_data_delete(clicon_handle h, OP_NONE, cbuf_get(cbx)) < 0){ notfound(r); - goto done; + goto ok; } if (clicon_rpc_commit(h) < 0) goto done; FCGX_SetExitStatus(201, r->out); FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); FCGX_FPrintF(r->out, "\r\n"); + ok: retval = 0; done: if (cbx) @@ -762,12 +766,15 @@ api_operation_post(clicon_handle h, int parse_xml = 0; /* By default expect and parse JSON */ char *media_accept; int use_xml = 0; /* By default return JSON */ + int pretty; clicon_debug(1, "%s json:\"%s\" path:\"%s\"", __FUNCTION__, data, path); + pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY"); if ((media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp)) && strcmp(media_accept, "application/yang-data+xml")==0) use_xml++; - if ((media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp)) && + media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp); + if (media_content_type && strcmp(media_content_type, "application/yang-data+xml")==0) parse_xml++; clicon_debug(1, "%s accept:\"%s\" content-type:\"%s\"", @@ -785,7 +792,7 @@ api_operation_post(clicon_handle h, goto done; if (yrpc == NULL){ retval = notfound(r); - goto done; + goto ok; } /* Create an xml message: * <"rpc">... @@ -801,13 +808,13 @@ api_operation_post(clicon_handle h, /* Parse input data as json or xml into xml */ if (parse_xml){ if (xml_parse_string(data, NULL, &xdata) < 0){ - clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); - goto done; + badrequest(r); + goto ok; } } else if (json_parse_str(data, &xdata) < 0){ - clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); - goto done; + badrequest(r); + goto ok; } /* xdata should have format */ if ((xinput = xpath_first(xdata, "/input")) != NULL){ @@ -854,9 +861,10 @@ api_operation_post(clicon_handle h, if ((cbx = cbuf_new()) == NULL) goto done; xoutput=xpath_first(xret, "/"); + xml_name_set(xoutput, "output"); if ((youtput = yang_find((yang_node*)yrpc, Y_OUTPUT, NULL)) != NULL && xoutput){ - xml_name_set(xoutput, "output"); + clicon_debug(1, "%s xoutput:%s", __FUNCTION__, cbuf_get(cbx)); cbuf_reset(cbx); xml_spec_set(xoutput, youtput); /* needed for xml_spec_populate */ @@ -874,16 +882,17 @@ api_operation_post(clicon_handle h, FCGX_FPrintF(r->out, "\r\n"); if (xoutput){ if (use_xml){ - if (clicon_xml2cbuf(cbx, xoutput, 0, 1) < 0) + if (clicon_xml2cbuf(cbx, xoutput, 0, pretty) < 0) goto done; } else - if (xml2json_cbuf(cbx, xoutput, 1) < 0) + if (xml2json_cbuf(cbx, xoutput, pretty) < 0) goto done; clicon_debug(1, "%s xoutput:%s", __FUNCTION__, cbuf_get(cbx)); FCGX_FPrintF(r->out, "%s", cbx?cbuf_get(cbx):""); FCGX_FPrintF(r->out, "\r\n\r\n"); } + ok: retval = 0; done: clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); diff --git a/example/routing_backend.c b/example/routing_backend.c index 449706c5..59c7388b 100644 --- a/example/routing_backend.c +++ b/example/routing_backend.c @@ -134,6 +134,18 @@ fib_route(clicon_handle h, /* Clicon handle */ return 0; } +/*! Smallest possible RPC declaration for test */ +static int +empty(clicon_handle h, /* Clicon handle */ + cxobj *xe, /* Request: */ + struct client_entry *ce, /* Client session */ + cbuf *cbret, /* Reply eg ... */ + void *arg) /* Argument given at register */ +{ + cprintf(cbret, ""); + return 0; +} + /*! IETF Routing route-count rpc */ static int route_count(clicon_handle h, @@ -201,6 +213,11 @@ plugin_init(clicon_handle h) "route-count"/* Xml tag when callback is made */ ) < 0) goto done; + if (backend_rpc_cb_register(h, empty, + NULL, + "empty"/* Xml tag when callback is made */ + ) < 0) + goto done; retval = 0; done: return retval; diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index e261183c..50fd06a4 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -623,7 +623,7 @@ json_parse(char *str, * @param[in] str String containing JSON * @param[out] xt On success a top of XML parse tree is created with name 'top' * @retval 0 OK - * @retval -1 Error with clicon_err called + * @retval -1 Error with clicon_err called. Includes parse errors * * @code * cxobj *cx = NULL; diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index a6021c4b..c412bebf 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -1355,8 +1355,8 @@ xml_parse_file(int fd, * @param[in] str String containing XML definition. * @param[in] yspec Yang specification, or NULL * @param[in,out] xt Pointer to XML parse tree. If empty will be created. - * @retval 0 OK - * @retval -1 Error with clicon_err called + * @retval 0 OK + * @retval -1 Error with clicon_err called. Includes parse error * * @code * cxobj *xt = NULL; diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index b7ea80fc..bb4d6ade 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -1750,6 +1750,7 @@ xml_merge1(cxobj *x0, /*! Merge XML trees x1 into x0 according to yang spec yspec * @note both x0 and x1 need to be top-level trees * @see text_modify_top as more generic variant (in datastore text) + * @note returns -1 if YANG do not match, you may want to have a softer error */ int xml_merge(cxobj *x0, diff --git a/test/test_restconf.sh b/test/test_restconf.sh index 395e2827..1593e633 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -7,11 +7,12 @@ cfg=$dir/conf.xml fyang=$dir/restconf.yang +# example cat < $cfg $cfg /usr/local/share/routing/yang - example + $fyang /usr/local/lib/routing/clispec /usr/local/lib/routing/backend /usr/local/lib/routing/restconf @@ -28,6 +29,7 @@ EOF cat < $fyang module example{ + prefix ex; import ietf-ip { prefix ip; } @@ -38,6 +40,16 @@ module example{ prefix "inet"; revision-date "2013-07-15"; } + rpc empty { + } + rpc input { + input { + } + } + rpc output { + output { + } + } } EOF @@ -60,7 +72,7 @@ new "kill old restconf daemon" sudo pkill -u www-data clixon_restconf new "start restconf daemon" -sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg # -D +sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg -D sleep 1 @@ -76,13 +88,15 @@ expectfn "curl -sS -I http://localhost/restconf/data" "HTTP/1.1 200 OK" new "restconf root discovery" expectfn "curl -sS -X GET http://localhost/.well-known/host-meta" "" +new "restconf empty rpc" +expectfn 'curl -sS -X POST -d {"input":{"name":""}} http://localhost/restconf/operations/ex:empty' '{"output": null}' + #new "restconf get restconf json XXX" #expectfn "curl -sSG http://localhost/restconf" "{\"restconf\" : $state }" #new "restconf get restconf/yang-library-version json XXX" #expectfn "curl -sSG http://localhost/restconf/yang-library-version" "{\"restconf\" : $state }" - new "restconf get empty config + state json" expectfn "curl -sSG http://localhost/restconf/data" "{\"data\": $state}" @@ -145,6 +159,9 @@ expectfn 'curl -sS -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabl new "Add leaf description using POST" expectfn 'curl -sS -X POST -d {"description":"The-first-interface"} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" +new "Add nothing using POST" +expectfn 'curl -sS -X POST http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "data is in some way badly formed" + new "restconf Check description added" expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": "true"}} $' @@ -165,15 +182,13 @@ new "restconf get subtree" expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} $' -new "restconf operation rpc using POST json" -expectfn 'curl -sS -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{ "output": { "route": { "address-family": "ipv4", "next-hop": { "next-hop-list": "2.3.4.5" } } } } ' - -exit # XXX +new "restconf rpc using POST json" +expectfn 'curl -sS -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{"output": {"route": {"address-family": "ipv4","next-hop": {"next-hop-list": "2.3.4.5"}}}}' new "restconf rpc using POST xml" # Cant get shell macros to work, inline matching from lib.sh ret=$(curl -sS -X POST -H "Accept: application/yang-data+xml" -d '{"input":{"routing-instance-name":"ipv4"}}' http://localhost/restconf/operations/rt:fib-route) -expect=" ipv4 2.3.4.5 " +expect="ipv42.3.4.5" match=`echo $ret | grep -EZo "$expect"` if [ -z "$match" ]; then err "$expect" "$ret" From d010937ba6599fc1f320b5b917bed5baf3a6c72f Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Tue, 16 Jan 2018 15:35:43 -0600 Subject: [PATCH 12/28] Augment help text for --with-configfile Include '=FILE' at the end to make correct usage clearer --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 0f897b4e..a3b0392f 100755 --- a/configure +++ b/configure @@ -1348,7 +1348,7 @@ Optional Packages: --without-restconf disable support for restconf --with-keyvalue enable support for key-value xmldb datastore --with-qdbm=dir Use QDBM here, if keyvalue - --with-configfile set default path to config file + --with-configfile=FILE set default path to config file Some influential environment variables: CC C compiler command diff --git a/configure.ac b/configure.ac index 73c6f07b..941f8e3d 100644 --- a/configure.ac +++ b/configure.ac @@ -163,7 +163,7 @@ fi # Set default config file location AC_ARG_WITH([configfile], - [AS_HELP_STRING([--with-configfile],[set default path to config file])], + [AS_HELP_STRING([--with-configfile=FILE],[set default path to config file])], [DEFAULT_CONFIG="$withval"], [DEFAULT_CONFIG="/etc/clixon.xml"]) From f6284ac933d07c46f2a9960403a819e73f867ca6 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Wed, 17 Jan 2018 22:59:54 +0100 Subject: [PATCH 13/28] restconf PUT delta --- apps/restconf/restconf_methods.c | 8 ++++---- lib/src/clixon_xml.c | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index f796d8c3..48313a9f 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -587,12 +587,13 @@ api_data_put(clicon_handle h, if ((xa = xml_new("operation", x, NULL)) == NULL) goto done; xml_type_set(xa, CX_ATTR); - if (xml_value_set(xa, xml_operation2str(op)) < 0) + if (xml_value_set(xa, xml_operation2str(op)) < 0) goto done; - /* Replace xbot with x */ + /* XXX Special case path=/restconf/data xml_name(x) == data */ + /* Replace xbot with x */ xp = xml_parent(xbot); xml_purge(xbot); - if (xml_addsub(xp, x) < 0) + if (xml_addsub(xp, x) < 0) goto done; if ((cbx = cbuf_new()) == NULL) goto done; @@ -605,7 +606,6 @@ api_data_put(clicon_handle h, notfound(r); goto ok; } - if (clicon_rpc_validate(h, "candidate") < 0){ if (clicon_rpc_discard_changes(h) < 0) goto done; diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index c412bebf..635498cc 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -595,7 +595,7 @@ xml_find(cxobj *x_up, } /*! Append xc as child to xp. Remove xc from previous parent. - * @param[in] xp Parent xml node + * @param[in] xp Parent xml node. If NULL just remove from old parent. * @param[in] xc Child xml node to insert under xp * @retval 0 OK * @retval -1 Error @@ -618,10 +618,12 @@ xml_addsub(cxobj *xp, xml_child_rm(oldp, i); } /* Add xc to new parent */ - if (xml_child_append(xp, xc) < 0) - return -1; - /* Set new parent in child */ - xml_parent_set(xc, xp); + if (xp){ + if (xml_child_append(xp, xc) < 0) + return -1; + /* Set new parent in child */ + xml_parent_set(xc, xp); + } return 0; } From ad4127541e9d0fceff5ebeb2e50ee0db65459003 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 20 Jan 2018 12:34:52 +0100 Subject: [PATCH 14/28] explaining test --- test/test_perf.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_perf.sh b/test/test_perf.sh index a91e6bb4..8e35feb6 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -73,7 +73,7 @@ if [ $? -ne 0 ]; then err fi -new "generate large list config" +new "generate 'large' config with $number list entries" echo -n "" > $fconfig for (( i=0; i<$number; i++ )); do echo -n "$i$i" >> $fconfig @@ -83,12 +83,12 @@ echo "]]>]]>" >> $fconfig # Just for manual dbg echo "$clixon_netconf -qf $cfg -y $fyang" -new "netconf edit large config" +new "netconf write large config" expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" #echo ']]>]]>' | $clixon_netconf -qf $cfg -y $fyang -new "netconf edit large config again" +new "netconf write large config again" expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" #echo ']]>]]>' | $clixon_netconf -qf $cfg -y $fyang @@ -98,10 +98,10 @@ rm $fconfig new "netconf commit large config" expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" -new "netconf commit same config again" +new "netconf commit large config again" expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" -new "netconf add one small config" +new "netconf add small (1 entry) config" expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "xy]]>]]>" "^]]>]]>$" new "netconf add $req small config" From 1ee3f7e67ea0387917bec3b9d7af4e32813d72d9 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 20 Jan 2018 16:19:56 +0100 Subject: [PATCH 15/28] Gnuplot script for transactions per second --- test/plot_perf.sh | 126 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100755 test/plot_perf.sh diff --git a/test/plot_perf.sh b/test/plot_perf.sh new file mode 100755 index 00000000..04b48f87 --- /dev/null +++ b/test/plot_perf.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# Transactions per second for large lists read/write plotter using gnuplot +# +. ./lib.sh +max=1000 # Nr of db entries +step=100 +reqs=1000 +cfg=$dir/scaling-conf.xml +fyang=$dir/scaling.yang +fconfig=$dir/config + +# For memcheck +# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" +# clixon_netconf="valgrind --tool=callgrind clixon_netconf +clixon_netconf=clixon_netconf + +cat < $fyang +module ietf-ip{ + container x { + list y { + key "a"; + leaf a { + type string; + } + leaf b { + type string; + } + } + leaf-list c { + type string; + } + } +} +EOF + +cat < $cfg + + $cfg + $fyang + ietf-ip + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + +run(){ + nr=$1 # Number of entries in DB + reqs=$2 + write=$3 + + echo -n "" > $fconfig + for (( i=0; i<$nr; i++ )); do + echo -n "$i$i" >> $fconfig + done + echo "]]>]]>" >> $fconfig + + expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" + + if $write; then + time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "$rnd$rnd]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + else # read + time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + fi +} + +once()( + # kill old backend (if any) + sudo clixon_backend -zf $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi + + # start new backend + sudo clixon_backend -s init -f $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi + + # Actual steps + for (( i=$step; i<=$max; i=i+$step )); do + t=$(TEST=%e run $i $reqs true $ 2>&1 | awk '/real/ {print $2}') + # t is time in secs of $reqs -> transactions per second. $reqs + p=$(echo "$reqs/$t" | bc -lq) + # p is transactions per second. + echo "$i $p" >> $dir/write + t=$(TEST=%e run $i $reqs false $ 2>&1 | awk '/real/ {print $2}') + # t is time in secs of $reqs -> transactions per second. $reqs + p=$(echo "$reqs/$t" | bc -lq) + # p is transactions per second. + echo "$i $p" >> $dir/read + done + + # Check if still alive + pid=`pgrep clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err "kill backend" + fi + +) + +once + +gnuplot -persist < Date: Sun, 21 Jan 2018 14:31:53 +0100 Subject: [PATCH 16/28] Restconf: get well-known, top-level resource, yang library version, put whole datastore, check for different keys in put lists. --- CHANGELOG.md | 4 +- apps/restconf/README.md | 15 +- apps/restconf/restconf_main.c | 179 ++++++++++++++++-- apps/restconf/restconf_methods.c | 167 +++++++++++++---- apps/restconf/restconf_methods.h | 4 + datastore/text/clixon_xmldb_text.c | 9 +- lib/src/clixon_xml_map.c | 6 +- test/plot_perf.sh | 71 +++++--- test/test_restconf.sh | 12 +- test/test_restconf2.sh | 141 ++++++++++++++ yang/ietf-yang-library@2016-06-21.yang | 242 +++++++++++++++++++++++++ 11 files changed, 759 insertions(+), 91 deletions(-) create mode 100755 test/test_restconf2.sh create mode 100644 yang/ietf-yang-library@2016-06-21.yang diff --git a/CHANGELOG.md b/CHANGELOG.md index 59f45c49..9e57114f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,15 @@ ### Major changes: ### Minor changes: + * The following backward compatible options to configure have been obsoleted. If you havent already migrated this code you must do this now. * Backend startup modes prior to 3.3.3. As enabled with `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted. * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` * new configuration option: CLICON_RESTCONF_PRETTY -* Changed RESTCONF GET to return object referenced. ie, GET /restconf/data/X returns X. Thanks Stephen Jones for getting this right. +* Changed restconf GET to return object referenced. ie, GET /restconf/data/X returns X. Thanks Stephen Jones for getting this right. +* Restconf: get well-known, top-level resource, yang library version, put whole datastore, check for different keys in put lists. * Default configure file added by Matt Smith. Config file is selected in the following priority order: * Provide -f option when starting a program. diff --git a/apps/restconf/README.md b/apps/restconf/README.md index a7952a9c..fdb1da37 100644 --- a/apps/restconf/README.md +++ b/apps/restconf/README.md @@ -2,16 +2,15 @@ ### Features Clixon restconf is a daemon based on FASTCGI. Instructions are available to -run with NGINX. -The implementatation supports plain OPTIONS, HEAD, GET, POST, PUT, PATCH, DELETE. -and is based on draft-ietf-netconf-restconf-13. -There is currently (2017) a [RFC 8040: RESTCONF Protocol](https://tools.ietf.org/html/rfc8040), many of those features are _not_ implemented, -including: +run with NGINX. +The implementatation is based on [RFC 8040: RESTCONF Protocol](https://tools.ietf.org/html/rfc8040). +The following featires are supported: +- OPTIONS, HEAD, GET, POST, PUT, DELETE +The following are not implemented +- PATCH - query parameters (section 4.9) - notifications (sec 6) -- GET /restconf/ (sec 3.3) -- GET /restconf/yang-library-version (sec 3.3.3) -- only rudimentary error reporting exists (sec 7) +- schema resource ### Installation using Nginx diff --git a/apps/restconf/restconf_main.c b/apps/restconf/restconf_main.c index f400edf0..b386616b 100644 --- a/apps/restconf/restconf_main.c +++ b/apps/restconf/restconf_main.c @@ -73,9 +73,14 @@ /* Command line options to be passed to getopt(3) */ #define RESTCONF_OPTS "hDf:p:y:" -/* Should be discovered via "/.well-known/host-meta" - resource ([RFC6415]) */ -#define RESTCONF_API_ROOT "/restconf/" +/* RESTCONF enables deployments to specify where the RESTCONF API is + located. The client discovers this by getting the "/.well-known/host-meta" + resource +*/ +#define RESTCONF_WELL_KNOWN "/.well-known/host-meta" + +#define RESTCONF_API "restconf" +#define RESTCONF_API_ROOT "/restconf" /*! Generic REST method, GET, PUT, DELETE, etc * @param[in] h CLIXON handle @@ -117,6 +122,7 @@ api_data(clicon_handle h, retval = api_data_delete(h, r, api_path, pi); else retval = notfound(r); + clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); return retval; } @@ -144,18 +150,118 @@ api_operations(clicon_handle h, clicon_debug(1, "%s", __FUNCTION__); request_method = FCGX_GetParam("REQUEST_METHOD", r->envp); clicon_debug(1, "%s method:%s", __FUNCTION__, request_method); - if (strcmp(request_method, "POST")==0) + if (strcmp(request_method, "GET")==0) + retval = api_operation_get(h, r, path, pcvec, pi, qvec, data); + else if (strcmp(request_method, "POST")==0) retval = api_operation_post(h, r, path, pcvec, pi, qvec, data); else retval = notfound(r); return retval; } +/*! Retrieve the Top-Level API Resource + * @note Only returns null for operations and data,... + */ +static int +api_root(clicon_handle h, + FCGX_Request *r) +{ + int retval = -1; + char *media_accept; + int use_xml = 0; /* By default use JSON */ + cxobj *xt = NULL; + cbuf *cb = NULL; + int pretty; + + clicon_debug(1, "%s", __FUNCTION__); + pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY"); + media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp); + if (strcmp(media_accept, "application/yang-data+xml")==0) + use_xml++; + FCGX_SetExitStatus(200, r->out); /* OK */ + FCGX_FPrintF(r->out, "Content-Type: application/yang-data+%s\r\n", use_xml?"xml":"json"); + FCGX_FPrintF(r->out, "\r\n"); + if (xml_parse_string("2016-06-21", NULL, &xt) < 0) + goto done; + if ((cb = cbuf_new()) == NULL){ + clicon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + if (xml_rootchild(xt, 0, &xt) < 0) + goto done; + if (use_xml){ + if (clicon_xml2cbuf(cb, xt, 0, pretty) < 0) + goto done; + } + else + if (xml2json_cbuf(cb, xt, pretty) < 0) + goto done; + FCGX_FPrintF(r->out, "%s", cb?cbuf_get(cb):""); + FCGX_FPrintF(r->out, "\r\n\r\n"); + retval = 0; + done: + if (cb) + cbuf_free(cb); + if (xt) + xml_free(xt); + return retval; +} + +/*! + * See https://tools.ietf.org/html/rfc7895 + */ +static int +api_yang_library_version(clicon_handle h, + FCGX_Request *r) +{ + int retval = -1; + char *media_accept; + int use_xml = 0; /* By default use JSON */ + cxobj *xt = NULL; + cbuf *cb = NULL; + int pretty; + + clicon_debug(1, "%s", __FUNCTION__); + pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY"); + media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp); + if (strcmp(media_accept, "application/yang-data+xml")==0) + use_xml++; + FCGX_SetExitStatus(200, r->out); /* OK */ + FCGX_FPrintF(r->out, "Content-Type: application/yang-data+%s\r\n", use_xml?"xml":"json"); + FCGX_FPrintF(r->out, "\r\n"); + if (xml_parse_string("2016-06-21", NULL, &xt) < 0) + goto done; + if (xml_rootchild(xt, 0, &xt) < 0) + goto done; + if ((cb = cbuf_new()) == NULL){ + clicon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + if (use_xml){ + if (clicon_xml2cbuf(cb, xt, 0, pretty) < 0) + goto done; + } + else{ + if (xml2json_cbuf(cb, xt, pretty) < 0) + goto done; + } + clicon_debug(1, "%s cb%s", __FUNCTION__, cbuf_get(cb)); + FCGX_FPrintF(r->out, "%s\r\n", cb?cbuf_get(cb):""); + FCGX_FPrintF(r->out, "\r\n\r\n"); + retval = 0; + done: + if (cb) + cbuf_free(cb); + if (xt) + xml_free(xt); + return retval; +} + /*! Process a FastCGI request * @param[in] r Fastcgi request handle */ static int -request_process(clicon_handle h, +api_restconf(clicon_handle h, FCGX_Request *r) { int retval = -1; @@ -176,7 +282,28 @@ request_process(clicon_handle h, query = FCGX_GetParam("QUERY_STRING", r->envp); if ((pvec = clicon_strsep(path, "/", &pn)) == NULL) goto done; - + /* Sanity check of path. Should be /restconf/ */ + if (pn < 2){ + retval = notfound(r); + goto done; + } + if (strlen(pvec[0]) != 0){ + retval = notfound(r); + goto done; + } + if (strcmp(pvec[1], RESTCONF_API)){ + retval = notfound(r); + goto done; + } + if (pn == 2){ + retval = api_root(h, r); + goto done; + } + if ((method = pvec[2]) == NULL){ + retval = notfound(r); + goto done; + } + clicon_debug(1, "method=%s", method); if (str2cvec(query, '&', '=', &qvec) < 0) goto done; if (str2cvec(path, '/', '=', &pcvec) < 0) /* rest url eg /album=ricky/foo */ @@ -188,10 +315,7 @@ request_process(clicon_handle h, clicon_debug(1, "DATA=%s", data); if (str2cvec(data, '&', '=', &dvec) < 0) goto done; - if ((method = pvec[2]) == NULL){ - retval = notfound(r); - goto done; - } + retval = 0; test(r, 1); /* If present, check credentials */ @@ -202,8 +326,9 @@ request_process(clicon_handle h, if (auth == 0) goto done; clicon_debug(1, "%s credentials ok 2", __FUNCTION__); - - if (strcmp(method, "data") == 0) /* restconf, skip /api/data */ + if (strcmp(method, "yang-library-version")==0) + retval = api_yang_library_version(h, r); + else if (strcmp(method, "data") == 0) /* restconf, skip /api/data */ retval = api_data(h, r, path, pcvec, 2, qvec, data); else if (strcmp(method, "operations") == 0) /* rpc */ retval = api_operations(h, r, path, pcvec, 2, qvec, data); @@ -226,6 +351,24 @@ request_process(clicon_handle h, return retval; } +/*! Process a FastCGI request + * @param[in] r Fastcgi request handle + */ +static int +api_well_known(clicon_handle h, + FCGX_Request *r) +{ + clicon_debug(1, "%s", __FUNCTION__); + FCGX_FPrintF(r->out, "Content-Type: application/xrd+xml\r\n"); + FCGX_FPrintF(r->out, "\r\n"); + FCGX_SetExitStatus(200, r->out); /* OK */ + FCGX_FPrintF(r->out, "\r\n"); + FCGX_FPrintF(r->out, " \r\n"); + FCGX_FPrintF(r->out, "\r\n"); + + return 0; +} + static int restconf_terminate(clicon_handle h) { @@ -384,13 +527,17 @@ main(int argc, } clicon_debug(1, "------------"); if ((path = FCGX_GetParam("REQUEST_URI", r->envp)) != NULL){ - if (strncmp(path, RESTCONF_API_ROOT, strlen(RESTCONF_API_ROOT)) == 0 || - strncmp(path, RESTCONF_API_ROOT, strlen(RESTCONF_API_ROOT)-1) == 0) - request_process(h, r); /* This is the function */ + clicon_debug(1, "path:%s", path); + if (strncmp(path, RESTCONF_API_ROOT, strlen(RESTCONF_API_ROOT)) == 0) + api_restconf(h, r); /* This is the function */ + else if (strncmp(path, RESTCONF_WELL_KNOWN, strlen(RESTCONF_WELL_KNOWN)) == 0) { + api_well_known(h, r); /* This is the function */ + } else{ - clicon_debug(1, "top-level not found"); + clicon_debug(1, "top-level %s not found", path); notfound(r); } + } else clicon_debug(1, "NULL URI"); diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 48313a9f..859cf210 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -379,7 +379,7 @@ api_data_get(clicon_handle h, return api_data_get2(h, r, pcvec, pi, qvec, 0); } -/*! REST POST method +/*! Generic REST POST method * @param[in] h CLIXON handle * @param[in] r Fastcgi request handle * @param[in] api_path According to restconf (Sec 3.5.3.1 in rfc8040) @@ -387,7 +387,7 @@ api_data_get(clicon_handle h, * @param[in] pi Offset, where to start pcvec * @param[in] qvec Vector of query string (QUERY_STRING) * @param[in] data Stream input data - * @note We map post to edit-config create. + * @note restconf POST is mapped to edit-config create. POST: target resource type is datastore --> create a top-level resource target resource type is data resource --> create child resource @@ -414,12 +414,12 @@ api_data_post(clicon_handle h, cvec *qvec, char *data) { - enum operation_type op = OP_CREATE; int retval = -1; + enum operation_type op = OP_CREATE; int i; cxobj *xdata = NULL; - cxobj *xtop = NULL; /* xpath root */ cbuf *cbx = NULL; + cxobj *xtop = NULL; /* xpath root */ cxobj *xbot = NULL; cxobj *x; yang_node *y = NULL; @@ -444,8 +444,8 @@ api_data_post(clicon_handle h, /* Create config top-of-tree */ if ((xtop = xml_new("config", NULL, NULL)) == NULL) goto done; + /* Translate api_path to xtop/xbot */ xbot = xtop; - /* xbot is resulting xml tree on exit */ if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0) goto done; /* Parse input data as json or xml into xml */ @@ -456,29 +456,35 @@ api_data_post(clicon_handle h, } } else if (json_parse_str(data, &xdata) < 0){ - badrequest(r); - goto ok; + badrequest(r); + goto ok; } - /* Add xdata to xbot */ - x = NULL; - while ((x = xml_child_each(xdata, x, CX_ELMNT)) != NULL) { - if ((xa = xml_new("operation", x, NULL)) == NULL) - goto done; - xml_type_set(xa, CX_ATTR); - if (xml_value_set(xa, xml_operation2str(op)) < 0) - goto done; - if (xml_addsub(xbot, x) < 0) - goto done; + /* The message-body MUST contain exactly one instance of the + * expected data resource. + */ + if (xml_child_nr(xdata) != 1){ + badrequest(r); + goto ok; } + x = xml_child_i(xdata,0); + /* Add operation (create/replace) as attribute */ + if ((xa = xml_new("operation", x, NULL)) == NULL) + goto done; + xml_type_set(xa, CX_ATTR); + if (xml_value_set(xa, xml_operation2str(op)) < 0) + goto done; + /* Replace xbot with x, ie bottom of api-path with data */ + if (xml_addsub(xbot, x) < 0) + goto done; + /* Create text buffer for transfer to backend */ if ((cbx = cbuf_new()) == NULL) goto done; if (clicon_xml2cbuf(cbx, xtop, 0, 0) < 0) goto done; - clicon_debug(1, "%s xml: %s",__FUNCTION__, cbuf_get(cbx)); + clicon_debug(1, "%s xml: %s api_path:%s",__FUNCTION__, cbuf_get(cbx), api_path); if (clicon_rpc_edit_config(h, "candidate", OP_NONE, cbuf_get(cbx)) < 0){ - // notfound(r); /* XXX */ conflict(r); goto ok; } @@ -492,7 +498,6 @@ api_data_post(clicon_handle h, goto done; FCGX_SetExitStatus(201, r->out); /* Created */ FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); - // XXX api_path can be null FCGX_FPrintF(r->out, "Location: %s\r\n", api_path); FCGX_FPrintF(r->out, "\r\n"); ok: retval = 0; @@ -505,6 +510,58 @@ api_data_post(clicon_handle h, if (cbx) cbuf_free(cbx); return retval; +} /* api_data_post */ + + +/*! Check matching keys + * + * @param[in] y Yang statement, should be list or leaf-list + * @param[in] xdata XML data tree + * @param[in] xapipath XML api-path tree + * @retval 0 Yes, keys match + * @retval -1 No keys do not match + * If the target resource represents a YANG leaf-list, then the PUT + * method MUST NOT change the value of the leaf-list instance. + * + * If the target resource represents a YANG list instance, then the key + * leaf values, in message-body representation, MUST be the same as the + * key leaf values in the request URI. The PUT method MUST NOT be used + * to change the key leaf values for a data resource instance. + */ +static int +match_list_keys(yang_stmt *y, + cxobj *xdata, + cxobj *xapipath) +{ + int retval = -1; + cvec *cvk = NULL; /* vector of index keys */ + cg_var *cvi; + char *keyname; + cxobj *xkeya; /* xml key object in api-path */ + cxobj *xkeyd; /* xml key object in data */ + char *keya; + char *keyd; + + if (y->ys_keyword != Y_LIST &&y->ys_keyword != Y_LEAF_LIST) + return -1; + cvk = y->ys_cvec; /* Use Y_LIST cache, see ys_populate_list() */ + cvi = NULL; + while ((cvi = cvec_each(cvk, cvi)) != NULL) { + keyname = cv_string_get(cvi); + if ((xkeya = xml_find(xapipath, keyname)) == NULL) + goto done; /* No key in api-path */ + + keya = xml_body(xkeya); + if ((xkeyd = xml_find(xdata, keyname)) == NULL) + goto done; /* No key in data */ + keyd = xml_body(xkeyd); + if (strcmp(keya, keyd) != 0) + goto done; /* keys dont match */ + } + retval = 0; + done: + clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); + return retval; } /*! Generic REST PUT method @@ -515,6 +572,7 @@ api_data_post(clicon_handle h, * @param[in] pi Offset, where to start pcvec * @param[in] qvec Vector of query string (QUERY_STRING) * @param[in] data Stream input data + * @note restconf PUT is mapped to edit-config replace. * @example curl -X PUT -d '{"enabled":"false"}' http://127.0.0.1/restconf/data/interfaces/interface=eth1 * @@ -528,7 +586,7 @@ api_data_post(clicon_handle h, int api_data_put(clicon_handle h, FCGX_Request *r, - char *api_path, + char *api_path0, cvec *pcvec, int pi, cvec *qvec, @@ -539,19 +597,19 @@ api_data_put(clicon_handle h, int i; cxobj *xdata = NULL; cbuf *cbx = NULL; - cxobj *x; + cxobj *xtop = NULL; /* xpath root */ cxobj *xbot = NULL; - cxobj *xtop = NULL; - cxobj *xp; + cxobj *xparent; + cxobj *x; yang_node *y = NULL; yang_spec *yspec; cxobj *xa; char *media_content_type; int parse_xml = 0; /* By default expect and parse JSON */ + char *api_path; clicon_debug(1, "%s api_path:\"%s\" json:\"%s\"", - __FUNCTION__, - api_path, data); + __FUNCTION__, api_path0, data); media_content_type = FCGX_GetParam("HTTP_CONTENT_TYPE", r->envp); if (media_content_type && strcmp(media_content_type, "application/yang-data+xml")==0) @@ -560,11 +618,13 @@ api_data_put(clicon_handle h, clicon_err(OE_FATAL, 0, "No DB_SPEC"); goto done; } + api_path=api_path0; for (i=0; iyn_keyword == Y_LIST ||y->yn_keyword == Y_LEAF_LIST)){ + if (match_list_keys((yang_stmt*)y, x, xbot) < 0){ + badrequest(r); + goto ok; + } + } + xparent = xml_parent(xbot); + xml_purge(xbot); + if (xml_addsub(xparent, x) < 0) + goto done; + } +#endif + /* Create text buffer for transfer to backend */ if ((cbx = cbuf_new()) == NULL) goto done; if (clicon_xml2cbuf(cbx, xtop, 0, 0) < 0) @@ -628,8 +714,7 @@ api_data_put(clicon_handle h, if (cbx) cbuf_free(cbx); return retval; - -} +} /* api_data_put */ /*! Generic REST PATCH method * @param[in] h CLIXON handle @@ -724,6 +809,20 @@ api_data_delete(clicon_handle h, return retval; } +/*! NYI + */ +int +api_operation_get(clicon_handle h, + FCGX_Request *r, + char *path, + cvec *pcvec, + int pi, + cvec *qvec, + char *data) +{ + return 0; +} + /*! REST operation POST method * @param[in] h CLIXON handle * @param[in] r Fastcgi request handle diff --git a/apps/restconf/restconf_methods.h b/apps/restconf/restconf_methods.h index 89ece997..0aef4428 100644 --- a/apps/restconf/restconf_methods.h +++ b/apps/restconf/restconf_methods.h @@ -60,6 +60,10 @@ int api_data_patch(clicon_handle h, FCGX_Request *r, char *api_path, cvec *qvec, char *data); int api_data_delete(clicon_handle h, FCGX_Request *r, char *api_path, int pi); +int api_operation_get(clicon_handle h, FCGX_Request *r, + char *path, + cvec *pcvec, int pi, cvec *qvec, char *data); + int api_operation_post(clicon_handle h, FCGX_Request *r, char *path, cvec *pcvec, int pi, cvec *qvec, char *data); diff --git a/datastore/text/clixon_xmldb_text.c b/datastore/text/clixon_xmldb_text.c index 9eae988e..739e2256 100644 --- a/datastore/text/clixon_xmldb_text.c +++ b/datastore/text/clixon_xmldb_text.c @@ -748,14 +748,19 @@ text_modify_top(cxobj *x0, cxobj *x0c; /* base child */ cxobj *x1c; /* mod child */ yang_stmt *yc; /* yang child */ + char *opstr; /* Assure top-levels are 'config' */ assert(x0 && strcmp(xml_name(x0),"config")==0); assert(x1 && strcmp(xml_name(x1),"config")==0); + /* Check for operations embedded in tree according to netconf */ + if ((opstr = xml_find_value(x1, "operation")) != NULL) + if (xml_operation(opstr, &op) < 0) + goto done; /* Special case if x1 is empty, top-level only */ - if (!xml_child_nr(x1)){ /* base tree not empty */ - if (xml_child_nr(x0)) + if (!xml_child_nr(x1)){ + if (xml_child_nr(x0)) /* base tree not empty */ switch(op){ case OP_DELETE: case OP_REMOVE: diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index bb4d6ade..bd7c1dbd 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -1654,9 +1654,9 @@ api_path2xml(char *api_path, if (nvec > 1 && !strlen(vec[nvec-1])) nvec--; if (nvec < 1){ - clicon_err(OE_XML, 0, "Malformed key: %s", api_path); - goto done; - } + clicon_err(OE_XML, 0, "Malformed key: %s", api_path); + goto done; + } nvec--; /* NULL-terminated */ if (api_path2xml_vec(vec+1, nvec, xpath, (yang_node*)yspec, schemanode, diff --git a/test/plot_perf.sh b/test/plot_perf.sh index 04b48f87..db73f7cc 100755 --- a/test/plot_perf.sh +++ b/test/plot_perf.sh @@ -2,7 +2,7 @@ # Transactions per second for large lists read/write plotter using gnuplot # . ./lib.sh -max=1000 # Nr of db entries +max=200 # Nr of db entries step=100 reqs=1000 cfg=$dir/scaling-conf.xml @@ -48,27 +48,55 @@ EOF run(){ nr=$1 # Number of entries in DB reqs=$2 - write=$3 + mode=$3 - echo -n "" > $fconfig + echo -n "replace" > $fconfig for (( i=0; i<$nr; i++ )); do + echo -n "$i" >> $fconfig echo -n "$i$i" >> $fconfig done echo "]]>]]>" >> $fconfig expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" - if $write; then - time -p for (( i=0; i<$reqs; i++ )); do - rnd=$(( ( RANDOM % $nr ) )) - echo "$rnd$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null - else # read + case $mode in + readlist) time -p for (( i=0; i<$reqs; i++ )); do rnd=$(( ( RANDOM % $nr ) )) echo "]]>]]>" done | $clixon_netconf -qf $cfg -y $fyang > /dev/null - fi + ;; + writelist) + time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "$rnd$rnd]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + ;; + readleaflist) + time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + ;; + writeleaflist) + time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "$rnd]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + ;; + esac + expecteof "$clixon_netconf -qf $cfg" "]]>]]>" "^]]>]]>$" +} + +step(){ + i=$1 + mode=$2 + echo -n "" > $fconfig + t=$(TEST=%e run $i $reqs $mode $ 2>&1 | awk '/real/ {print $2}') + # t is time in secs of $reqs -> transactions per second. $reqs + p=$(echo "$reqs/$t" | bc -lq) + # p is transactions per second. + echo "$i $p" >> $dir/$mode } once()( @@ -84,18 +112,19 @@ once()( err fi + # Always as a start + for (( i=10; i<=$step; i=i+10 )); do + step $i readlist + step $i writelist + step $i readleaflist + step $i writeleaflist + done # Actual steps for (( i=$step; i<=$max; i=i+$step )); do - t=$(TEST=%e run $i $reqs true $ 2>&1 | awk '/real/ {print $2}') - # t is time in secs of $reqs -> transactions per second. $reqs - p=$(echo "$reqs/$t" | bc -lq) - # p is transactions per second. - echo "$i $p" >> $dir/write - t=$(TEST=%e run $i $reqs false $ 2>&1 | awk '/real/ {print $2}') - # t is time in secs of $reqs -> transactions per second. $reqs - p=$(echo "$reqs/$t" | bc -lq) - # p is transactions per second. - echo "$i $p" >> $dir/read + step $i readlist + step $i readleaflist + step $i writelist + step $i writeleaflist done # Check if still alive @@ -118,7 +147,7 @@ set title "Clixon transactions per second r/w large lists" font ",14" textcolor set xlabel "entries" set ylabel "transactions per second" set terminal wxt enhanced title "CLixon transactions " persist raise -plot "$dir/read" with linespoints title "read", "$dir/write" with linespoints title "write" +plot "$dir/readlist" with linespoints title "read list", "$dir/writelist" with linespoints title "write list", "$dir/readleaflist" with linespoints title "read leaf-list", "$dir/writeleaflist" with linespoints title "write leaf-list" EOF rm -rf $dir diff --git a/test/test_restconf.sh b/test/test_restconf.sh index 1593e633..8cc700d6 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -88,15 +88,15 @@ expectfn "curl -sS -I http://localhost/restconf/data" "HTTP/1.1 200 OK" new "restconf root discovery" expectfn "curl -sS -X GET http://localhost/.well-known/host-meta" "" +new "restconf get restconf json" +expectfn "curl -sSG http://localhost/restconf" '{"data": null,"operations": null,"yang-library-version": "2016-06-21"}}' + +new "restconf get restconf/yang-library-version json" +expectfn "curl -sSG http://localhost/restconf/yang-library-version" '{"yang-library-version": "2016-06-21"}' + new "restconf empty rpc" expectfn 'curl -sS -X POST -d {"input":{"name":""}} http://localhost/restconf/operations/ex:empty' '{"output": null}' -#new "restconf get restconf json XXX" -#expectfn "curl -sSG http://localhost/restconf" "{\"restconf\" : $state }" - -#new "restconf get restconf/yang-library-version json XXX" -#expectfn "curl -sSG http://localhost/restconf/yang-library-version" "{\"restconf\" : $state }" - new "restconf get empty config + state json" expectfn "curl -sSG http://localhost/restconf/data" "{\"data\": $state}" diff --git a/test/test_restconf2.sh b/test/test_restconf2.sh new file mode 100755 index 00000000..1419f41b --- /dev/null +++ b/test/test_restconf2.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Restconf basic functionality +# Assume http server setup, such as nginx described in apps/restconf/README.md + +# include err() and new() functions and creates $dir +. ./lib.sh +cfg=$dir/conf.xml +fyang=$dir/restconf.yang + +# example +cat < $cfg + + $cfg + /usr/local/var + $fyang + + false + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + 1 + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + +cat < $fyang +module example{ + container interfaces-config{ + list interface{ + key name; + leaf name{ + type string; + } + leaf type{ + type string; + } + leaf description{ + type string; + } + leaf netgate-if-type{ + type string; + } + leaf enabled{ + type boolean; + } + } + } +} +EOF + +# kill old backend (if any) +new "kill old backend" +sudo clixon_backend -zf $cfg +if [ $? -ne 0 ]; then + err +fi +new "start backend -s init -f $cfg -y $fyang" +sudo clixon_backend -s init -f $cfg -y $fyang +if [ $? -ne 0 ]; then + err +fi + +new "kill old restconf daemon" +sudo pkill -u www-data clixon_restconf + +new "start restconf daemon" +sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg + +sleep 1 + +new "restconf tests" + +new "restconf PUT change key error" +#expectfn 'curl -s -X PUT -d {"interface":{"name":"ALPHA","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "fail" +#exit + +new "restconf POST initial tree" +expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","netgate-if-type":"regular"}}} http://localhost/restconf/data' "" + +new "restconf GET datastore" +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","netgate-if-type": "regular"}}}}' + +new "restconf GET interface" +expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0" '{"interface": {"name": "local0","netgate-if-type": "regular"}}' + +new "restconf GET if-type" +expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0/netgate-if-type" '{"netgate-if-type": "regular"}' + +new "restconf POST interface" +expectfn 'curl -s -X POST -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config' "" + +new "restconf POST again" +expectfn 'curl -s -X POST -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config' "Data resource already exis" + +new "restconf POST from top" +expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"TEST","type":"eth0"}}} http://localhost/restconf/data' "Data resource already exists" + +new "restconf DELETE" +expectfn 'curl -s -X DELETE http://localhost/restconf/data/interfaces-config' "" + +new "restconf GET null datastore" +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": null}' + +new "restconf POST initial tree" +expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","netgate-if-type":"regular"}}} http://localhost/restconf/data' "" + +new "restconf PUT initial datastore" + +expectfn 'curl -s -X PUT -d {"data":{"interfaces-config":{"interface":{"name":"local0","netgate-if-type":"regular"}}}} http://localhost/restconf/data' "" + +new "restconf GET datastore" +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","netgate-if-type": "regular"}}}}' + +new "restconf PUT change interface" +expectfn 'curl -s -X PUT -d {"interface":{"name":"local0","type":"atm0"}} http://localhost/restconf/data/interfaces-config/interface=local0' "" + +new "restconf GET datastore" +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","type": "atm0"}}}}' + +new "restconf PUT add interface" +expectfn 'curl -s -X PUT -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "" + +new "restconf PUT change key error" +expectfn 'curl -is -X PUT -d {"interface":{"name":"ALPHA","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "Bad request" + +new "Kill restconf daemon" +sudo pkill -u www-data clixon_restconf + +new "Kill backend" +# Check if still alive +pid=`pgrep clixon_backend` +if [ -z "$pid" ]; then + err "backend already dead" +fi +# kill backend +sudo clixon_backend -zf $cfg +if [ $? -ne 0 ]; then + err "kill backend" +fi + +rm -rf $dir diff --git a/yang/ietf-yang-library@2016-06-21.yang b/yang/ietf-yang-library@2016-06-21.yang new file mode 100644 index 00000000..1e897180 --- /dev/null +++ b/yang/ietf-yang-library@2016-06-21.yang @@ -0,0 +1,242 @@ +module ietf-yang-library { + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-library"; + prefix "yanglib"; + + import ietf-yang-types { + prefix yang; + } + import ietf-inet-types { + prefix inet; + } + organization + "IETF NETCONF (Network Configuration) Working Group"; + + contact + "WG Web: + WG List: + + WG Chair: Mehmet Ersue + + + WG Chair: Mahesh Jethanandani + + + Editor: Andy Bierman + + + Editor: Martin Bjorklund + + + Editor: Kent Watsen + "; + + description + "This module contains monitoring information about the YANG + modules and submodules that are used within a YANG-based + server. + + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Simplified BSD License + set forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7895; see + the RFC itself for full legal notices."; + + revision 2016-06-21 { + description + "Initial revision."; + reference + "RFC 7895: YANG Module Library."; + } + + /* + * Typedefs + */ + + typedef revision-identifier { + type string { + pattern '\d{4}-\d{2}-\d{2}'; + } + description + "Represents a specific date in YYYY-MM-DD format."; + } + + /* + * Groupings + */ + + grouping module-list { + description + "The module data structure is represented as a grouping + so it can be reused in configuration or another monitoring + data structure."; + + grouping common-leafs { + description + "Common parameters for YANG modules and submodules."; + + leaf name { + type yang:yang-identifier; + description + "The YANG module or submodule name."; + } + leaf revision { + type union { + type revision-identifier; + type string { length 0; } + } + description + "The YANG module or submodule revision date. + A zero-length string is used if no revision statement + is present in the YANG module or submodule."; + } + } + + grouping schema-leaf { + description + "Common schema leaf parameter for modules and submodules."; + leaf schema { + type inet:uri; + description + "Contains a URL that represents the YANG schema + resource for this module or submodule. + + This leaf will only be present if there is a URL + available for retrieval of the schema for this entry."; + } + } + + list module { + key "name revision"; + description + "Each entry represents one revision of one module + currently supported by the server."; + + uses common-leafs; + uses schema-leaf; + + leaf namespace { + type inet:uri; + mandatory true; + description + "The XML namespace identifier for this module."; + } + leaf-list feature { + type yang:yang-identifier; + description + "List of YANG feature names from this module that are + supported by the server, regardless of whether they are + defined in the module or any included submodule."; + } + list deviation { + key "name revision"; + description + "List of YANG deviation module names and revisions + used by this server to modify the conformance of + the module associated with this entry. Note that + the same module can be used for deviations for + multiple modules, so the same entry MAY appear + within multiple 'module' entries. + + The deviation module MUST be present in the 'module' + list, with the same name and revision values. + The 'conformance-type' value will be 'implement' for + the deviation module."; + uses common-leafs; + } + leaf conformance-type { + type enumeration { + enum implement { + description + "Indicates that the server implements one or more + protocol-accessible objects defined in the YANG module + identified in this entry. This includes deviation + statements defined in the module. + + For YANG version 1.1 modules, there is at most one + module entry with conformance type 'implement' for a + particular module name, since YANG 1.1 requires that, + at most, one revision of a module is implemented. + + For YANG version 1 modules, there SHOULD NOT be more + than one module entry for a particular module name."; + } + enum import { + description + "Indicates that the server imports reusable definitions + from the specified revision of the module but does + not implement any protocol-accessible objects from + this revision. + + Multiple module entries for the same module name MAY + exist. This can occur if multiple modules import the + same module but specify different revision dates in + the import statements."; + } + } + mandatory true; + description + "Indicates the type of conformance the server is claiming + for the YANG module identified by this entry."; + } + list submodule { + key "name revision"; + description + "Each entry represents one submodule within the + parent module."; + uses common-leafs; + uses schema-leaf; + } + } + } + + + + /* + * Operational state data nodes + */ + + container modules-state { + config false; + description + "Contains YANG module monitoring information."; + + leaf module-set-id { + type string; + mandatory true; + description + "Contains a server-specific identifier representing + the current set of modules and submodules. The + server MUST change the value of this leaf if the + information represented by the 'module' list instances + has changed."; + } + + uses module-list; + } + + /* + * Notifications + */ + notification yang-library-change { + description + "Generated when the set of modules and submodules supported + by the server has changed."; + leaf module-set-id { + type leafref { + path "/yanglib:modules-state/yanglib:module-set-id"; + } + mandatory true; + description + "Contains the module-set-id value representing the + set of modules and submodules supported at the server at + the time the notification is generated."; + } + } +} \ No newline at end of file From 6c7554de25ee0685dc542d8f616482b1dab8dd9a Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Tue, 23 Jan 2018 22:09:15 +0100 Subject: [PATCH 17/28] RESTCONF -Candidate Roll Back # --- apps/restconf/restconf_methods.c | 23 ++++++++------ test/test_restconf.sh | 54 ++++++++++++++++---------------- test/test_restconf2.sh | 31 +++++++----------- 3 files changed, 51 insertions(+), 57 deletions(-) diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 859cf210..ecec7d17 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -488,14 +488,13 @@ api_data_post(clicon_handle h, conflict(r); goto ok; } - if (clicon_rpc_validate(h, "candidate") < 0){ + /* Assume this is validation failed since commit includes validate */ + if (clicon_rpc_commit(h) < 0){ if (clicon_rpc_discard_changes(h) < 0) goto done; badrequest(r); - goto ok; - } - if (clicon_rpc_commit(h) < 0) goto done; + } FCGX_SetExitStatus(201, r->out); /* Created */ FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); FCGX_FPrintF(r->out, "\r\n"); @@ -692,14 +691,13 @@ api_data_put(clicon_handle h, notfound(r); goto ok; } - if (clicon_rpc_validate(h, "candidate") < 0){ - if (clicon_rpc_discard_changes(h) < 0) + /* Assume this is validation failed since commit includes validate */ + if (clicon_rpc_commit(h) < 0){ + if (clicon_rpc_discard_changes(h) < 0) goto done; badrequest(r); - goto ok; - } - if (clicon_rpc_commit(h) < 0) goto done; + } FCGX_SetExitStatus(201, r->out); /* Created */ FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); FCGX_FPrintF(r->out, "\r\n"); @@ -793,8 +791,13 @@ api_data_delete(clicon_handle h, notfound(r); goto ok; } - if (clicon_rpc_commit(h) < 0) + /* Assume this is validation failed since commit includes validate */ + if (clicon_rpc_commit(h) < 0){ + if (clicon_rpc_discard_changes(h) < 0) + goto done; + badrequest(r); goto done; + } FCGX_SetExitStatus(201, r->out); FCGX_FPrintF(r->out, "Content-Type: text/plain\r\n"); FCGX_FPrintF(r->out, "\r\n"); diff --git a/test/test_restconf.sh b/test/test_restconf.sh index 8cc700d6..c1d7ee0c 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -82,26 +82,26 @@ new "restconf options" expectfn "curl -i -sS -X OPTIONS http://localhost/restconf/data" "Allow: OPTIONS,HEAD,GET,POST,PUT,DELETE" new "restconf head" -expectfn "curl -sS -I http://localhost/restconf/data" "HTTP/1.1 200 OK" +expectfn "curl -s -I http://localhost/restconf/data" "HTTP/1.1 200 OK" #Content-Type: application/yang-data+json" new "restconf root discovery" -expectfn "curl -sS -X GET http://localhost/.well-known/host-meta" "" +expectfn "curl -s -X GET http://localhost/.well-known/host-meta" "" new "restconf get restconf json" -expectfn "curl -sSG http://localhost/restconf" '{"data": null,"operations": null,"yang-library-version": "2016-06-21"}}' +expectfn "curl -sG http://localhost/restconf" '{"data": null,"operations": null,"yang-library-version": "2016-06-21"}}' new "restconf get restconf/yang-library-version json" -expectfn "curl -sSG http://localhost/restconf/yang-library-version" '{"yang-library-version": "2016-06-21"}' +expectfn "curl -sG http://localhost/restconf/yang-library-version" '{"yang-library-version": "2016-06-21"}' new "restconf empty rpc" -expectfn 'curl -sS -X POST -d {"input":{"name":""}} http://localhost/restconf/operations/ex:empty' '{"output": null}' +expectfn 'curl -s -X POST -d {"input":{"name":""}} http://localhost/restconf/operations/ex:empty' '{"output": null}' new "restconf get empty config + state json" expectfn "curl -sSG http://localhost/restconf/data" "{\"data\": $state}" new "restconf get empty config + state xml" -ret=$(curl -sS -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data) +ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data) expect="eth0eth42" match=`echo $ret | grep -EZo "$expect"` if [ -z "$match" ]; then @@ -109,11 +109,11 @@ if [ -z "$match" ]; then fi new "restconf get data/interfaces-state/interface=eth0 json" -expectfn "curl -sS -G http://localhost/restconf/data/interfaces-state/interface=eth0" '{"interface": {"name": "eth0","type": "eth","if-index": "42"}}' +expectfn "curl -s -G http://localhost/restconf/data/interfaces-state/interface=eth0" '{"interface": {"name": "eth0","type": "eth","if-index": "42"}}' new "restconf get state operation eth0 xml" # Cant get shell macros to work, inline matching from lib.sh -ret=$(curl -sS -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0) +ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0) expect="eth0eth42" match=`echo $ret | grep -EZo "$expect"` if [ -z "$match" ]; then @@ -121,12 +121,12 @@ if [ -z "$match" ]; then fi new "restconf get state operation eth0 type json" -expectfn "curl -sS -G http://localhost/restconf/data/interfaces-state/interface=eth0/type" '{"type": "eth"} +expectfn "curl -s -G http://localhost/restconf/data/interfaces-state/interface=eth0/type" '{"type": "eth"} $' new "restconf get state operation eth0 type xml" # Cant get shell macros to work, inline matching from lib.sh -ret=$(curl -sS -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0/type) +ret=$(curl -s -H "Accept: application/yang-data+xml" -G http://localhost/restconf/data/interfaces-state/interface=eth0/type) expect="eth" match=`echo $ret | grep -EZo "$expect"` if [ -z "$match" ]; then @@ -134,60 +134,60 @@ if [ -z "$match" ]; then fi new "restconf Add subtree to datastore using POST" -expectfn 'curl -sS -X POST -d {"interfaces":{"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}}} http://localhost/restconf/data' "" +expectfn 'curl -s -X POST -d {"interfaces":{"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}}} http://localhost/restconf/data' "" new "restconf Check interfaces eth/0/0 added" -expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} $' new "restconf delete interfaces" -expectfn 'curl -sS -X DELETE http://localhost/restconf/data/interfaces' "" +expectfn 'curl -s -X DELETE http://localhost/restconf/data/interfaces' "" new "restconf Check empty config" -expectfn "curl -sSG http://localhost/restconf/data" $state +expectfn "curl -sG http://localhost/restconf/data" $state new "restconf Add interfaces subtree eth/0/0 using POST" -expectfn 'curl -sS -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' "" +expectfn 'curl -s -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' "" new "restconf Check eth/0/0 added" -expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} $' new "restconf Re-post eth/0/0 which should generate error" -expectfn 'curl -sS -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' "Data resource already exists" +expectfn 'curl -s -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' "Data resource already exists" new "Add leaf description using POST" -expectfn 'curl -sS -X POST -d {"description":"The-first-interface"} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" +expectfn 'curl -s -X POST -d {"description":"The-first-interface"} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" new "Add nothing using POST" -expectfn 'curl -sS -X POST http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "data is in some way badly formed" +expectfn 'curl -s -X POST http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "data is in some way badly formed" new "restconf Check description added" -expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": "true"}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": "true"}} $' new "restconf delete eth/0/0" -expectfn 'curl -sS -X DELETE http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" +expectfn 'curl -s -X DELETE http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" new "Check deleted eth/0/0" -expectfn 'curl -sS -G http://localhost/restconf/data' $state +expectfn 'curl -s -G http://localhost/restconf/data' $state new "restconf Re-Delete eth/0/0 using none should generate error" -expectfn 'curl -sS -X DELETE http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "Not Found" +expectfn 'curl -s -X DELETE http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "Not Found" new "restconf Add subtree eth/0/0 using PUT" -expectfn 'curl -sS -X PUT -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" +expectfn 'curl -s -X PUT -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" new "restconf get subtree" -expectfn "curl -sS -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} $' new "restconf rpc using POST json" -expectfn 'curl -sS -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{"output": {"route": {"address-family": "ipv4","next-hop": {"next-hop-list": "2.3.4.5"}}}}' +expectfn 'curl -s -X POST -d {"input":{"routing-instance-name":"ipv4"}} http://localhost/restconf/operations/rt:fib-route' '{"output": {"route": {"address-family": "ipv4","next-hop": {"next-hop-list": "2.3.4.5"}}}}' new "restconf rpc using POST xml" # Cant get shell macros to work, inline matching from lib.sh -ret=$(curl -sS -X POST -H "Accept: application/yang-data+xml" -d '{"input":{"routing-instance-name":"ipv4"}}' http://localhost/restconf/operations/rt:fib-route) +ret=$(curl -s -X POST -H "Accept: application/yang-data+xml" -d '{"input":{"routing-instance-name":"ipv4"}}' http://localhost/restconf/operations/rt:fib-route) expect="ipv42.3.4.5" match=`echo $ret | grep -EZo "$expect"` if [ -z "$match" ]; then diff --git a/test/test_restconf2.sh b/test/test_restconf2.sh index 1419f41b..aa939976 100755 --- a/test/test_restconf2.sh +++ b/test/test_restconf2.sh @@ -32,17 +32,9 @@ module example{ type string; } leaf type{ + mandatory true; type string; } - leaf description{ - type string; - } - leaf netgate-if-type{ - type string; - } - leaf enabled{ - type boolean; - } } } } @@ -70,21 +62,17 @@ sleep 1 new "restconf tests" -new "restconf PUT change key error" -#expectfn 'curl -s -X PUT -d {"interface":{"name":"ALPHA","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "fail" -#exit - new "restconf POST initial tree" -expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","netgate-if-type":"regular"}}} http://localhost/restconf/data' "" +expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' "" new "restconf GET datastore" -expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","netgate-if-type": "regular"}}}}' +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","type": "regular"}}}}' new "restconf GET interface" -expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0" '{"interface": {"name": "local0","netgate-if-type": "regular"}}' +expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0" '{"interface": {"name": "local0","type": "regular"}}' new "restconf GET if-type" -expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0/netgate-if-type" '{"netgate-if-type": "regular"}' +expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0/type" '{"type": "regular"}' new "restconf POST interface" expectfn 'curl -s -X POST -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config' "" @@ -102,14 +90,14 @@ new "restconf GET null datastore" expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": null}' new "restconf POST initial tree" -expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","netgate-if-type":"regular"}}} http://localhost/restconf/data' "" +expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' "" new "restconf PUT initial datastore" -expectfn 'curl -s -X PUT -d {"data":{"interfaces-config":{"interface":{"name":"local0","netgate-if-type":"regular"}}}} http://localhost/restconf/data' "" +expectfn 'curl -s -X PUT -d {"data":{"interfaces-config":{"interface":{"name":"local0","type":"regular"}}}} http://localhost/restconf/data' "" new "restconf GET datastore" -expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","netgate-if-type": "regular"}}}}' +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","type": "regular"}}}}' new "restconf PUT change interface" expectfn 'curl -s -X PUT -d {"interface":{"name":"local0","type":"atm0"}} http://localhost/restconf/data/interfaces-config/interface=local0' "" @@ -123,6 +111,9 @@ expectfn 'curl -s -X PUT -d {"interface":{"name":"TEST","type":"eth0"}} http://l new "restconf PUT change key error" expectfn 'curl -is -X PUT -d {"interface":{"name":"ALPHA","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "Bad request" +new "restconf POST invalid no type" +expectfn 'curl -s -X POST -d {"interface":{"name":"ALPHA"}} http://localhost/restconf/data/interfaces-config' "Bad request" + new "Kill restconf daemon" sudo pkill -u www-data clixon_restconf From 99abac76a7dc8431a737a8d12a800766396d0201 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Thu, 25 Jan 2018 22:16:44 +0100 Subject: [PATCH 18/28] Fix for cli expansion of two variables --- lib/src/clixon_xml_map.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index bd7c1dbd..fac69056 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -889,7 +889,7 @@ api_path_fmt2api_path(char *api_path_fmt, char *strenc=NULL; cg_var *cv; -#if 1 +#if 0 /* XXX Does not work in expansion case */ /* Sanity check */ j = 0; /* Count % */ for (i=0; i'"); // goto done; } else{ if ((valvec = clicon_strsep(restval, ",", &nvalvec)) == NULL) goto done; - if (cvec_len(cvk) != nvalvec){ + if (nvalvec > cvec_len(cvk)){ clicon_err(OE_XML, errno, "List %s key length mismatch", name); goto done; } From 9914847d6a88dd563a4ad88ddaac0f987cf6a94b Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 27 Jan 2018 13:32:51 +0100 Subject: [PATCH 19/28] GET Single element JSON lists use {list:[element]}, not {list:element}. --- CHANGELOG.md | 8 +++- lib/src/clixon_json.c | 106 +++++++++++++++++++++++------------------ test/test_restconf.sh | 12 ++--- test/test_restconf2.sh | 10 ++-- 4 files changed, 76 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e57114f..ccd08e1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## 3.5.0 (Upcoming) ### Major changes: +* Major Restconf feature update to compy to RFC 8040. Thanks Stephen Jones for getting right. + * GET well-known, top-level resource, yang library version, + * PUT whole datastore, check for different keys in put lists. + * GET Single element JSON lists use {list:[element]}, not {list:element}. + ### Minor changes: @@ -11,9 +16,8 @@ * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` -* new configuration option: CLICON_RESTCONF_PRETTY +* New configuration option: CLICON_RESTCONF_PRETTY * Changed restconf GET to return object referenced. ie, GET /restconf/data/X returns X. Thanks Stephen Jones for getting this right. -* Restconf: get well-known, top-level resource, yang library version, put whole datastore, check for different keys in put lists. * Default configure file added by Matt Smith. Config file is selected in the following priority order: * Provide -f option when starting a program. diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index 50fd06a4..2605468e 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -77,9 +77,10 @@ enum array_element_type{ NO_ARRAY=0, - FIRST_ARRAY, - MIDDLE_ARRAY, - LAST_ARRAY, + FIRST_ARRAY, /* [a, */ + MIDDLE_ARRAY, /* a, */ + LAST_ARRAY, /* a] */ + SINGLE_ARRAY, /* [a] */ BODY_ARRAY }; @@ -143,6 +144,9 @@ arraytype2str(enum array_element_type lt) case LAST_ARRAY: return "last"; break; + case SINGLE_ARRAY: + return "single"; + break; case BODY_ARRAY: return "body"; break; @@ -152,21 +156,23 @@ arraytype2str(enum array_element_type lt) static enum array_element_type array_eval(cxobj *xprev, - cxobj *x, - cxobj *xnext) + cxobj *x, + cxobj *xnext) { enum array_element_type array = NO_ARRAY; - int eqprev=0; - int eqnext=0; + int eqprev=0; + int eqnext=0; + yang_stmt *ys; if (xml_type(x)!=CX_ELMNT){ array=BODY_ARRAY; goto done; } + ys = xml_spec(x); if (xnext && xml_type(xnext)==CX_ELMNT && strcmp(xml_name(x),xml_name(xnext))==0) - eqnext++; + eqnext++; if (xprev && xml_type(xprev)==CX_ELMNT && strcmp(xml_name(x),xml_name(xprev))==0) @@ -177,6 +183,8 @@ array_eval(cxobj *xprev, array = LAST_ARRAY; else if (eqnext) array = FIRST_ARRAY; + else if (ys && ys->ys_keyword == Y_LIST) + array = SINGLE_ARRAY; else array = NO_ARRAY; done: @@ -228,29 +236,29 @@ json_escape(char *str) * The following matrix explains how the mapping is done. * You need to understand what arraytype means (no/first/middle/last) * and what childtype is (null,body,any) - +---------+--------------+--------------+--------------+ + +----------+--------------+--------------+--------------+ |array,leaf| null | body | any | - +---------+--------------+--------------+--------------+ - |no | |1 | | - | | | | | - | json: |\ta:null |\ta: |\ta:{\n | - | | | |\n} | - +---------+--------------+--------------+--------------+ - |first |11 |..a>1 | | - | | | | | - | json: |\tnull |\t |\t{a | - | |\n\t] |\n\t] |\n\t}\t] | - +---------+--------------+--------------+--------------+ + +----------+--------------+--------------+--------------+ + |no | |1 | | + | | | | | + | json: |\ta:null |\ta: |\ta:{\n | + | | | |\n} | + +----------+--------------+--------------+--------------+ + |first |11 |..a>1 | | + | | | | | + | json: |\tnull |\t |\t{a | + | |\n\t] |\n\t] |\n\t}\t] | + +----------+--------------+--------------+--------------+ */ static int xml2json1_cbuf(cbuf *cb, @@ -299,6 +307,7 @@ xml2json1_cbuf(cbuf *cb, } break; case FIRST_ARRAY: + case SINGLE_ARRAY: cprintf(cb, "%*s\"%s\": ", pretty?(level*JSON_INDENT):0, "", xml_name(x)); @@ -387,6 +396,7 @@ xml2json1_cbuf(cbuf *cb, break; } break; + case SINGLE_ARRAY: case LAST_ARRAY: switch (childt){ case NULL_CHILD: @@ -419,7 +429,7 @@ xml2json1_cbuf(cbuf *cb, * @param[in,out] cb Cligen buffer to write to * @param[in] x XML tree to translate from * @param[in] pretty Set if output is pretty-printed - * @param[in] top By default only children are printed, set if include top + * @param[in] top By default only children are printed, set if include top * @retval 0 OK * @retval -1 Error * @@ -433,9 +443,9 @@ xml2json1_cbuf(cbuf *cb, * @see clicon_xml2cbuf */ int -xml2json_cbuf(cbuf *cb, - cxobj *x, - int pretty) +xml2json_cbuf(cbuf *cb, + cxobj *x, + int pretty) { int retval = 1; int level = 0; @@ -472,10 +482,10 @@ xml2json_cbuf(cbuf *cb, * @see xml2json1_cbuf */ int -xml2json_cbuf_vec(cbuf *cb, - cxobj **vec, - size_t veclen, - int pretty) +xml2json_cbuf_vec(cbuf *cb, + cxobj **vec, + size_t veclen, + int pretty) { int retval = -1; int level = 0; @@ -519,15 +529,17 @@ xml2json_cbuf_vec(cbuf *cb, * @retval 0 OK * @retval -1 Error * + * @note yang is necessary to translate to one-member lists, + * eg if a is a yang LIST 0 -> {"a":["0"]} and not {"a":"0"} * @code * if (xml2json(stderr, xn, 0) < 0) * goto err; * @endcode */ int -xml2json(FILE *f, - cxobj *x, - int pretty) +xml2json(FILE *f, + cxobj *x, + int pretty) { int retval = 1; cbuf *cb = NULL; @@ -560,10 +572,10 @@ xml2json(FILE *f, * @see xml2json1_cbuf */ int -xml2json_vec(FILE *f, - cxobj **vec, - size_t veclen, - int pretty) +xml2json_vec(FILE *f, + cxobj **vec, + size_t veclen, + int pretty) { int retval = 1; cbuf *cb = NULL; @@ -645,7 +657,7 @@ json_parse_str(char *str, /*! Read a JSON definition from file and parse it into a parse-tree. * * @param[in] fd A file descriptor containing the JSON file (as ASCII characters) - * @param[in] yspec Yang specification, or NULL + * @param[in] yspec Yang specification, or NULL XXX Not yet used * @param[in,out] xt Pointer to (XML) parse tree. If empty, create. * @retval 0 OK * @retval -1 Error with clicon_err called @@ -722,7 +734,7 @@ json_parse_file(int fd, /* * Turn this on to get a json parse and pretty print test program * Usage: xpath - * read xml from input + * read json from input * Example compile: gcc -g -o json -I. -I../clixon ./clixon_json.c -lclixon -lcligen * Example run: diff --git a/test/test_restconf.sh b/test/test_restconf.sh index c1d7ee0c..8f429ef1 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -54,7 +54,7 @@ module example{ EOF # This is a fixed 'state' implemented in routing_backend. It is assumed to be always there -state='{"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}}' +state='{"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": "42"}\]}}' # kill old backend (if any) new "kill old backend" @@ -109,7 +109,7 @@ if [ -z "$match" ]; then fi new "restconf get data/interfaces-state/interface=eth0 json" -expectfn "curl -s -G http://localhost/restconf/data/interfaces-state/interface=eth0" '{"interface": {"name": "eth0","type": "eth","if-index": "42"}}' +expectfn "curl -s -G http://localhost/restconf/data/interfaces-state/interface=eth0" '{"interface": \[{"name": "eth0","type": "eth","if-index": "42"}\]}' new "restconf get state operation eth0 xml" # Cant get shell macros to work, inline matching from lib.sh @@ -137,7 +137,7 @@ new "restconf Add subtree to datastore using POST" expectfn 'curl -s -X POST -d {"interfaces":{"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}}} http://localhost/restconf/data' "" new "restconf Check interfaces eth/0/0 added" -expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","type": "eth","enabled": "true"}\]},"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": "42"}\]}} $' new "restconf delete interfaces" @@ -150,7 +150,7 @@ new "restconf Add interfaces subtree eth/0/0 using POST" expectfn 'curl -s -X POST -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces' "" new "restconf Check eth/0/0 added" -expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","type": "eth","enabled": "true"}\]},"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": "42"}\]}} $' new "restconf Re-post eth/0/0 which should generate error" @@ -163,7 +163,7 @@ new "Add nothing using POST" expectfn 'curl -s -X POST http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "data is in some way badly formed" new "restconf Check description added" -expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": "true"}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","description": "The-first-interface","type": "eth","enabled": "true"}\]} $' new "restconf delete eth/0/0" @@ -179,7 +179,7 @@ new "restconf Add subtree eth/0/0 using PUT" expectfn 'curl -s -X PUT -d {"interface":{"name":"eth/0/0","type":"eth","enabled":"true"}} http://localhost/restconf/data/interfaces/interface=eth%2f0%2f0' "" new "restconf get subtree" -expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": {"name": "eth/0/0","type": "eth","enabled": "true"}},"interfaces-state": {"interface": {"name": "eth0","type": "eth","if-index": "42"}}} +expectfn "curl -s -G http://localhost/restconf/data" '{"interfaces": {"interface": \[{"name": "eth/0/0","type": "eth","enabled": "true"}\]},"interfaces-state": {"interface": \[{"name": "eth0","type": "eth","if-index": "42"}\]}} $' new "restconf rpc using POST json" diff --git a/test/test_restconf2.sh b/test/test_restconf2.sh index aa939976..e9d709f1 100755 --- a/test/test_restconf2.sh +++ b/test/test_restconf2.sh @@ -66,10 +66,10 @@ new "restconf POST initial tree" expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' "" new "restconf GET datastore" -expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","type": "regular"}}}}' +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": \[{"name": "local0","type": "regular"}\]}}}' new "restconf GET interface" -expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0" '{"interface": {"name": "local0","type": "regular"}}' +expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0" '{"interface": \[{"name": "local0","type": "regular"}\]}' new "restconf GET if-type" expectfn "curl -s -X GET http://localhost/restconf/data/interfaces-config/interface=local0/type" '{"type": "regular"}' @@ -97,13 +97,13 @@ new "restconf PUT initial datastore" expectfn 'curl -s -X PUT -d {"data":{"interfaces-config":{"interface":{"name":"local0","type":"regular"}}}} http://localhost/restconf/data' "" new "restconf GET datastore" -expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","type": "regular"}}}}' +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": \[{"name": "local0","type": "regular"}\]}}}' new "restconf PUT change interface" expectfn 'curl -s -X PUT -d {"interface":{"name":"local0","type":"atm0"}} http://localhost/restconf/data/interfaces-config/interface=local0' "" -new "restconf GET datastore" -expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": {"name": "local0","type": "atm0"}}}}' +new "restconf GET datastore atm" +expectfn "curl -s -X GET http://localhost/restconf/data" '{"data": {"interfaces-config": {"interface": \[{"name": "local0","type": "atm0"}\]}}}' new "restconf PUT add interface" expectfn 'curl -s -X PUT -d {"interface":{"name":"TEST","type":"eth0"}} http://localhost/restconf/data/interfaces-config/interface=TEST' "" From f65187df7fca1541fc2227ddc50f57a979968448 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 27 Jan 2018 14:28:57 +0100 Subject: [PATCH 20/28] code cleanup --- lib/src/clixon_xml_map.c | 129 ++++++++++++++------------------------- 1 file changed, 47 insertions(+), 82 deletions(-) diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index fac69056..8f755666 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -88,9 +88,6 @@ #include "clixon_xml_sort.h" #include "clixon_xml_map.h" -/* Something to do with reverse engineering of junos syntax? */ -#undef SPECIAL_TREATMENT_OF_NAME - /* * A node is a leaf if it contains a body. */ @@ -131,9 +128,6 @@ xml2txt(FILE *f, char *term = NULL; int retval = -1; int encr=0; -#ifdef SPECIAL_TREATMENT_OF_NAME - cxobj *xname; -#endif xe = NULL; /* count children */ while ((xe = xml_child_each(x, xe, -1)) != NULL) @@ -157,32 +151,11 @@ xml2txt(FILE *f, goto done; } fprintf(f, "%*s", 4*level, ""); - -#ifdef SPECIAL_TREATMENT_OF_NAME - if (strcmp(xml_name(x), "name") != 0) - fprintf(f, "%s ", xml_name(x)); - if ((xname = xml_find(x, "name")) != NULL){ - if (children > 1) - fprintf(f, "%s ", xml_body(xname)); - if (!tleaf(x)) - fprintf(f, "{\n"); - } - else - if (!tleaf(x)) - fprintf(f, "{\n"); -#else - fprintf(f, "%s ", xml_name(x)); - if (!tleaf(x)) - fprintf(f, "{\n"); -#endif /* SPECIAL_TREATMENT_OF_NAME */ - + fprintf(f, "%s ", xml_name(x)); + if (!tleaf(x)) + fprintf(f, "{\n"); xe = NULL; while ((xe = xml_child_each(x, xe, -1)) != NULL){ -#ifdef SPECIAL_TREATMENT_OF_NAME - if (xml_type(xe) == CX_ELMNT && (strcmp(xml_name(xe), "name")==0) && - (children > 1)) /* skip if this is a name element (unless 0 children) */ - continue; -#endif if (xml2txt(f, xe, level+1) < 0) break; } @@ -870,9 +843,18 @@ yang2api_path_fmt(yang_stmt *ys, * @param[out] yang_arg yang-stmt argument name. Free after use * @note first and last elements of cvv are not used,.. * @see api_path_fmt2xpath - * - * /interfaces/interface=%s/name --> /interfaces/interface/name - * /interfaces/interface=%s/ipv4/address=%s --> /interfaces/interface=e/ipv4/address + * @example + * api_path_fmt: /interfaces/interface=%s/name + * cvv: - + * api_path: /interfaces/interface/name + * @example + * api_path_fmt: /interfaces/interface=%s/name + * cvv: e0 + * api_path: /interfaces/interface=e0/name + * @example + * api_path_fmt: /subif-entry=%s,%s/subid + * cvv: foo + * api_path: /subif-entry=foo/subid */ int api_path_fmt2api_path(char *api_path_fmt, @@ -889,20 +871,6 @@ api_path_fmt2api_path(char *api_path_fmt, char *strenc=NULL; cg_var *cv; -#if 0 /* XXX Does not work in expansion case */ - /* Sanity check */ - j = 0; /* Count % */ - for (i=0; i cvec_len(cvv)) { //cvec_len can be longer - clicon_log(LOG_WARNING, "%s api_path_fmt number of %% is %d, does not match number of cvv entries %d", - api_path_fmt, - j, - cvec_len(cvv)); - goto done; - } -#endif if ((cb = cbuf_new()) == NULL){ clicon_err(OE_UNIX, errno, "cbuf_new"); goto done; @@ -953,18 +921,22 @@ api_path_fmt2api_path(char *api_path_fmt, /*! Transform an xml key format and a vector of values to an XML path * Used to input xmldb_get() or xmldb_get_vec - * Add .* in last %s position. - * Example: - * api_path_fmt: /interface/%s/address/%s - * cvv: name=eth0 - * xmlkey: /interface/[name=eth0]/address - * Example2: - * xmlkeyfmt: /ip/me/%s (if key) - * cvv: - - * xmlkey: /ipv4/me/a * @param[in] api_path_fmt XML key format * @param[in] cvv cligen variable vector, one for every wildchar in api_path_fmt * @param[out] xpath XPATH + * Add .* in last %s position. + * @example + * api_path_fmt: /interface/%s/address/%s + * cvv: name=eth0 + * xpath: /interface/[name=eth0]/address + * @example + * api_path_fmt: /ip/me/%s (if key) + * cvv: - + * xpath: /ipv4/me/a + * @example + * api_path_fmt: /subif-entry=%s,%s/subid + * cvv: foo + * xpath: /subif-entry[if-name=foo]/subid" */ int api_path_fmt2xpath(char *api_path_fmt, @@ -980,21 +952,6 @@ api_path_fmt2xpath(char *api_path_fmt, char *str; cg_var *cv; - /* Sanity check: count '%' */ -#if 0 /* XXX Does not work in expansion case */ - j = 0; /* Count % */ - for (i=0; i cvec_len(cvv)) { - clicon_log(LOG_WARNING, "%s xmlkey format string mismatch(j=%d, cvec_len=%d): %s", - api_path_fmt, - j, - cvec_len(cvv), - cv_string_get(cvec_i(cvv, 0))); - goto done; - } -#endif if ((cb = cbuf_new()) == NULL){ clicon_err(OE_UNIX, errno, "cbuf_new"); goto done; @@ -1624,20 +1581,29 @@ api_path2xml_vec(char **vec, } /*! Create xml tree from api-path - * @param[in] api_path API-path as defined in RFC 8040 - * @param[in] yspec Yang spec - * @param[in] schemanode If set use schema nodes otherwise data nodes. - * @param[out] xpathp Resulting xml tree - * @param[out] ypathp Yang spec matching xpathp + * @param[in] api_path API-path as defined in RFC 8040 + * @param[in] yspec Yang spec + * @param[in,out] xtop Incoming XML tree + * @param[in] schemanode If set use schema nodes otherwise data nodes. + * @param[out] xbotp Resulting xml tree (end of xpath) + * @param[out] ybotp Yang spec matching xbotp * @see api_path2xml_vec + * @example + * api_path: /subif-entry=foo/subid + * xtop[in] + * xtop[out]: + * foo> + * + * xbotp: + * ybotp: Y_LEAF subid */ int api_path2xml(char *api_path, yang_spec *yspec, - cxobj *xpath, + cxobj *xtop, int schemanode, - cxobj **xpathp, - yang_node **ypathp) + cxobj **xbotp, + yang_node **ybotp) { int retval = -1; char **vec = NULL; @@ -1659,8 +1625,8 @@ api_path2xml(char *api_path, } nvec--; /* NULL-terminated */ if (api_path2xml_vec(vec+1, nvec, - xpath, (yang_node*)yspec, schemanode, - xpathp, ypathp) < 0) + xtop, (yang_node*)yspec, schemanode, + xbotp, ybotp) < 0) goto done; retval = 0; done: @@ -1669,7 +1635,6 @@ api_path2xml(char *api_path, return retval; } - /*! Merge a base tree x0 with x1 with yang spec y * @param[in] x0 Base xml tree (can be NULL in add scenarios) * @param[in] y0 Yang spec corresponding to xml-node x0. NULL if x0 is NULL From ffe77c9127c33d10337e0cb7d0d4cd1e6727412e Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 27 Jan 2018 14:46:53 +0100 Subject: [PATCH 21/28] restconf performance test --- test/plot_perf.sh | 63 +++++++++++++++++++++++++++-------------------- test/test_perf.sh | 59 ++++++++++++++++++++++++++++++++------------ 2 files changed, 79 insertions(+), 43 deletions(-) diff --git a/test/plot_perf.sh b/test/plot_perf.sh index db73f7cc..c5eaaa98 100755 --- a/test/plot_perf.sh +++ b/test/plot_perf.sh @@ -1,12 +1,12 @@ #!/bin/bash # Transactions per second for large lists read/write plotter using gnuplot -# +# WORK IN PROGRESS . ./lib.sh -max=200 # Nr of db entries -step=100 -reqs=1000 -cfg=$dir/scaling-conf.xml -fyang=$dir/scaling.yang +max=2000 # Nr of db entries +step=200 +reqs=500 +cfg=$dir/plot-conf.xml +fyang=$dir/plot.yang fconfig=$dir/config # For memcheck @@ -40,6 +40,7 @@ cat < $cfg ietf-ip /usr/local/var/routing/routing.sock /usr/local/var/routing/routing.pidfile +false /usr/local/var/routing /usr/local/lib/xmldb/text.so @@ -52,9 +53,16 @@ run(){ echo -n "replace" > $fconfig for (( i=0; i<$nr; i++ )); do - echo -n "$i" >> $fconfig - echo -n "$i$i" >> $fconfig + case $mode in + readlist|writelist|restreadlist|restwritelist) + echo -n "$i$i" >> $fconfig + ;; + writeleaflist) + echo -n "$i" >> $fconfig + ;; + esac done + echo "]]>]]>" >> $fconfig expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" @@ -64,42 +72,45 @@ run(){ time -p for (( i=0; i<$reqs; i++ )); do rnd=$(( ( RANDOM % $nr ) )) echo "]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null ;; writelist) time -p for (( i=0; i<$reqs; i++ )); do rnd=$(( ( RANDOM % $nr ) )) echo "$rnd$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null - ;; - readleaflist) +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + ;; + restreadlist) time -p for (( i=0; i<$reqs; i++ )); do rnd=$(( ( RANDOM % $nr ) )) - echo "]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null - ;; + curl -sSG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null +done + ;; writeleaflist) time -p for (( i=0; i<$reqs; i++ )); do rnd=$(( ( RANDOM % $nr ) )) echo "$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null ;; esac - expecteof "$clixon_netconf -qf $cfg" "]]>]]>" "^]]>]]>$" + expecteof "$clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" + } step(){ i=$1 mode=$2 echo -n "" > $fconfig - t=$(TEST=%e run $i $reqs $mode $ 2>&1 | awk '/real/ {print $2}') - # t is time in secs of $reqs -> transactions per second. $reqs + t=$(TEST=%e run $i $reqs $mode 2>&1 | awk '/real/ {print $2}') + #TEST=%e run $i $reqs $mode 2>&1 + # t is time in secs of $reqs -> transactions per second. $reqs p=$(echo "$reqs/$t" | bc -lq) # p is transactions per second. echo "$i $p" >> $dir/$mode +# echo "m:$mode i:$i t=$t p=$p" } -once()( +once(){ # kill old backend (if any) sudo clixon_backend -zf $cfg -y $fyang if [ $? -ne 0 ]; then @@ -116,14 +127,14 @@ once()( for (( i=10; i<=$step; i=i+10 )); do step $i readlist step $i writelist - step $i readleaflist + step $i restreadlist step $i writeleaflist done # Actual steps for (( i=$step; i<=$max; i=i+$step )); do step $i readlist - step $i readleaflist step $i writelist + step $i restreadlist step $i writeleaflist done @@ -137,8 +148,7 @@ once()( if [ $? -ne 0 ]; then err "kill backend" fi - -) +} once @@ -146,10 +156,9 @@ gnuplot -persist < $cfg ietf-ip /usr/local/var/routing/routing.sock /usr/local/var/routing/routing.pidfile +false /usr/local/var/routing /usr/local/lib/xmldb/text.so @@ -73,6 +74,14 @@ if [ $? -ne 0 ]; then err fi +new "kill old restconf daemon" +sudo pkill -u www-data clixon_restconf + +new "start restconf daemon" +sudo start-stop-daemon -S -q -o -b -x /www-data/clixon_restconf -d /www-data -c www-data -- -Df $cfg -y $fyang + +sleep 1 + new "generate 'large' config with $number list entries" echo -n "" > $fconfig for (( i=0; i<$number; i++ )); do @@ -84,25 +93,37 @@ echo "]]>]]>" >> $fconfig echo "$clixon_netconf -qf $cfg -y $fyang" new "netconf write large config" -expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" #echo ']]>]]>' | $clixon_netconf -qf $cfg -y $fyang new "netconf write large config again" -expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" #echo ']]>]]>' | $clixon_netconf -qf $cfg -y $fyang rm $fconfig new "netconf commit large config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" new "netconf commit large config again" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" new "netconf add small (1 entry) config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "xy]]>]]>" "^]]>]]>$" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "xy]]>]]>" "^]]>]]>$" + +new "netconf get $req small config" +time -p for (( i=0; i<$req; i++ )); do + rnd=$(( ( RANDOM % $number ) )) + echo "]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + +new "netconf get $req restconf small config" +time -p for (( i=0; i<$req; i++ )); do + rnd=$(( ( RANDOM % $number ) )) +#XXX curl -sX PUT -d {"y":{"a":"$rnd","b":"$rnd"}} http://localhost/restconf/data/x/y=$rnd,$rnd +done new "netconf add $req small config" time -p for (( i=0; i<$req; i++ )); do @@ -110,14 +131,14 @@ time -p for (( i=0; i<$req; i++ )); do echo "$rnd$rnd]]>]]>" done | $clixon_netconf -qf $cfg -y $fyang > /dev/null -new "netconf get large config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^0011" - -new "netconf get $req small config" +new "netconf add $req restconf small config" time -p for (( i=0; i<$req; i++ )); do rnd=$(( ( RANDOM % $number ) )) - echo "]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + curl -sG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null +done + +new "netconf get large config" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^0011" new "generate large leaf-list config" echo -n "replace" > $fconfig @@ -127,21 +148,27 @@ done echo "]]>]]>" >> $fconfig new "netconf replace large list-leaf config" -expecteof_file "time -p $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" +expecteof_file "time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" rm $fconfig new "netconf commit large leaf-list config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" + +new "netconf add $req small leaf-list config" +time -p for (( i=0; i<$req; i++ )); do + rnd=$(( ( RANDOM % $number ) )) + echo "$rnd]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang > /dev/null new "netconf add small leaf-list config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "x]]>]]>" "^]]>]]>$" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "x]]>]]>" "^]]>]]>$" new "netconf commit small leaf-list config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" new "netconf get large leaf-list config" -expecteof "time -p $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^01" +expecteof "time -f %e $clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^01" new "Kill backend" # Check if still alive From 2bbf0b8a15941d4f44086df89e4637b6e3216184 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Fri, 2 Feb 2018 16:35:37 +0700 Subject: [PATCH 22/28] xml2json dont put doublequotes on numbers and bool values in json --- lib/src/clixon_json.c | 58 +++++++++++++++++++++++++++++++++++-------- test/test_restconf.sh | 22 ++++++++-------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index 2605468e..3e225396 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -191,8 +191,10 @@ array_eval(cxobj *xprev, return array; } -char * -json_escape(char *str) +/*! Escape a json string + */ +static char * +json_str_escape(char *str) { int i, j; char *snew; @@ -231,6 +233,7 @@ json_escape(char *str) * @param[in] level Indentation level * @param[in] pretty Pretty-print output (2 means debug) * @param[in] flat Dont print NO_ARRAY object name (for _vec call) + * @param[in] bodystr Set if value is string, 0 otherwise. Only if body * * @note Does not work with XML attributes * The following matrix explains how the mapping is done. @@ -266,26 +269,35 @@ xml2json1_cbuf(cbuf *cb, enum array_element_type arraytype, int level, int pretty, - int flat) + int flat, + int bodystr) { int retval = -1; int i; cxobj *xc; enum childtype childt; enum array_element_type xc_arraytype; + yang_stmt *ys; + int bodystr0=1; childt = childtype(x); + ys = xml_spec(x); if (pretty==2) cprintf(cb, "#%s_array, %s_child ", arraytype2str(arraytype), childtype2str(childt)); switch(arraytype){ case BODY_ARRAY:{ - char *str; - if ((str = json_escape(xml_value(x))) == NULL) - goto done; - cprintf(cb, "\"%s\"", str); - free(str); + if (bodystr){ + char *str; + if ((str = json_str_escape(xml_value(x))) == NULL) + goto done; + cprintf(cb, "\"%s\"", str); + free(str); + } + else + cprintf(cb, "%s", xml_value(x)); + break; } case NO_ARRAY: @@ -349,6 +361,30 @@ xml2json1_cbuf(cbuf *cb, default: break; } + /* Check for typed sub-body if: + * arracytype=* but chilt-type is BODY_CHILD + * This is code for writing 42 as "a":42 and not "a":"42" + */ + if (childt == BODY_CHILD && ys!=NULL && + (ys->ys_keyword == Y_LEAF || ys->ys_keyword == Y_LEAF_LIST)) + switch (cv_type_get(ys->ys_cv)){ + case CGV_INT8: + case CGV_INT16: + case CGV_INT32: + case CGV_INT64: + case CGV_UINT8: + case CGV_UINT16: + case CGV_UINT32: + case CGV_UINT64: + case CGV_DEC64: + case CGV_BOOL: + bodystr0 = 0; + break; + default: + bodystr0 = 1; + break; + } + for (i=0; i Date: Sat, 3 Feb 2018 14:56:27 +0700 Subject: [PATCH 23/28] xml2cvec: range error (eg 1000 for int8) is not treated as error, just log and skip. --- CHANGELOG.md | 3 ++- lib/src/clixon_xml_map.c | 28 +++++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccd08e1e..44f07982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,8 @@ ### Corrected Bugs * Corrected "No yang spec" printed on tty on leafref CLI usage - +* xml2cvec: range error (eg 1000 for int8) is not treated as error, just log and skip. + ### Known issues ## 3.4.0 (1 January 2018) diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 8f755666..dd913329 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -415,6 +415,8 @@ xml_yang_validate_all(cxobj *xt, * @retval 0 Everything OK, cvv allocated and set * @retval -1 Something wrong, clicon_err() called to set error. No cvv returned * 'Not recursive' means that only one level of XML bodies is translated to cvec:s. + * If range is wriong (eg 1000 for uint8) a warning is logged, the value is + * skipped, and continues. * yang is needed to know which type an xml element has. * Example: @@ -457,28 +459,30 @@ xml2cvec(cxobj *xt, clicon_debug(0, "%s: yang sanity problem: %s in xml but not present in yang under %s", __FUNCTION__, name, yt->ys_argument); if ((body = xml_body(xc)) != NULL){ - if ((cv = cvec_add(cvv, CGV_STRING)) == NULL){ - clicon_err(OE_PLUGIN, errno, "cvec_add"); + if ((cv = cv_new(CGV_STRING)) == NULL){ + clicon_err(OE_PLUGIN, errno, "cv_new"); goto err; } cv_name_set(cv, name); if ((ret = cv_parse1(body, cv, &reason)) < 0){ - clicon_err(OE_PLUGIN, errno, "cv_parse"); + clicon_err(OE_PLUGIN, errno, "cv_parse %s",name); goto err; } + /* If value is out-of-range, log and skip value, and continue */ if (ret == 0){ - clicon_err(OE_PLUGIN, errno, "cv_parse: %s", reason); + clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason); if (reason) free(reason); - goto err; } + else + cvec_append_var(cvv, cv); /* Add to variable vector */ + cv_free(cv); } } else if ((ycv = ys->ys_cv) != NULL){ if ((body = xml_body(xc)) != NULL){ - /* XXX: cvec_add uses realloc, can we avoid that? */ - if ((cv = cvec_add(cvv, CGV_STRING)) == NULL){ - clicon_err(OE_PLUGIN, errno, "cvec_add"); + if ((cv = cv_new(CGV_STRING)) == NULL){ + clicon_err(OE_PLUGIN, errno, "cv_new"); goto err; } if (cv_cp(cv, ycv) < 0){ @@ -486,15 +490,17 @@ xml2cvec(cxobj *xt, goto err; } if ((ret = cv_parse1(body, cv, &reason)) < 0){ - clicon_err(OE_PLUGIN, errno, "cv_parse"); + clicon_err(OE_PLUGIN, errno, "cv_parse: %s", name); goto err; } if (ret == 0){ - clicon_err(OE_PLUGIN, errno, "cv_parse: %s", reason); + clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason); if (reason) free(reason); - goto err; } + else + cvec_append_var(cvv, cv); /* Add to variable vector */ + cv_free(cv); } } } From d670b6af8af4f0c8d4e60c0e434b149a18abf717 Mon Sep 17 00:00:00 2001 From: Renato Botelho do Couto Date: Tue, 6 Feb 2018 05:54:05 -0600 Subject: [PATCH 24/28] Make default config to respect sysconfdir Instead of hardcode default config to /etc/clixon.xml, make it to respect sysconfdir variable. Since autoconf doesn't expand variables on header substitution it would end up being defined on clixon_config.h as "${prefix}/etc/clixon.xml" what makes no sense for the header file. Use eval to expand ${sysconfdir} and make .h file to be defined with final directory name. --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index a3b0392f..1b6ffc9e 100755 --- a/configure +++ b/configure @@ -4053,7 +4053,7 @@ fi if test "${with_configfile+set}" = set; then : withval=$with_configfile; DEFAULT_CONFIG="$withval" else - DEFAULT_CONFIG="/etc/clixon.xml" + DEFAULT_CONFIG="$(eval echo ${sysconfdir}/clixon.xml)" fi diff --git a/configure.ac b/configure.ac index 941f8e3d..503bc9b8 100644 --- a/configure.ac +++ b/configure.ac @@ -165,7 +165,7 @@ fi AC_ARG_WITH([configfile], [AS_HELP_STRING([--with-configfile=FILE],[set default path to config file])], [DEFAULT_CONFIG="$withval"], - [DEFAULT_CONFIG="/etc/clixon.xml"]) + [DEFAULT_CONFIG="$(eval echo ${sysconfdir}/clixon.xml)"]) AC_CHECK_LIB(crypt, crypt) AC_CHECK_HEADERS(crypt.h) From e40d785d5ca7b81167f9e5bc6dc119d81db83e0d Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Thu, 8 Feb 2018 15:24:05 +0700 Subject: [PATCH 25/28] * Added a "user" parameter to plugin_credentials() restconf callback. To enable authentication and in preparation for access control a la RFC 6536. * yang string length "max" keyword set to MAXPATHLEN --- CHANGELOG.md | 4 +- apps/cli/cli_generate.c | 17 +- apps/cli/cli_show.c | 2 +- apps/restconf/restconf_lib.c | 24 +- apps/restconf/restconf_lib.h | 2 +- apps/restconf/restconf_main.c | 24 +- apps/restconf/restconf_methods.c | 18 +- apps/restconf/restconf_methods.h | 3 +- lib/clixon/clixon_plugin.h | 5 +- lib/src/clixon_xml.c | 3 + yang/Makefile.in | 1 + yang/ietf-netconf-acm@2012-02-22.yang | 445 ++++++++++++++++++++++++++ 12 files changed, 514 insertions(+), 34 deletions(-) create mode 100644 yang/ietf-netconf-acm@2012-02-22.yang diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f07982..25cedf06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.5.0 (Upcoming) ### Major changes: +* Added a "user" parameter to plugin_credentials() restconf callback. To enable authentication and in preparation for access control a la RFC 6536. * Major Restconf feature update to compy to RFC 8040. Thanks Stephen Jones for getting right. * GET well-known, top-level resource, yang library version, * PUT whole datastore, check for different keys in put lists. @@ -10,7 +11,7 @@ ### Minor changes: - +* Added RFC 6536 ietf-netconf-acm@2012-02-22.yang access control (but not implemented). * The following backward compatible options to configure have been obsoleted. If you havent already migrated this code you must do this now. * Backend startup modes prior to 3.3.3. As enabled with `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted. * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. @@ -25,6 +26,7 @@ * /etc/clixon.xml ### Corrected Bugs +* yang string length "max" keyword set to MAXPATHLEN * Corrected "No yang spec" printed on tty on leafref CLI usage * xml2cvec: range error (eg 1000 for int8) is not treated as error, just log and skip. diff --git a/apps/cli/cli_generate.c b/apps/cli/cli_generate.c index 3b8f2ce4..023b531d 100644 --- a/apps/cli/cli_generate.c +++ b/apps/cli/cli_generate.c @@ -238,12 +238,21 @@ yang2cli_var_sub(clicon_handle h, goto done; } } + else{ /* Cligen does not have 'max' keyword in range so need to find actual - max value of type if yang range expression is 0..max */ - if ((r = cvtype_max2str_dup(cvtype)) == NULL){ - clicon_err(OE_UNIX, errno, "cvtype_max2str"); - goto done; + max value of type if yang range expression is 0..max + */ + if (cvtype==CGV_STRING){ + if ((r = malloc(512)) == NULL){ + clicon_err(OE_UNIX, errno, "malloc"); + goto done; + } + snprintf(r, 512, "%d", MAXPATHLEN); } + else if ((r = cvtype_max2str_dup(cvtype)) == NULL){ + clicon_err(OE_UNIX, errno, "cvtype_max2str"); + goto done; + } } cprintf(cb, "%s]", r); free(r); diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index 0948f46b..0c3769d7 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -79,7 +79,7 @@ * Returns an expand-type list of commands as used by cligen 'expand' * functionality. * - * Assume callback given in a cligen spec: a ") * @param[in] h clicon handle * @param[in] name Name of this function (eg "expand_dbvar") * @param[in] cvv The command so far. Eg: cvec [0]:"a 5 b"; [1]: x=5; diff --git a/apps/restconf/restconf_lib.c b/apps/restconf/restconf_lib.c index 819ed211..c8b49512 100644 --- a/apps/restconf/restconf_lib.c +++ b/apps/restconf/restconf_lib.c @@ -296,11 +296,10 @@ readdata(FCGX_Request *r) return cb; } -typedef int (credentials_t)(clicon_handle h, FCGX_Request *r); static int nplugins = 0; static plghndl_t *plugins = NULL; -static credentials_t *p_credentials = NULL; /* Credentials callback */ +static plgcredentials_t *_credentials_fn = NULL; /* Credentials callback */ /*! Load all plugins you can find in CLICON_RESTCONF_DIR */ @@ -330,7 +329,7 @@ restconf_plugin_load(clicon_handle h) (int)strlen(filename), filename); if ((handle = plugin_load(h, filename, RTLD_NOW)) == NULL) goto quit; - p_credentials = dlsym(handle, PLUGIN_CREDENTIALS); + _credentials_fn = dlsym(handle, PLUGIN_CREDENTIALS); if ((plugins = realloc(plugins, (nplugins+1) * sizeof (*plugins))) == NULL) { clicon_err(OE_UNIX, errno, "realloc"); goto quit; @@ -385,23 +384,24 @@ restconf_plugin_start(clicon_handle h, } int -plugin_credentials(clicon_handle h, - FCGX_Request *r, - int *auth) +restconf_credentials(clicon_handle h, + FCGX_Request *r, + char **user) { int retval = -1; clicon_debug(1, "%s", __FUNCTION__); /* If no authentication callback then allow anything. Is this OK? */ - if (p_credentials == 0){ - *auth = 1; + if (_credentials_fn == NULL){ + if ((*user = strdup("none")) == NULL){ + clicon_err(OE_XML, errno, "strdup"); + goto done; + } retval = 0; goto done; } - if (p_credentials(h, r) < 0) - *auth = 0; - else - *auth = 1; + if (_credentials_fn(h, r, user) < 0) + user = NULL; retval = 0; done: return retval; diff --git a/apps/restconf/restconf_lib.h b/apps/restconf/restconf_lib.h index 2813295f..271207ff 100644 --- a/apps/restconf/restconf_lib.h +++ b/apps/restconf/restconf_lib.h @@ -56,7 +56,7 @@ cbuf *readdata(FCGX_Request *r); int restconf_plugin_load(clicon_handle h); int restconf_plugin_start(clicon_handle h, int argc, char **argv); int restconf_plugin_unload(clicon_handle h); -int plugin_credentials(clicon_handle h, FCGX_Request *r, int *auth); +int restconf_credentials(clicon_handle h, FCGX_Request *r, char **user); int get_user_cookie(char *cookiestr, char *attribute, char **val); diff --git a/apps/restconf/restconf_main.c b/apps/restconf/restconf_main.c index b386616b..b0216204 100644 --- a/apps/restconf/restconf_main.c +++ b/apps/restconf/restconf_main.c @@ -142,7 +142,8 @@ api_operations(clicon_handle h, cvec *pcvec, int pi, cvec *qvec, - char *data) + char *data, + char *username) { int retval = -1; char *request_method; @@ -153,7 +154,7 @@ api_operations(clicon_handle h, if (strcmp(request_method, "GET")==0) retval = api_operation_get(h, r, path, pcvec, pi, qvec, data); else if (strcmp(request_method, "POST")==0) - retval = api_operation_post(h, r, path, pcvec, pi, qvec, data); + retval = api_operation_post(h, r, path, pcvec, pi, qvec, data, username); else retval = notfound(r); return retval; @@ -275,7 +276,7 @@ api_restconf(clicon_handle h, cvec *pcvec = NULL; /* for rest api */ cbuf *cb = NULL; char *data; - int auth = 0; + char *username = NULL; clicon_debug(1, "%s", __FUNCTION__); path = FCGX_GetParam("REQUEST_URI", r->envp); @@ -318,20 +319,21 @@ api_restconf(clicon_handle h, retval = 0; test(r, 1); - /* If present, check credentials */ - if (plugin_credentials(h, r, &auth) < 0) + /* If present, check credentials. See "plugin_credentials" in plugin + * See RFC 8040 section 2.5 + */ + if (restconf_credentials(h, r, &username) < 0) goto done; - clicon_debug(1, "%s credentials ok auth:%d (should be 1)", - __FUNCTION__, auth); - if (auth == 0) + clicon_debug(1, "%s credentials ok username:%s (should be non-NULL)", + __FUNCTION__, username); + if (username == NULL) goto done; - clicon_debug(1, "%s credentials ok 2", __FUNCTION__); if (strcmp(method, "yang-library-version")==0) retval = api_yang_library_version(h, r); else if (strcmp(method, "data") == 0) /* restconf, skip /api/data */ retval = api_data(h, r, path, pcvec, 2, qvec, data); else if (strcmp(method, "operations") == 0) /* rpc */ - retval = api_operations(h, r, path, pcvec, 2, qvec, data); + retval = api_operations(h, r, path, pcvec, 2, qvec, data, username); else if (strcmp(method, "test") == 0) retval = test(r, 0); else @@ -348,6 +350,8 @@ api_restconf(clicon_handle h, cvec_free(pcvec); if (cb) cbuf_free(cb); + if (username) + free(username); return retval; } diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index ecec7d17..0248ee7b 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -846,7 +846,8 @@ api_operation_post(clicon_handle h, cvec *pcvec, int pi, cvec *qvec, - char *data) + char *data, + char *username) { int retval = -1; int i; @@ -869,7 +870,8 @@ api_operation_post(clicon_handle h, char *media_accept; int use_xml = 0; /* By default return JSON */ int pretty; - + cxobj *xa; + clicon_debug(1, "%s json:\"%s\" path:\"%s\"", __FUNCTION__, data, path); pretty = clicon_option_bool(h, "CLICON_RESTCONF_PRETTY"); if ((media_accept = FCGX_GetParam("HTTP_ACCEPT", r->envp)) && @@ -939,8 +941,17 @@ api_operation_post(clicon_handle h, } } } - /* Non-standard: add cookie as attribute for backend + /* Non-standard: add username attribute for backend ACM (RFC 6536) + * */ + if (username){ + if ((xa = xml_new("username", xtop, NULL)) == NULL) + goto done; + xml_type_set(xa, CX_ATTR); + if (xml_value_set(xa, username) < 0) + goto done; + } +#ifdef obsolete { cxobj *xa; char *cookie; @@ -957,6 +968,7 @@ api_operation_post(clicon_handle h, free(cookieval); } } +#endif /* Send to backend */ if (clicon_rpc_netconf_xml(h, xtop, &xret, NULL) < 0) goto done; diff --git a/apps/restconf/restconf_methods.h b/apps/restconf/restconf_methods.h index 0aef4428..659a2c6c 100644 --- a/apps/restconf/restconf_methods.h +++ b/apps/restconf/restconf_methods.h @@ -66,6 +66,7 @@ int api_operation_get(clicon_handle h, FCGX_Request *r, int api_operation_post(clicon_handle h, FCGX_Request *r, char *path, - cvec *pcvec, int pi, cvec *qvec, char *data); + cvec *pcvec, int pi, cvec *qvec, char *data, + char *username); #endif /* _RESTCONF_METHODS_H_ */ diff --git a/lib/clixon/clixon_plugin.h b/lib/clixon/clixon_plugin.h index fbb8ffce..22ef53b5 100644 --- a/lib/clixon/clixon_plugin.h +++ b/lib/clixon/clixon_plugin.h @@ -77,7 +77,10 @@ typedef int (plgexit_t)(clicon_handle); /* Plugin exit */ * Returns 0 if credentials OK, -1 if failed */ #define PLUGIN_CREDENTIALS "plugin_credentials" -typedef int (plgcredentials_t)(clicon_handle, void *); /* Plugin credentials */ +/* Plugin credentials + * username should be freed after use + */ +typedef int (plgcredentials_t)(clicon_handle, void *, char **username); /* Find a function in global namespace or a plugin. XXX clicon internal */ void *clicon_find_func(clicon_handle h, char *plugin, char *func); diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index 635498cc..75286ed2 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -1704,6 +1704,7 @@ xml_apply_ancestor(cxobj *xn, * @param[out] cvp CLIgen variable containing the parsed value * @note free cv with cv_free after use. * @see xml_body_int32 etc, for type-specific parse functions + * @note range check failure returns 0 */ int xml_body_parse(cxobj *xb, @@ -1751,6 +1752,7 @@ xml_body_parse(cxobj *xb, * alloc error. * @note extend to all other cligen var types and generalize * @note use yang type info? + * @note range check failure returns 0 */ int xml_body_int32(cxobj *xb, @@ -1774,6 +1776,7 @@ xml_body_int32(cxobj *xb, * alloc error. * @note extend to all other cligen var types and generalize * @note use yang type info? + * @note range check failure returns 0 */ int xml_body_uint32(cxobj *xb, diff --git a/yang/Makefile.in b/yang/Makefile.in index c52badc0..134f67d2 100644 --- a/yang/Makefile.in +++ b/yang/Makefile.in @@ -40,6 +40,7 @@ datarootdir = @datarootdir@ YANGSPECS = clixon-config@2017-12-27.yang YANGSPECS += ietf-netconf@2011-06-01.yang +YANGSPECS += ietf-netconf-acm@2012-02-22.yang YANGSPECS += ietf-inet-types@2013-07-15.yang APPNAME = clixon # subdir ehere these files are installed diff --git a/yang/ietf-netconf-acm@2012-02-22.yang b/yang/ietf-netconf-acm@2012-02-22.yang new file mode 100644 index 00000000..99ad961f --- /dev/null +++ b/yang/ietf-netconf-acm@2012-02-22.yang @@ -0,0 +1,445 @@ +module ietf-netconf-acm { + namespace "urn:ietf:params:xml:ns:yang:ietf-netconf-acm"; + prefix "nacm"; + import ietf-yang-types { + prefix yang; + } + organization + "IETF NETCONF (Network Configuration) Working Group"; + + contact + "WG Web: + WG List: + + WG Chair: Mehmet Ersue + + + WG Chair: Bert Wijnen + + + Editor: Andy Bierman + + + Editor: Martin Bjorklund + "; + + description + "NETCONF Access Control Model. + + Copyright (c) 2012 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Simplified BSD + License set forth in Section 4.c of the IETF Trust's + Legal Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 6536; see + the RFC itself for full legal notices."; + + revision "2012-02-22" { + description + "Initial version"; + reference + "RFC 6536: Network Configuration Protocol (NETCONF) + Access Control Model"; + } + + /* + * Extension statements + */ + + extension default-deny-write { + description + "Used to indicate that the data model node + represents a sensitive security system parameter. + + If present, and the NACM module is enabled (i.e., + /nacm/enable-nacm object equals 'true'), the NETCONF server + will only allow the designated 'recovery session' to have + write access to the node. An explicit access control rule is + required for all other users. + + The 'default-deny-write' extension MAY appear within a data + definition statement. It is ignored otherwise."; + } + + extension default-deny-all { + description + "Used to indicate that the data model node + controls a very sensitive security system parameter. + + If present, and the NACM module is enabled (i.e., + /nacm/enable-nacm object equals 'true'), the NETCONF server + will only allow the designated 'recovery session' to have + read, write, or execute access to the node. An explicit + access control rule is required for all other users. + + The 'default-deny-all' extension MAY appear within a data + definition statement, 'rpc' statement, or 'notification' + statement. It is ignored otherwise."; + } + + /* + * Derived types + */ + + typedef user-name-type { + type string { + length "1..max"; + } + description + "General Purpose Username string."; + } + + typedef matchall-string-type { + type string { + pattern "\*"; + } + description + "The string containing a single asterisk '*' is used + to conceptually represent all possible values + for the particular leaf using this data type."; + } + + typedef access-operations-type { + type bits { + bit create { + description + "Any protocol operation that creates a + new data node."; + } + bit read { + description + "Any protocol operation or notification that + returns the value of a data node."; + } + bit update { + description + "Any protocol operation that alters an existing + data node."; + } + bit delete { + description + "Any protocol operation that removes a data node."; + } + bit exec { + description + "Execution access to the specified protocol operation."; + } + } + description + "NETCONF Access Operation."; + } + + typedef group-name-type { + type string { + length "1..max"; + pattern "[^\*].*"; + } + description + "Name of administrative group to which + users can be assigned."; + } + + typedef action-type { + type enumeration { + enum permit { + description + "Requested action is permitted."; + } + enum deny { + description + "Requested action is denied."; + } + } + description + "Action taken by the server when a particular + rule matches."; + } + + typedef node-instance-identifier { + type yang:xpath1.0; + description + "Path expression used to represent a special + data node instance identifier string. + + A node-instance-identifier value is an + unrestricted YANG instance-identifier expression. + All the same rules as an instance-identifier apply + except predicates for keys are optional. If a key + predicate is missing, then the node-instance-identifier + represents all possible server instances for that key. + + This XPath expression is evaluated in the following context: + + o The set of namespace declarations are those in scope on + the leaf element where this type is used. + + o The set of variable bindings contains one variable, + 'USER', which contains the name of the user of the current + session. + + o The function library is the core function library, but + note that due to the syntax restrictions of an + instance-identifier, no functions are allowed. + + o The context node is the root node in the data tree."; + } + + /* + * Data definition statements + */ + + container nacm { + /* nacm:default-deny-all; XXX How is this parsed ?? */ + + description + "Parameters for NETCONF Access Control Model."; + + leaf enable-nacm { + type boolean; + default true; + description + "Enables or disables all NETCONF access control + enforcement. If 'true', then enforcement + is enabled. If 'false', then enforcement + is disabled."; + } + + leaf read-default { + type action-type; + default "permit"; + description + "Controls whether read access is granted if + no appropriate rule is found for a + particular read request."; + } + + leaf write-default { + type action-type; + default "deny"; + description + "Controls whether create, update, or delete access + is granted if no appropriate rule is found for a + particular write request."; + } + + leaf exec-default { + type action-type; + default "permit"; + description + "Controls whether exec access is granted if no appropriate + rule is found for a particular protocol operation request."; + } + + leaf enable-external-groups { + type boolean; + default true; + description + "Controls whether the server uses the groups reported by the + NETCONF transport layer when it assigns the user to a set of + NACM groups. If this leaf has the value 'false', any group + names reported by the transport layer are ignored by the + server."; + } + + leaf denied-operations { + type yang:zero-based-counter32; + config false; + mandatory true; + description + "Number of times since the server last restarted that a + protocol operation request was denied."; + } + + leaf denied-data-writes { + type yang:zero-based-counter32; + config false; + mandatory true; + description + "Number of times since the server last restarted that a + protocol operation request to alter + a configuration datastore was denied."; + } + + leaf denied-notifications { + type yang:zero-based-counter32; + config false; + mandatory true; + description + "Number of times since the server last restarted that + a notification was dropped for a subscription because + access to the event type was denied."; + } + + container groups { + description + "NETCONF Access Control Groups."; + + list group { + key name; + + description + "One NACM Group Entry. This list will only contain + configured entries, not any entries learned from + any transport protocols."; + + leaf name { + type group-name-type; + description + "Group name associated with this entry."; + } + + leaf-list user-name { + type user-name-type; + description + "Each entry identifies the username of + a member of the group associated with + this entry."; + } + } + } + + list rule-list { + key "name"; + ordered-by user; + description + "An ordered collection of access control rules."; + + leaf name { + type string { + length "1..max"; + } + description + "Arbitrary name assigned to the rule-list."; + } + leaf-list group { + type union { + type matchall-string-type; + type group-name-type; + } + description + "List of administrative groups that will be + assigned the associated access rights + defined by the 'rule' list. + + The string '*' indicates that all groups apply to the + entry."; + } + + list rule { + key "name"; + ordered-by user; + description + "One access control rule. + + Rules are processed in user-defined order until a match is + found. A rule matches if 'module-name', 'rule-type', and + 'access-operations' match the request. If a rule + matches, the 'action' leaf determines if access is granted + or not."; + + leaf name { + type string { + length "1..max"; + } + description + "Arbitrary name assigned to the rule."; + } + + leaf module-name { + type union { + type matchall-string-type; + type string; + } + default "*"; + description + "Name of the module associated with this rule. + + This leaf matches if it has the value '*' or if the + object being accessed is defined in the module with the + specified module name."; + } + choice rule-type { + description + "This choice matches if all leafs present in the rule + match the request. If no leafs are present, the + choice matches all requests."; + case protocol-operation { + leaf rpc-name { + type union { + type matchall-string-type; + type string; + } + description + "This leaf matches if it has the value '*' or if + its value equals the requested protocol operation + name."; + } + } + case notification { + leaf notification-name { + type union { + type matchall-string-type; + type string; + } + description + "This leaf matches if it has the value '*' or if its + value equals the requested notification name."; + } + } + case data-node { + leaf path { + type node-instance-identifier; + mandatory true; + description + "Data Node Instance Identifier associated with the + data node controlled by this rule. + + Configuration data or state data instance + identifiers start with a top-level data node. A + complete instance identifier is required for this + type of path value. + + The special value '/' refers to all possible + datastore contents."; + } + } + } + + leaf access-operations { + type union { + type matchall-string-type; + type access-operations-type; + } + default "*"; + description + "Access operations associated with this rule. + + This leaf matches if it has the value '*' or if the + bit corresponding to the requested operation is set."; + } + + leaf action { + type action-type; + mandatory true; + description + "The access control action associated with the + rule. If a rule is determined to match a + particular request, then this object is used + to determine whether to permit or deny the + request."; + } + + leaf comment { + type string; + description + "A textual description of the access rule."; + } + } + } + } +} \ No newline at end of file From 55010e7541c7d4ea28b01b1ffcc3a1b6102dcb87 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Fri, 9 Feb 2018 12:18:39 +0700 Subject: [PATCH 26/28] restconf credentials plugin --- apps/backend/backend_client.c | 2 +- apps/restconf/restconf_lib.c | 132 ++++++++++++++++++++++++------- apps/restconf/restconf_lib.h | 9 ++- apps/restconf/restconf_main.c | 32 +++++--- apps/restconf/restconf_methods.c | 9 +++ lib/src/clixon_xml_map.c | 5 +- 6 files changed, 141 insertions(+), 48 deletions(-) diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 0fd9fad6..97b0ae79 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -1095,8 +1095,8 @@ from_client(int s, goto done; retval = 0; done: + clicon_debug(1, "%s retval=%d", __FUNCTION__, retval); if (msg) free(msg); - clicon_debug(1, "%s retval=%d", __FUNCTION__, retval); return retval; /* -1 here terminates backend */ } diff --git a/apps/restconf/restconf_lib.c b/apps/restconf/restconf_lib.c index c8b49512..15c868ca 100644 --- a/apps/restconf/restconf_lib.c +++ b/apps/restconf/restconf_lib.c @@ -148,7 +148,60 @@ restconf_code2reason(int code) return clicon_int2str(http_reason_phrase_map, code); } -/*! +/*! HTTP error 400 + * @param[in] r Fastcgi request handle + */ +int +badrequest(FCGX_Request *r) +{ + char *path; + + clicon_debug(1, "%s", __FUNCTION__); + path = FCGX_GetParam("DOCUMENT_URI", r->envp); + FCGX_FPrintF(r->out, "Status: 400\r\n"); /* 400 bad request */ + FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); + FCGX_FPrintF(r->out, "

Clixon Bad request/h1>\n"); + FCGX_FPrintF(r->out, "The requested URL %s or data is in some way badly formed.\n", + path); + return 0; +} + +/*! HTTP error 401 + * @param[in] r Fastcgi request handle + */ +int +unauthorized(FCGX_Request *r) +{ + char *path; + + clicon_debug(1, "%s", __FUNCTION__); + path = FCGX_GetParam("DOCUMENT_URI", r->envp); + FCGX_FPrintF(r->out, "Status: 401\r\n"); /* 401 unauthorized */ + FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); + FCGX_FPrintF(r->out, "access-denied\n"); + FCGX_FPrintF(r->out, "The requested URL %s was unauthorized.\n", path); + return 0; +} + +/*! HTTP error 403 + * @param[in] r Fastcgi request handle + */ +int +forbidden(FCGX_Request *r) +{ + char *path; + + clicon_debug(1, "%s", __FUNCTION__); + path = FCGX_GetParam("DOCUMENT_URI", r->envp); + FCGX_FPrintF(r->out, "Status: 403\r\n"); /* 403 forbidden */ + FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); + FCGX_FPrintF(r->out, "

Grideye Forbidden

\n"); + FCGX_FPrintF(r->out, "The requested URL %s was forbidden.\n", path); + return 0; +} + +/*! HTTP error 404 + * @param[in] r Fastcgi request handle */ int notfound(FCGX_Request *r) @@ -165,31 +218,9 @@ notfound(FCGX_Request *r) return 0; } -int -badrequest(FCGX_Request *r) -{ - char *path; - - clicon_debug(1, "%s", __FUNCTION__); - path = FCGX_GetParam("DOCUMENT_URI", r->envp); - FCGX_FPrintF(r->out, "Status: 400\r\n"); /* 400 bad request */ - FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); - FCGX_FPrintF(r->out, "

Clixon Bad request/h1>\n"); - FCGX_FPrintF(r->out, "The requested URL %s or data is in some way badly formed.\n", - path); - return 0; -} - -int -notimplemented(FCGX_Request *r) -{ - clicon_debug(1, "%s", __FUNCTION__); - FCGX_FPrintF(r->out, "Status: 501\r\n"); - FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); - FCGX_FPrintF(r->out, "

Not Implemented/h1>\n"); - return 0; -} - +/*! HTTP error 409 + * @param[in] r Fastcgi request handle + */ int conflict(FCGX_Request *r) { @@ -200,6 +231,35 @@ conflict(FCGX_Request *r) return 0; } +/*! HTTP error 500 + * @param[in] r Fastcgi request handle + */ +int +internal_server_error(FCGX_Request *r) +{ + char *path; + + clicon_debug(1, "%s", __FUNCTION__); + path = FCGX_GetParam("DOCUMENT_URI", r->envp); + FCGX_FPrintF(r->out, "Status: 500\r\n"); /* 500 internal server error */ + FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); + FCGX_FPrintF(r->out, "

Grideye Internal server error when accessing %s

\n", path); + return 0; +} + +/*! HTTP error 501 + * @param[in] r Fastcgi request handle + */ +int +notimplemented(FCGX_Request *r) +{ + clicon_debug(1, "%s", __FUNCTION__); + FCGX_FPrintF(r->out, "Status: 501\r\n"); + FCGX_FPrintF(r->out, "Content-Type: text/html\r\n\r\n"); + FCGX_FPrintF(r->out, "

Not Implemented/h1>\n"); + return 0; +} + /*! Specialization of clicon_debug with xml tree */ int clicon_debug_xml(int dbglevel, @@ -329,7 +389,10 @@ restconf_plugin_load(clicon_handle h) (int)strlen(filename), filename); if ((handle = plugin_load(h, filename, RTLD_NOW)) == NULL) goto quit; - _credentials_fn = dlsym(handle, PLUGIN_CREDENTIALS); + if ((_credentials_fn = dlsym(handle, PLUGIN_CREDENTIALS)) == NULL) + clicon_debug(1, "Failed to load %s", PLUGIN_CREDENTIALS); + else + clicon_debug(1, "%s callback loaded", PLUGIN_CREDENTIALS); if ((plugins = realloc(plugins, (nplugins+1) * sizeof (*plugins))) == NULL) { clicon_err(OE_UNIX, errno, "realloc"); goto quit; @@ -383,6 +446,14 @@ restconf_plugin_start(clicon_handle h, return 0; } +/*! Run the restconf user-defined credentials callback if present + * The callback is expected to return the authenticated user, or NULL if not + * authenticasted. + * If no callback exists, return user "none" + * @param[in] h Clicon handle + * @param[in] r Fastcgi request handle + * @param[out] user The authenticated user (or NULL). Malloced, must be freed. + */ int restconf_credentials(clicon_handle h, FCGX_Request *r, @@ -397,13 +468,14 @@ restconf_credentials(clicon_handle h, clicon_err(OE_XML, errno, "strdup"); goto done; } - retval = 0; - goto done; + goto ok; } if (_credentials_fn(h, r, user) < 0) - user = NULL; + *user = NULL; + ok: retval = 0; done: + clicon_debug(1, "%s retval:%d user:%s", __FUNCTION__, retval, *user); return retval; } diff --git a/apps/restconf/restconf_lib.h b/apps/restconf/restconf_lib.h index 271207ff..59e381f4 100644 --- a/apps/restconf/restconf_lib.h +++ b/apps/restconf/restconf_lib.h @@ -45,10 +45,15 @@ */ int restconf_err2code(char *tag); const char *restconf_code2reason(int code); -int notfound(FCGX_Request *r); + int badrequest(FCGX_Request *r); -int notimplemented(FCGX_Request *r); +int unauthorized(FCGX_Request *r); +int forbidden(FCGX_Request *r); +int notfound(FCGX_Request *r); int conflict(FCGX_Request *r); +int internal_server_error(FCGX_Request *r); +int notimplemented(FCGX_Request *r); + int clicon_debug_xml(int dbglevel, char *str, cxobj *cx); int test(FCGX_Request *r, int dbg); cbuf *readdata(FCGX_Request *r); diff --git a/apps/restconf/restconf_main.c b/apps/restconf/restconf_main.c index b0216204..54b422ed 100644 --- a/apps/restconf/restconf_main.c +++ b/apps/restconf/restconf_main.c @@ -317,27 +317,37 @@ api_restconf(clicon_handle h, if (str2cvec(data, '&', '=', &dvec) < 0) goto done; - retval = 0; test(r, 1); /* If present, check credentials. See "plugin_credentials" in plugin * See RFC 8040 section 2.5 */ if (restconf_credentials(h, r, &username) < 0) goto done; + clicon_debug(1, "%s username:%s", __FUNCTION__, username); clicon_debug(1, "%s credentials ok username:%s (should be non-NULL)", __FUNCTION__, username); - if (username == NULL) - goto done; - if (strcmp(method, "yang-library-version")==0) - retval = api_yang_library_version(h, r); - else if (strcmp(method, "data") == 0) /* restconf, skip /api/data */ - retval = api_data(h, r, path, pcvec, 2, qvec, data); - else if (strcmp(method, "operations") == 0) /* rpc */ - retval = api_operations(h, r, path, pcvec, 2, qvec, data, username); + if (username == NULL){ + unauthorized(r); + goto ok; + } + if (strcmp(method, "yang-library-version")==0){ + if (api_yang_library_version(h, r) < 0) + goto done; + } + else if (strcmp(method, "data") == 0){ /* restconf, skip /api/data */ + if (api_data(h, r, path, pcvec, 2, qvec, data) < 0) + goto done; + } + else if (strcmp(method, "operations") == 0){ /* rpc */ + if (api_operations(h, r, path, pcvec, 2, qvec, data, username) < 0) + goto done; + } else if (strcmp(method, "test") == 0) - retval = test(r, 0); + test(r, 0); else - retval = notfound(r); + notfound(r); + ok: + retval = 0; done: clicon_debug(1, "%s retval:%d", __FUNCTION__, retval); if (pvec) diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 0248ee7b..6be25d41 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -906,8 +906,17 @@ api_operation_post(clicon_handle h, if ((xtop = xml_new("rpc", NULL, NULL)) == NULL) goto done; xbot = xtop; + /* XXX: something strange for rpc user */ if (api_path2xml(oppath, yspec, xtop, 1, &xbot, &y) < 0) goto done; +#if 1 + { + cbuf *c = cbuf_new(); + clicon_xml2cbuf(c, xtop, 0, 0); + clicon_debug(1, "%s xinput:%s", __FUNCTION__, cbuf_get(c)); + cbuf_free(c); + } +#endif if (data && strlen(data)){ /* Parse input data as json or xml into xml */ if (parse_xml){ diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index dd913329..105b8661 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -1500,11 +1500,9 @@ api_path2xml_vec(char **vec, name = local; } if (y0->yn_keyword == Y_SPEC){ /* top-node */ - clicon_debug(1, "%s 1 %s", __FUNCTION__, name); y = yang_find_topnode((yang_spec*)y0, name, schemanode); } else { - clicon_debug(1, "%s 2 %s", __FUNCTION__, name); y = schemanode?yang_find_schemanode((yang_node*)y0, name): yang_find_datanode((yang_node*)y0, name); } @@ -1593,7 +1591,6 @@ api_path2xml_vec(char **vec, * @param[in] schemanode If set use schema nodes otherwise data nodes. * @param[out] xbotp Resulting xml tree (end of xpath) * @param[out] ybotp Yang spec matching xbotp - * @see api_path2xml_vec * @example * api_path: /subif-entry=foo/subid * xtop[in] @@ -1602,6 +1599,7 @@ api_path2xml_vec(char **vec, * * xbotp: * ybotp: Y_LEAF subid + * @see api_path2xml_vec */ int api_path2xml(char *api_path, @@ -1615,7 +1613,6 @@ api_path2xml(char *api_path, char **vec = NULL; int nvec; - clicon_debug(1, "%s 0", __FUNCTION__); if (*api_path!='/'){ clicon_err(OE_DB, 0, "Invalid key: %s", api_path); goto done; From 6b0e0a9d1819dc3e07b9bdef2e850aef9d3319e7 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sun, 11 Feb 2018 16:46:10 +0700 Subject: [PATCH 27/28] preparations for 3.5.0 --- CHANGELOG.md | 33 ++++++++++++++++++--------------- test/lib.sh | 12 ++++++++++++ test/test_cli.sh | 6 +----- test/test_leafref.sh | 5 ----- test/test_netconf.sh | 5 ----- test/test_order.sh | 3 +-- test/test_perf.sh | 6 ------ test/test_startup.sh | 5 ----- test/test_type.sh | 6 ------ test/test_yang.sh | 5 ----- 10 files changed, 32 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25cedf06..4afe193a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,32 +3,35 @@ ## 3.5.0 (Upcoming) ### Major changes: -* Added a "user" parameter to plugin_credentials() restconf callback. To enable authentication and in preparation for access control a la RFC 6536. * Major Restconf feature update to compy to RFC 8040. Thanks Stephen Jones for getting right. - * GET well-known, top-level resource, yang library version, - * PUT whole datastore, check for different keys in put lists. + * GET: Always return object referenced (and nothing else). ie, GET /restconf/data/X returns X. + * GET Added support for the following resources: Well-known, top-level resource, and yang library version, * GET Single element JSON lists use {list:[element]}, not {list:element}. + * PUT Whole datastore ### Minor changes: +* Changed signature of plugin_credentials() restconf callback. Added a "user" parameter. To enable authentication and in preparation for access control a la RFC 6536. * Added RFC 6536 ietf-netconf-acm@2012-02-22.yang access control (but not implemented). -* The following backward compatible options to configure have been obsoleted. If you havent already migrated this code you must do this now. - * Backend startup modes prior to 3.3.3. As enabled with `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted. - * Configuration files (non-XML) prior to 3.3.3. As enabled with `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. - * Clixon XML C-lib prior to 3.4.0. As enabled with `configure --with-xml-compat` +* The following backward compatible options to configure have been _obsoleted_. If you havent already migrated this code you must do this now. + * `configure --with-startup-compat`. Configure option CLICON_USE_STARTUP_CONFIG is also obsoleted. + * `configure --with-config-compat`. The template clicon.conf.cpp files are also removed. + * `configure --with-xml-compat` -* New configuration option: CLICON_RESTCONF_PRETTY -* Changed restconf GET to return object referenced. ie, GET /restconf/data/X returns X. Thanks Stephen Jones for getting this right. +* New configuration option: CLICON_RESTCONF_PRETTY. Default true. Set to false to get more compact Restconf output. -* Default configure file added by Matt Smith. Config file is selected in the following priority order: - * Provide -f option when starting a program. + +* Default configure file handling generalized by Renato Botelho/Matt Smith. Config file FILE is selected in the following priority order: + * Provide -f FILE option when starting a program (eg clixon_backend -F FILE) * Provide --with-configfile=FILE when configuring - * /etc/clixon.xml + * Provide --with-sysconfig= when configuring, then FILE is /clixon.xml + * Provide --sysconfig= when configuring then FILE is /etc/clixon.xml + * FILE is /usr/local/etc/clixon.xml ### Corrected Bugs -* yang string length "max" keyword set to MAXPATHLEN -* Corrected "No yang spec" printed on tty on leafref CLI usage -* xml2cvec: range error (eg 1000 for int8) is not treated as error, just log and skip. +* yang max keyword was not supported for string type. Corrected by setting "max" to MAXPATHLEN +* Corrected "No yang spec" printed on tty when using leafref in CLI. +* Fixed error in xml2cvec. If a (for example) int8 value has range error (eg 1000), it was treated as an error and the program terminated. Now this is just logged and skipped. Reported by Fredrik Pettai. ### Known issues diff --git a/test/lib.sh b/test/lib.sh index 00eec2d7..8f991c39 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -5,6 +5,18 @@ testnr=0 testname= +# For memcheck +#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli" +clixon_cli=clixon_cli + +# For memcheck / performance +#clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" +# clixon_netconf="valgrind --tool=callgrind clixon_netconf +clixon_netconf=clixon_netconf + +#clixon_backend="valgrind --leak-check=full --show-leak-kinds=all clixon_backend" +clixon_backend=clixon_backend + dir=/var/tmp/$0 if [ ! -d $dir ]; then mkdir $dir diff --git a/test/test_cli.sh b/test/test_cli.sh index 75f596eb..496b9d95 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -28,10 +28,6 @@ cat < $cfg EOF -# For memcheck -#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli" -clixon_cli=clixon_cli - # kill old backend (if any) new "kill old backend" sudo clixon_backend -z -f $cfg @@ -39,7 +35,7 @@ if [ $? -ne 0 ]; then err fi new "start backend -s init -f $cfg" -sudo clixon_backend -s init -f $cfg +sudo $clixon_backend -s init -f $cfg if [ $? -ne 0 ]; then err fi diff --git a/test/test_leafref.sh b/test/test_leafref.sh index f31893af..bb5649ff 100755 --- a/test/test_leafref.sh +++ b/test/test_leafref.sh @@ -22,11 +22,6 @@ cat < $cfg EOF -# For memcheck -# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -clixon_netconf=clixon_netconf -clixon_cli=clixon_cli - cat < $fyang module example{ import ietf-ip { diff --git a/test/test_netconf.sh b/test/test_netconf.sh index 5b6798ec..78070864 100755 --- a/test/test_netconf.sh +++ b/test/test_netconf.sh @@ -25,11 +25,6 @@ cat < $cfg EOF - -# For memcheck -#clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -clixon_netconf=clixon_netconf - echo "clixon_backend -zf $cfg" # kill old backend (if any) new "kill old backend" diff --git a/test/test_order.sh b/test/test_order.sh index 433ee9bd..d29ad4f2 100755 --- a/test/test_order.sh +++ b/test/test_order.sh @@ -13,8 +13,7 @@ fyang=$dir/order.yang # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -clixon_netconf=clixon_netconf -clixon_cli=clixon_cli + dbdir=$dir/order new "Set up $dbdir" diff --git a/test/test_perf.sh b/test/test_perf.sh index ff3efa48..ece6a3b6 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -22,12 +22,6 @@ cfg=$dir/scaling-conf.xml fyang=$dir/scaling.yang fconfig=$dir/config - -# For memcheck -# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -# clixon_netconf="valgrind --tool=callgrind clixon_netconf -clixon_netconf=clixon_netconf - cat < $fyang module ietf-ip{ container x { diff --git a/test/test_startup.sh b/test/test_startup.sh index 53f7c257..c81f88b9 100755 --- a/test/test_startup.sh +++ b/test/test_startup.sh @@ -10,11 +10,6 @@ . ./lib.sh cfg=$dir/conf_startup.xml -# For memcheck -# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -clixon_netconf=clixon_netconf -clixon_cli=clixon_cli - cat < $cfg $cfg diff --git a/test/test_type.sh b/test/test_type.sh index 432d30cb..dcc2288b 100755 --- a/test/test_type.sh +++ b/test/test_type.sh @@ -23,12 +23,6 @@ cat < $cfg EOF - -# For memcheck -#clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli" -clixon_cli=clixon_cli -clixon_netconf=clixon_netconf - cat < $fyang module example{ typedef ab { diff --git a/test/test_yang.sh b/test/test_yang.sh index c2cdead5..15eeba11 100755 --- a/test/test_yang.sh +++ b/test/test_yang.sh @@ -7,11 +7,6 @@ cfg=$dir/conf_yang.xml fyang=$dir/test.yang -# For memcheck -# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" -clixon_netconf=clixon_netconf -clixon_cli=clixon_cli - cat < $cfg $cfg From 7431757e531d9f05cf9d9b7e77cf6f0bde0f9c74 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Mon, 12 Feb 2018 09:05:27 +0700 Subject: [PATCH 28/28] Preparing for 3.5.0 --- apps/restconf/restconf_methods.c | 5 +- lib/src/clixon_yang.c | 2 +- test/lib.sh | 5 +- test/test_restconf2.sh | 1 - yang/Makefile.in | 2 +- yang/clixon-config@2017-12-27.yang | 15 +- yang/clixon-config@2018-02-12.yang | 304 +++++++++++++++++++++++++++++ 7 files changed, 314 insertions(+), 20 deletions(-) create mode 100644 yang/clixon-config@2018-02-12.yang diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 6be25d41..69bae763 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -652,11 +652,12 @@ api_data_put(clicon_handle h, xml_type_set(xa, CX_ATTR); if (xml_value_set(xa, xml_operation2str(op)) < 0) goto done; -#if 1 /* This is different from POST */ /* Replace xparent with x, ie bottom of api-path with data */ if (api_path==NULL && strcmp(xml_name(x),"data")==0){ if (xml_addsub(NULL, x) < 0) goto done; + if (xtop) + xml_free(xtop); xtop = x; xml_name_set(xtop, "config"); } @@ -678,7 +679,7 @@ api_data_put(clicon_handle h, if (xml_addsub(xparent, x) < 0) goto done; } -#endif + /* Create text buffer for transfer to backend */ if ((cbx = cbuf_new()) == NULL) goto done; diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index 2dcb578d..1014b0f0 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -1451,7 +1451,7 @@ yang_parse_file(clicon_handle h, FILE *f = NULL; struct stat st; - clicon_debug(1, "Yang parse file: %s", filename); + clicon_log(LOG_DEBUG, "Parsing yang file: %s", filename); if (stat(filename, &st) < 0){ clicon_err(OE_YANG, errno, "%s not found", filename); goto done; diff --git a/test/lib.sh b/test/lib.sh index 8f991c39..2872126e 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -14,7 +14,10 @@ clixon_cli=clixon_cli # clixon_netconf="valgrind --tool=callgrind clixon_netconf clixon_netconf=clixon_netconf -#clixon_backend="valgrind --leak-check=full --show-leak-kinds=all clixon_backend" +# How to run restconf stand-alone and using valgrind +#sudo su -c "/www-data/clixon_restconf -f $cfg -D" -s /bin/sh www-data +#sudo su -c "valgrind --leak-check=full --show-leak-kinds=all /www-data/clixon_restconf -f $cfg -D" -s /bin/sh www-data + clixon_backend=clixon_backend dir=/var/tmp/$0 diff --git a/test/test_restconf2.sh b/test/test_restconf2.sh index e9d709f1..f0ae519e 100755 --- a/test/test_restconf2.sh +++ b/test/test_restconf2.sh @@ -93,7 +93,6 @@ new "restconf POST initial tree" expectfn 'curl -s -X POST -d {"interfaces-config":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' "" new "restconf PUT initial datastore" - expectfn 'curl -s -X PUT -d {"data":{"interfaces-config":{"interface":{"name":"local0","type":"regular"}}}} http://localhost/restconf/data' "" new "restconf GET datastore" diff --git a/yang/Makefile.in b/yang/Makefile.in index 134f67d2..cd3f988c 100644 --- a/yang/Makefile.in +++ b/yang/Makefile.in @@ -38,7 +38,7 @@ bindir = @bindir@ includedir = @includedir@ datarootdir = @datarootdir@ -YANGSPECS = clixon-config@2017-12-27.yang +YANGSPECS = clixon-config@2018-02-12.yang YANGSPECS += ietf-netconf@2011-06-01.yang YANGSPECS += ietf-netconf-acm@2012-02-22.yang YANGSPECS += ietf-inet-types@2013-07-15.yang diff --git a/yang/clixon-config@2017-12-27.yang b/yang/clixon-config@2017-12-27.yang index d9190da7..1bbc1dcb 100644 --- a/yang/clixon-config@2017-12-27.yang +++ b/yang/clixon-config@2017-12-27.yang @@ -11,7 +11,7 @@ module clixon-config { description "Clixon configuration file ***** BEGIN LICENSE BLOCK ***** - Copyright (C) 2009-2018 Olof Hagsand and Benny Holmgren + Copyright (C) 2009-2017 Olof Hagsand and Benny Holmgren This file is part of CLIXON @@ -130,19 +130,6 @@ module clixon-config { "FastCGI unix socket. Should be specified in webserver Eg in nginx: fastcgi_pass unix:/www-data/clicon_restconf.sock"; } - leaf CLICON_RESTCONF_PRETTY { - type boolean; - default true; - description - "Restconf return value pretty print. - Restconf clients may add HTTP header: - Accept: application/yang-data+json, or - Accept: application/yang-data+xml - to get return value in XML or JSON. - RFC 8040 examples print XML and JSON in pretty-printed form. - Setting this value to false makes restconf return not pretty-printed - which may be desirable for performance or tests"; - } leaf CLICON_CLI_DIR { type string; description diff --git a/yang/clixon-config@2018-02-12.yang b/yang/clixon-config@2018-02-12.yang new file mode 100644 index 00000000..883a002f --- /dev/null +++ b/yang/clixon-config@2018-02-12.yang @@ -0,0 +1,304 @@ +module clixon-config { + + prefix cc; + + organization + "Clicon / Clixon"; + + contact + "Olof Hagsand "; + + description + "Clixon configuration file + ***** BEGIN LICENSE BLOCK ***** + Copyright (C) 2009-2018 Olof Hagsand and Benny Holmgren + + This file is part of CLIXON + + Licensed under the Apache License, Version 2.0 (the \"License\"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an \"AS IS\" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the \"GPL\"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK *****"; + + revision 2018-02-12 { + description + "Added pretty print for datastore"; + } + typedef startup_mode{ + description + "Which method to boot/start clicon backend. + The methods differ in how they reach a running state + Which source database to commit from, if any."; + type enumeration{ + enum none{ + description + "Do not touch running state + Typically after crash when running state and db are synched"; + } + enum init{ + description + "Initialize running state. + Start with a completely clean running state"; + } + enum running{ + description + "Commit running db configuration into running state + After reboot if a persistent running db exists"; + } + enum startup{ + description + "Commit startup configuration into running state + After reboot when no persistent running db exists"; + } + } + } + typedef xmldb_format{ + description + "Format of TEXT xml database format."; + type enumeration{ + enum xml{ + description "Save and load xmldb as XML"; + } + enum json{ + description "Save and load xmldb as JSON"; + } + } + } + container config { + leaf CLICON_CONFIGFILE{ + type string; + description + "Location of configuration-file for default values (this file)"; + } + leaf CLICON_YANG_DIR { + type string; + mandatory true; + description + "Location of YANG module and submodule files."; + } + leaf CLICON_YANG_MODULE_MAIN { + type string; + default "clicon"; + description + "Option used to construct initial yang file: + [@]"; + } + leaf CLICON_YANG_MODULE_REVISION { + type string; + description + "Option used to construct initial yang file: + [@]"; + } + leaf CLICON_BACKEND_DIR { + type string; + description + "Location of backend .so plugins. Load all .so + plugins in this dir as backend plugins"; + } + leaf CLICON_NETCONF_DIR { + type string; + description "Location of netconf (frontend) .so plugins"; + } + leaf CLICON_RESTCONF_DIR { + type string; + description + "Location of restconf (frontend) .so plugins. Load all .so + plugins in this dir as restconf code plugins"; + } + leaf CLICON_RESTCONF_PATH { + type string; + default "/www-data/fastcgi_restconf.sock"; + description + "FastCGI unix socket. Should be specified in webserver + Eg in nginx: fastcgi_pass unix:/www-data/clicon_restconf.sock"; + } + leaf CLICON_RESTCONF_PRETTY { + type boolean; + default true; + description + "Restconf return value pretty print. + Restconf clients may add HTTP header: + Accept: application/yang-data+json, or + Accept: application/yang-data+xml + to get return value in XML or JSON. + RFC 8040 examples print XML and JSON in pretty-printed form. + Setting this value to false makes restconf return not pretty-printed + which may be desirable for performance or tests"; + } + leaf CLICON_CLI_DIR { + type string; + description + "Location of cli frontend .so plugins. Load all .so + plugins in this dir as CLI object plugins"; + } + leaf CLICON_CLISPEC_DIR { + type string; + description + "Location of frontend .cli cligen spec files. Load all .cli + files in this dir as CLI specification files"; + } + leaf CLICON_CLISPEC_FILE { + type string; + description "Specific frontend .cli cligen spec file."; + } + leaf CLICON_CLI_MODE { + type string; + default "base"; + description + "Startup CLI mode. This should match a CLICON_MODE set in + one of the clispec files"; + } + leaf CLICON_CLI_GENMODEL { + type int32; + default 1; + description + "Generate code for CLI completion of existing db symbols. + Example: Add name=\"myspec\" in datamodel spec and reference + as @myspec"; + } + leaf CLICON_CLI_GENMODEL_COMPLETION { + type int32; + default 1; + description "Generate code for CLI completion of existing db symbols"; + } + leaf CLICON_CLI_GENMODEL_TYPE { + type string; + default "VARS"; + description "How to generate and show CLI syntax: VARS|ALL"; + } + leaf CLICON_CLI_VARONLY { + type int32; + default 1; + description + "Dont include keys in cvec in cli vars callbacks, + ie a & k in 'a k ' ignored"; + } + leaf CLICON_CLI_LINESCROLLING { + type int32; + default 1; + description + "Set to 0 if you want CLI to wrap to next line. + Set to 1 if you want CLI to scroll sideways when approaching + right margin"; + } + leaf CLICON_SOCK_FAMILY { + type string; + default "UNIX"; + description + "Address family for communicating with clixon_backend + (UNIX|IPv4|IPv6)"; + } + leaf CLICON_SOCK { + type string; + mandatory true; + description + "If family above is AF_UNIX: Unix socket for communicating + with clixon_backend. If family is AF_INET: IPv4 address"; + } + leaf CLICON_SOCK_PORT { + type int32; + default 4535; + description + "Inet socket port for communicating with clixon_backend + (only IPv4|IPv6)"; + } + leaf CLICON_SOCK_GROUP { + type string; + default "clicon"; + description "Group membership to access clixon_backend unix socket"; + } + leaf CLICON_BACKEND_PIDFILE { + type string; + mandatory true; + description "Process-id file of backend daemon"; + } + leaf CLICON_AUTOCOMMIT { + type int32; + default 0; + description + "Set if all configuration changes are committed automatically + on every edit change. Explicit commit commands unnecessary"; + } + leaf CLICON_MASTER_PLUGIN { + type string; + default "master"; + description + "Name of master plugin (both frontend and backend). + Master plugin has special callbacks for frontends. + See clicon user manual for more info. (Obsolete?)"; + } + leaf CLICON_XMLDB_DIR { + type string; + mandatory true; + description + "Directory where \"running\", \"candidate\" and \"startup\" are placed"; + } + leaf CLICON_XMLDB_PLUGIN { + type string; + mandatory true; + description + "XMLDB datastore plugin filename + (see datastore/ and clixon_xml_db.[ch])"; + } + leaf CLICON_XMLDB_CACHE { + type boolean; + default true; + description + "XMLDB datastore cache. + If set, XML candidate/running parsed tree is stored in memory + If not set, candidate/running is always accessed via disk."; + } + leaf CLICON_XMLDB_FORMAT { + type xmldb_format; + default xml; + description "XMLDB datastore format."; + } + leaf CLICON_XMLDB_PRETTY { + type boolean; + default true; + description + "XMLDB datastore pretty print. + If set, insert spaces and line-feeds making the XML/JSON human + readable. If not set, make the XML/JSON more compact."; + } + leaf CLICON_XML_SORT { + type boolean; + default true; + description + "If set, sort XML lists and leaf-lists alphabetically and uses binary + search. Unless ordered-by user is used. + Only works for Yang specified XML. + If not set, all lists accessed via linear search."; + } + leaf CLICON_USE_STARTUP_CONFIG { + type int32; + default 0; + description + "Enabled uses \"startup\" configuration on boot. It is called + startup_db and exists in XMLDB_DIR. + NOTE: Obsolete with 1.3.3 and CLICON_STARTUP_MODE"; + } + leaf CLICON_STARTUP_MODE { + type startup_mode; + description "Which method to boot/start clicon backend"; + } + } +}