diff --git a/.gitignore b/.gitignore index c191c5b0..4927378c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ docker/Makefile docker/*/Makefile etc/Makefile example/Makefile +example/*/Makefile lib/Makefile lib/*/Makefile test/Makefile diff --git a/CHANGELOG.md b/CHANGELOG.md index 652a597d..1aaa51a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,19 @@ * Two config options control: * CLICON_XML_CHANGELOG enables the yang changelog feature * CLICON_XML_CHANGELOG_FILE where the changelog resides +* Optimization work + * Improved performance of validation of (large) lists + * A scaling of [large lists](doc/scaling) report is added + * New xmldb_get1() returning actual cache - not a copy. This has lead to some householding instead of just deleting the copy + * xml_diff rewritten to work linearly instead of O(2) + * New xml_insert function using tree search. The new code uses this in insertion xmldb_put and defaults. (Note previous xml_insert renamed to xml_wrap_all) + * A yang type regex cache added, this helps the performance by avoiding re-running the `regcomp` command on every iteration. + * An XML namespace cache added (see `xml2ns()`) + * Better performance of XML whitespace parsing/scanning. ### API changes on existing features (you may need to change your code) +* The directory `docker/system` has been moved to `docker/main`, to reflect that it runs the main example. * xmldb_get() removed "config" parameter: * Change all calls to dbget from: `xmldb_get(h, db, xpath, 0|1, &xret, msd)` to `xmldb_get(h, db, xpath, &xret, msd)` * Structural change: removed datastore plugin and directory, and merged into regular clixon lib code. @@ -51,9 +61,8 @@ * Change all y->ys_keyword to yang_keyword_get(y) * Change all y->ys_argument to yang_argument_get(y) * Change all y->ys_cv to yang_cv_get(y) - * Change all y->ys_cvec to yang_cvec_get(y) + * Change all y->ys_cvec to yang_cvec_get(y) or yang_cvec_set(y, cvv) * Removed external direct access to the yang_stmt struct. - * xmldb_get() removed unnecessary config option: * Change all calls to dbget from: `xmldb_get(h, db, xpath, 0|1, &xret, msd)` to `xmldb_get(h, db, xpath, &xret, msd)` @@ -101,11 +110,12 @@ ``` ### Minor changes + +* A new "hello world" example is added * Experimental customized error output strings, see [lib/clixon/clixon_err_string.h] * Empty leaf values, eg are now checked at validation. * Empty values were skipped in validation. * They are now checked and invalid for ints, dec64, etc, but are treated as empty string "" for string types. -* Optimized validation by making xml_diff work on raw cache tree (not copies) * Added syntactic check for yang status: current, deprecated or obsolete. * Added `xml_wrap` function that adds an XML node above a node as a wrapper * also renamed `xml_insert` to `xml_wrap_all`. @@ -128,6 +138,8 @@ * Added libgen.h for baseline() ### Corrected Bugs +* Failure in startup with -m startup or running left running_db cleared. + * Running-db should not be changed on failure. Unless failure-db defined. Or if SEGV, etc. In those cases, tmp_db should include the original running-db. * Backend plugin returning NULL was still installed - is now logged and skipped. * [Parent list key is not validated if not provided via RESTCONF #83](https://github.com/clicon/clixon/issues/83), thanks achernavin22. * [Invalid JSON if GET /operations via RESTCONF #82](https://github.com/clicon/clixon/issues/82), thanks achernavin22 diff --git a/README.md b/README.md index af2b3b3b..b26a32d2 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ support. * [Background](#background) * [Frequently asked questions (FAQ)](doc/FAQ.md) + * [Hello world](example/hello/README.md) * [Changelog](CHANGELOG.md) * [Installation](#installation) * [Licenses](#licenses) @@ -26,6 +27,7 @@ support. * [Runtime](#runtime) * [Clixon project page](http://www.clicon.org) * [Tests and CI](test/README.md) + * [Scaling: large lists](doc/scaling/large-lists.md) * [Containers](docker/README.md) * [Roadmap](doc/ROADMAP.md) * [Reference manual](#reference) diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 22839104..3565ea83 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -146,7 +146,7 @@ generic_validate(yang_stmt *yspec, * and call application callback validations. * @param[in] h Clicon handle * @param[in] db The startup database. The wanted backend state - * @param[out] xtr Transformed XML + * @param[in] td Transaction * @param[out] cbret CLIgen buffer w error stmt if retval = 0 * @retval -1 Error - or validation failed (but cbret not set) * @retval 0 Validation failed (with cbret set) @@ -180,8 +180,13 @@ startup_common(clicon_handle h, if (clicon_option_bool(h, "CLICON_XMLDB_MODSTATE")) if ((msd = modstate_diff_new()) == NULL) goto done; + clicon_debug(1, "Reading startup config from %s", db); if (xmldb_get(h, db, "/", &xt, msd) < 0) goto done; + if (xml_child_nr(xt) == 0){ /* If empty skip */ + td->td_target = xt; + goto ok; + } if (msd){ if ((ret = clixon_module_upgrade(h, xt, msd, cbret)) < 0) goto done; @@ -211,6 +216,7 @@ startup_common(clicon_handle h, /* 5. Make generic validation on all new or changed data. Note this is only call that uses 3-values */ + clicon_debug(1, "Validating startup %s", db); if ((ret = generic_validate(yspec, td, cbret)) < 0) goto done; if (ret == 0) @@ -223,6 +229,7 @@ startup_common(clicon_handle h, /* 7. Call plugin transaction complete callbacks */ if (plugin_transaction_complete(h, td) < 0) goto done; + ok: retval = 1; done: if (msd) @@ -282,6 +289,7 @@ startup_validate(clicon_handle h, * @retval -1 Error - or validation failed (but cbret not set) * @retval 0 Validation failed (with cbret set) * @retval 1 Validation OK + * Only called from startup_mode_startup */ int startup_commit(clicon_handle h, @@ -292,6 +300,10 @@ startup_commit(clicon_handle h, int ret; transaction_data_t *td = NULL; + if (strcmp(db,"running")==0){ + clicon_err(OE_FATAL, 0, "Invalid startup db: %s", db); + goto done; + } /* Handcraft a transition with only target and add trees */ if ((td = transaction_new()) == NULL) goto done; @@ -302,6 +314,13 @@ startup_commit(clicon_handle h, /* 8. Call plugin transaction commit callbacks */ if (plugin_transaction_commit(h, td) < 0) goto done; + /* [Delete and] create running db */ + if (xmldb_exists(h, "running") == 1){ + if (xmldb_delete(h, "running") != 0 && errno != ENOENT) + goto done;; + } + if (xmldb_create(h, "running") < 0) + goto done; /* 9, write (potentially modified) tree to running * XXX note here startup is copied to candidate, which may confuse everything */ diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index 1cac8931..658447e8 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -638,16 +638,27 @@ main(int argc, status = STARTUP_OK; break; case SM_RUNNING: /* Use running as startup */ - /* Copy original running to startup and treat as startup */ + /* Copy original running to tmp as backup (restore if error) */ if (xmldb_copy(h, "running", "tmp") < 0) goto done; ret = startup_mode_startup(h, "tmp", cbret); + /* If ret fails, copy tmp back to running */ + if (ret != 1) + if (xmldb_copy(h, "tmp", "running") < 0) + goto done; if (ret2status(ret, &status) < 0) goto done; break; case SM_STARTUP: + /* Copy original running to tmp as backup (restore if error) */ + if (xmldb_copy(h, "running", "tmp") < 0) + goto done; /* Load and commit from startup */ ret = startup_mode_startup(h, "startup", cbret); + /* If ret fails, copy tmp back to running */ + if (ret != 1) + if (xmldb_copy(h, "tmp", "running") < 0) + goto done; if (ret2status(ret, &status) < 0) goto done; /* if status = STARTUP_INVALID, cbret contains info */ @@ -683,8 +694,9 @@ main(int argc, /* Call backend plugin_start with user -- options */ if (clixon_plugin_start(h) < 0) goto done; + /* -1 option to run only once */ if (once) - goto done; + goto ok; /* Daemonize and initiate logging. Note error is initiated here to make demonized errors OK. Before this stage, errors are logged on stderr @@ -722,6 +734,7 @@ main(int argc, goto done; if (event_loop() < 0) goto done; + ok: retval = 0; done: if (cbret) diff --git a/apps/backend/backend_startup.c b/apps/backend/backend_startup.c index 9c9ab382..367ed627 100644 --- a/apps/backend/backend_startup.c +++ b/apps/backend/backend_startup.c @@ -114,6 +114,7 @@ db_merge(clicon_handle h, /*! Clixon startup startup mode: Commit startup configuration into running state * @param[in] h Clixon handle + * @param[in] db tmp or startup * @param[out] cbret If status is invalid contains error message * @retval -1 Error * @retval 0 Validation failed @@ -150,9 +151,10 @@ startup_mode_startup(clicon_handle h, int retval = -1; int ret; - /* [Delete and] create running db */ - if (startup_db_reset(h, "running") < 0) + if (strcmp(db, "running")==0){ + clicon_err(OE_FATAL, 0, "Invalid startup db: %s", db); goto done; + } /* Load plugins and call plugin_init() */ if (backend_plugin_initiate(h) != 0) goto done; @@ -258,6 +260,8 @@ startup_extraxml(clicon_handle h, goto done; if (ret == 0) goto fail; + if (xt==NULL || xml_child_nr(xt)==0) + goto ok; /* Write (potentially modified) xml tree xt back to tmp */ if ((ret = xmldb_put(h, "tmp", OP_REPLACE, xt, @@ -268,6 +272,7 @@ startup_extraxml(clicon_handle h, goto fail; if (ret == 0) goto fail; + ok: retval = 1; done: if (xt) @@ -299,15 +304,22 @@ startup_failsafe(clicon_handle h) clicon_err(OE_XML, errno, "cbuf_new"); goto done; } - if (startup_db_reset(h, "running") < 0) - goto done; if ((ret = xmldb_exists(h, db)) < 0) goto done; if (ret == 0){ /* No it does not exist, fail */ clicon_err(OE_DB, 0, "Startup failed and no Failsafe database found, exiting"); goto done; } - if ((ret = candidate_commit(h, db, cbret)) < 0) /* diff */ + /* Copy original running to tmp as backup (restore if error) */ + if (xmldb_copy(h, "running", "tmp") < 0) + goto done; + if (startup_db_reset(h, "running") < 0) + goto done; + ret = candidate_commit(h, db, cbret); + if (ret != 1) + if (xmldb_copy(h, "tmp", "running") < 0) + goto done; + if (ret < 0) goto done; if (ret == 0){ clicon_err(OE_DB, 0, "Startup failed, Failsafe database validation failed %s", cbuf_get(cbret)); diff --git a/configure b/configure index 64161685..bc0eed46 100755 --- a/configure +++ b/configure @@ -4447,7 +4447,7 @@ _ACEOF -ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile" +ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile example/hello/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure @@ -5155,9 +5155,10 @@ do "etc/clixonrc") CONFIG_FILES="$CONFIG_FILES etc/clixonrc" ;; "example/Makefile") CONFIG_FILES="$CONFIG_FILES example/Makefile" ;; "example/main/Makefile") CONFIG_FILES="$CONFIG_FILES example/main/Makefile" ;; + "example/hello/Makefile") CONFIG_FILES="$CONFIG_FILES example/hello/Makefile" ;; "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;; "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;; - "docker/system/Makefile") CONFIG_FILES="$CONFIG_FILES docker/system/Makefile" ;; + "docker/main/Makefile") CONFIG_FILES="$CONFIG_FILES docker/main/Makefile" ;; "docker/base/Makefile") CONFIG_FILES="$CONFIG_FILES docker/base/Makefile" ;; "util/Makefile") CONFIG_FILES="$CONFIG_FILES util/Makefile" ;; "yang/Makefile") CONFIG_FILES="$CONFIG_FILES yang/Makefile" ;; diff --git a/configure.ac b/configure.ac index d7e2e741..47b79aa2 100644 --- a/configure.ac +++ b/configure.ac @@ -248,10 +248,11 @@ AC_OUTPUT(Makefile etc/Makefile etc/clixonrc example/Makefile - example/main/Makefile + example/main/Makefile + example/hello/Makefile extras/rpm/Makefile docker/Makefile - docker/system/Makefile + docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile diff --git a/doc/FAQ.md b/doc/FAQ.md index 0afffdc0..a5c81437 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -6,6 +6,7 @@ * [Is Clixon extendible?](#is-clixon-extendible) * [Which programming language is used?](#which-programming-language-is-used) * [How to best understand Clixon?](#how-to-best-understand-clixon) + * [Hello world?](#hello-world) * [How do you build and install Clixon (and the example)?](how-do-you-build-and-install-clixon) * [How do I run Clixon example commands?](#how-do-i-run-clixon-example-commands) * [Do I need to setup anything? (IMPORTANT)](#do-i-need-to-setup-anything)) @@ -67,6 +68,10 @@ specification uses [CLIgen](http://github.com/olofhagsand/cligen) ## How to best understand Clixon? Run the Clixon example, in the [example](../example) directory. +## Hello world? + +One of the examples is [a hello world example](../example/hello). Please start with that. + ## How do you build and install Clixon? Clixon: ``` diff --git a/doc/scaling/clixon-commit-0.png b/doc/scaling/clixon-commit-0.png new file mode 100644 index 00000000..cbab820a Binary files /dev/null and b/doc/scaling/clixon-commit-0.png differ diff --git a/doc/scaling/clixon-delete-100.png b/doc/scaling/clixon-delete-100.png new file mode 100644 index 00000000..9396c483 Binary files /dev/null and b/doc/scaling/clixon-delete-100.png differ diff --git a/doc/scaling/clixon-get-0.png b/doc/scaling/clixon-get-0.png new file mode 100644 index 00000000..2879c21e Binary files /dev/null and b/doc/scaling/clixon-get-0.png differ diff --git a/doc/scaling/clixon-get-100.png b/doc/scaling/clixon-get-100.png new file mode 100644 index 00000000..246768d4 Binary files /dev/null and b/doc/scaling/clixon-get-100.png differ diff --git a/doc/scaling/clixon-put-0.png b/doc/scaling/clixon-put-0.png new file mode 100644 index 00000000..2e9acc15 Binary files /dev/null and b/doc/scaling/clixon-put-0.png differ diff --git a/doc/scaling/clixon-put-100.png b/doc/scaling/clixon-put-100.png new file mode 100644 index 00000000..3876d231 Binary files /dev/null and b/doc/scaling/clixon-put-100.png differ diff --git a/doc/scaling/clixon-startup.png b/doc/scaling/clixon-startup.png new file mode 100644 index 00000000..5115bb67 Binary files /dev/null and b/doc/scaling/clixon-startup.png differ diff --git a/doc/scaling/large-lists.md b/doc/scaling/large-lists.md new file mode 100644 index 00000000..7bb3653b --- /dev/null +++ b/doc/scaling/large-lists.md @@ -0,0 +1,225 @@ +# Large lists in Clixon + +Olof Hagsand, 2019-04-17 + + * [1. Background](#1-background) + * [2. Overview](#2-overview) + * [3. Tests](#3-tests) + * [4. Results](#4-results) + * [5. Discussion](#5-discussion) + * [6. Future work](#6-future-work) + * [7. References](#7-references) + +## 1. Background + +Clixon can handle large configurations. Here, measurements using a +large number of elements in a simple "flat" list is analysed. This +includes starting up with alarge existing database; initializing an +empty database with a large number of entries, accessing single +entries with a large database, etc. + +In short, the results show a linear dependency on the number of +entries. This is OK for startup scenarions, but single-enrty (transactional) operations need improvement. + +There are other scaling usecases, such as large configuratin "depth", +large number of requesting clients, etc. + +Thanks to [Netgate](www.netgate.com) for supporting this work. + +## 2. Overview + +The basic case is a large list, according to the following Yang specification: +``` + container x { + description "top-level container"; + list y { + description "List with potential large number of elements"; + key "a"; + leaf a { + description "key in list"; + type int32; + } + leaf b { + description "payload data"; + type string; + } + } + } +``` +where `a` is a unique key and `b` is a payload, useful in replace operations. + +With this XML lists with `N` elements are generated based on +this configuration, eg for `N=10`: +``` + 00 + 11 + 22 + 33 + 44 + 55 + 66 + 77 + 88 + 99 +``` + +Requests are either made over the _whole_ dataset, or for one specific element. The following example shows a Restconf GET operation of a single element: +``` + curl -X GET http://localhost/restconf/data/scaling:x/y=3 + {"scaling:y": [{"a": 3,"b": "3"}]} + +``` + +Operations of single elements (transactions) are made in a burst of +random elements, typically 100. + + +## 3. Tests + +All details of the setup are in the [test script](../../test/plot_perf.sh). + +### Testcases + +All tests measure the "real" time of a command on a lightly loaded +machine using the Linux command `time(1)`. + +The following tests were made (for each architecture and protocol): +* Write `N` entries into the startup configuration. The clixon_backend was started with options `-1s startup`. +* Write `N` entries in one single operation. (With an empty datastore) +* Read `N` entries in one single operation. (With a datastore of `N` entries) +* Commit `N` entries (With a candidate of `N` entries and empty running) +* Read 1 entry (In a datastore of `N` entries) +* Write/Replace 1 entry (In a datastore of `N` entries) +* Delete 1 entry (In a datastore of `N` entries) + +The tests are made using Netconf and Restconf, except commit which is made only for Netconf and startup where protocol is irrelevant. + +### Architecture and OS + +The tests were made on the following hardware, all running Ubuntu Linux: + +#### i686 + +* IBM Thinkpad X60 +* Dual Intel Core Duo processor +* Ubuntu 16.04.6 LTS +* Linux version 4.4.0-143-generic (buildd@lgw01-amd64-037) + * gcc version 5.4.0 20160609 (Ubuntu 5.4.0-6ubuntu1~16.04.10) + * #169-Ubuntu SMP Thu Feb 7 07:56:51 UTC 2019 + +#### ARM + +* Raspberry PI 2 Model B +* ARMv7 Processor rev 5 (v7l) +* Raspbian GNU/Linux 9 +* Linux version 4.14.79-v7+ (dc4@dc4-XPS13-9333) + * gcc version 4.9.3 (crosstool-NG crosstool-ng-1.22.0-88-g8460611)) + * #1159 SMP Sun Nov 4 17:50:20 GMT 2018 + +#### x86_64 + +* Intel NUC Coffee Lake +* Intel Quad-core I5-8259U +* Ubuntu 18.04.1 LTS +* Linux version 4.15.0-47-generic (buildd@lgw01-amd64-001) + * gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3) + * #50-Ubuntu SMP Wed Mar 13 10:44:52 UTC 2019 + +## 4. Results + +This section shows the results of the measurements as defined in [Tests](#tests). +### Startup + +![Startup](clixon-startup.png "Startup") + +### Access of the whole datastore + +![Get config](clixon-get-0.png "Get config") + +![Put config](clixon-put-0.png "Put config") + +![Commit config](clixon-commit-0.png "Commit config") + +### Access of single entries + +![Get single entry](clixon-get-100.png "Get single entry") + +![Put single entry](clixon-put-100.png "Put single entry") + +![Delete single entry](clixon-delete-100.png "Delete single entry") + +### Profiling + +An example profiling of the most demanding case was made: Put single restconf for with 5000 existing entries. +The tool used is valgrind/callgrind with the following approximate result (percentage of total cycles): +* from_rpc_callback 100% + * from_client_commit 65% + * candidate_commit 65% + * from_validate_common 30% + * xml_diff 13% + * xml_yang_validate_all 10% + * xmldb_copy 29% + * xml_copy 22% + * from_client_edit_config 35% + * xmldb_put 35% + * clicon_xml2file 30% + * fprintf 13% + * xml_chardata_encode 12% + +It can be seen that most cycles are spend in file copying and writing +the existing datastore to file. This explains the linear behaviour, ie +the larger existing datastore, the larger number of cycles spent. + +Why is the existing datastore accessed in this way? When a small PUT request is received, several things happen: +* The existing candidate database is modified with the change and written to disk. (35%) +* The difference between candidate and running is computed (13%) +* The new candidate is validated (10%) +* The candidate db is copied to running (29%) + +## 5. Discussion + +All measurements show clear performance differences between the +architectures, which was expected. + +By looking at top and other tools, it seems clear that the main +bootleneck is CPU for the `clixon_backend`. The clients, eg `nginx`, +clixon_restconf` and `clixon_netconf` seem negligable. Memory +footprint is also not limiting. + +Accessing the whole configuration is similar between protocols and +linear in time. This is to be expected since the tranfer is dependent +on the size of the database. + +Accessing a single entry is also similar in all cases and shows large +differences. As expected, the CPU architecture is significant, but +there is also a large difference between Netconf and Restconf. + +The Netconf and restconf setups differ somewhat which may explain +differences in performance. Primarily, the Netconf input is piped to a +single Netconf client while a curl is re-started for each Restconf +call. Also, the Restconf PUT includes a commit. + +Further, the single entry access is linear wrt number of entries. This +means it is possible to run large scale applications on high +performance CPUs, such as 100K entries on a x86_64 in the results, but +it also means that very large lists may not be supported, and that the +system degrades with the size of the lists. + +Examining the profiling of the most demanding Restconf PUT case, most +cycles are spent on handling writing and copying the existing datastore. + +Note that the experiments here contains _very_ simple +data-structures. A more realistic complex example will require more +CPU effort. Ad-hoc measurement of a more complex datastructure, +generated four times the duration of the simple yang model in this work. + +## 6. Future work + +* Improve access of individual elements to sub-linear performance. +* CLI access on large lists (not included in this study) + +## 7. References + +* [RFC6241](https://tools.ietf.org/html/rfc6241) "Network Configuration Protocol (NETCONF)" +* [RFC8040](https://tools.ietf.org/html/rfc8040) "RESTCONF Protocol" +* [plot_perf.sh](../../test/plot_perf.sh) Test script diff --git a/doc/startup.md b/doc/startup.md index 921c0072..d9937cc0 100644 --- a/doc/startup.md +++ b/doc/startup.md @@ -254,7 +254,8 @@ When the startup process is completed, a startup status is set and is accessible If the startup fails, the backend looks for a `failsafe` configuration in `CLICON_XMLDB_DIR/failsafe_db`. If such a config is not found, the -backend terminates. +backend terminates. In this mode, running and startup mode should be +unchanged. If the failsafe is found, the failsafe config is loaded and committed into the running db. @@ -405,6 +406,7 @@ running |--------+------------> GOTO EXTRA XML ### Running mode +On failure, running is restored to initial state ``` running ----+ |----------+--------> GOTO EXTRA XML \ copy parse validate OK / commit @@ -420,7 +422,7 @@ running |--------+------------> GOTO EXTRA XML startup -------+--+-------+------------+ ``` -### Failure +### Failure if failsafe ``` failsafe ----------------------+ reset \ commit diff --git a/docker/Makefile.in b/docker/Makefile.in index 51443a4e..83bb7191 100644 --- a/docker/Makefile.in +++ b/docker/Makefile.in @@ -41,7 +41,7 @@ LIBS = @LIBS@ SHELL = /bin/sh SUBDIRS = base -SUBDIRS += system +SUBDIRS += main #SUBDIRS += cluster .PHONY: all clean distclean depend install-include install uninstall test $(SUBDIRS) diff --git a/docker/README.md b/docker/README.md index ad190774..117705ee 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,6 +2,6 @@ This directory contains sub-directories with examples of Clixon docker images: - * [base](base/README.md) Clixon base image - * [system](system/README.md) Example and test application + * [base](base/README.md) Clixon base image + * [main](main/README.md) Main example and test application diff --git a/docker/base/README.md b/docker/base/README.md index de26e094..0acdca79 100644 --- a/docker/base/README.md +++ b/docker/base/README.md @@ -8,7 +8,7 @@ The clixon docker base image can be used to build clixon applications. It has all the whole code for a clixon release which it downloads from git. -See [clixon-system](../system/README.md) for a more complete clixon image. +See [clixon-system](../main/README.md) for a more complete clixon image. ## Build and push @@ -20,7 +20,7 @@ You may also do `make push` if you want to push the image, but you may then cons ## Example run -The base container is a minimal and primitive example. Look at the [clixon-system](../system) for a more stream-lined application. +The base container is a minimal and primitive example. Look at the [clixon-system](../main) for a more stream-lined application. The following shows a simple example of how to run the example application. First, the container is started with the backend running: diff --git a/docker/system/Dockerfile b/docker/main/Dockerfile similarity index 100% rename from docker/system/Dockerfile rename to docker/main/Dockerfile diff --git a/docker/system/Makefile.in b/docker/main/Makefile.in similarity index 100% rename from docker/system/Makefile.in rename to docker/main/Makefile.in diff --git a/docker/system/README.md b/docker/main/README.md similarity index 86% rename from docker/system/README.md rename to docker/main/README.md index 73378264..61febc54 100644 --- a/docker/system/README.md +++ b/docker/main/README.md @@ -54,3 +54,10 @@ To check status and then kill it: ``` You trigger the test scripts inside the container using `make test`. + +## Changing code + +If you want to edit clixon code so it runs in the container? +You either +(1) "persistent": make your changes in the actual clixon code and commit; make clean to remove the local clone; make test again +(2) "volatile" edit the local clone; make test. \ No newline at end of file diff --git a/docker/system/cleanup.sh b/docker/main/cleanup.sh similarity index 100% rename from docker/system/cleanup.sh rename to docker/main/cleanup.sh diff --git a/docker/system/start.sh b/docker/main/start.sh similarity index 100% rename from docker/system/start.sh rename to docker/main/start.sh diff --git a/docker/system/startsystem.sh b/docker/main/startsystem.sh similarity index 100% rename from docker/system/startsystem.sh rename to docker/main/startsystem.sh diff --git a/example/Makefile.in b/example/Makefile.in index 90a0aa19..3c9a2a89 100644 --- a/example/Makefile.in +++ b/example/Makefile.in @@ -40,7 +40,7 @@ LIBS = @LIBS@ SHELL = /bin/sh -SUBDIRS = main +SUBDIRS = main hello .PHONY: all clean depend install $(SUBDIRS) diff --git a/example/README.md b/example/README.md index 709cf2dd..66149d8b 100644 --- a/example/README.md +++ b/example/README.md @@ -1,4 +1,5 @@ # Clixon examples Clixon have the following examples: - * [Main example](main/README.md) \ No newline at end of file + * [Hello world](hello/README.md) + * [Main example](main/README.md) diff --git a/example/hello/Makefile.in b/example/hello/Makefile.in new file mode 100644 index 00000000..7ba400aa --- /dev/null +++ b/example/hello/Makefile.in @@ -0,0 +1,167 @@ +# +# ***** BEGIN LICENSE BLOCK ***** +# +# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# +# This file is part of CLIXON +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Alternatively, the contents of this file may be used under the terms of +# the GNU General Public License Version 3 or later (the "GPL"), +# in which case the provisions of the GPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of the GPL, and not to allow others to +# use your version of this file under the terms of Apache License version 2, +# indicate your decision by deleting the provisions above and replace them with +# the notice and other provisions required by the GPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the Apache License version 2 or the GPL. +# +# ***** END LICENSE BLOCK ***** +# +VPATH = @srcdir@ +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ +prefix = @prefix@ +bindir = @bindir@ +includedir = @includedir@ +datarootdir = @datarootdir@ +sysconfdir = @sysconfdir@ +datarootdir = @datarootdir@ +localstatedir = @localstatedir@ +libdir = @exec_prefix@/lib + +APPNAME = hello + +# Here is where example yang appears +CLIXON_DATADIR = @CLIXON_DATADIR@ +# Install here if you want default clixon location: +CLIXON_DEFAULT_CONFIG = @CLIXON_DEFAULT_CONFIG@ + +CC = @CC@ +CFLAGS = @CFLAGS@ -rdynamic -fPIC +INSTALLFLAGS = @INSTALLFLAGS@ +with_restconf = @with_restconf@ + +INCLUDES = -I$(includedir) @INCLUDES@ +CPPFLAGS = @CPPFLAGS@ -fPIC + +BE_PLUGIN = $(APPNAME)_backend.so +CLI_PLUGIN = $(APPNAME)_cli.so +NETCONF_PLUGIN = $(APPNAME)_netconf.so +RESTCONF_PLUGIN = $(APPNAME)_restconf.so + +PLUGINS = $(BE_PLUGIN) $(CLI_PLUGIN) $(NETCONF_PLUGIN) +ifeq ($(with_restconf),yes) +PLUGINS += $(RESTCONF_PLUGIN) +endif + +.PHONY: all clean depend install + +all: $(PLUGINS) + +.SUFFIXES: .c .o + +# implicit rule +.c.o: + $(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -c $< + +CLISPECS = $(APPNAME)_cli.cli + +YANGSPECS = clixon-hello@2019-04-17.yang + +# CLI frontend plugin (see also install rule) +#CLI_SRC = $(APPNAME)_cli.c +CLI_OBJ = $(CLI_SRC:%.c=%.o) +$(CLI_PLUGIN): $(CLI_OBJ) + $(CC) -Wall -shared -o $@ -lc $^ + +# Backend plugin (see also install rule) +#BE_SRC = $(APPNAME)_backend.c +BE_OBJ = $(BE_SRC:%.c=%.o) +$(BE_PLUGIN): $(BE_OBJ) + $(CC) -Wall -shared -o $@ -lc $< + +# NETCONF frontend plugin (see also install rule) +#NETCONF_SRC = $(APPNAME)_netconf.c +NETCONF_OBJ = $(NETCONF_SRC:%.c=%.o) +$(NETCONF_PLUGIN): $(NETCONF_OBJ) + $(CC) -Wall -shared -o $@ -lc $^ + +# See configure.ac for disabling restconf +# RESTCONF frontend plugin (see also install rule) +#RESTCONF_SRC = $(APPNAME)_restconf.c +RESTCONF_OBJ = $(RESTCONF_SRC:%.c=%.o) +$(RESTCONF_PLUGIN): $(RESTCONF_OBJ) + $(CC) -Wall -shared -o $@ -lc $^ + +SRC = $(BE_SRC) $(CLI_SRC) $(NETCONF_SRC) +SRC += $(RESTCONF_SRC) + +OBJS = $(BE_OBJ) $(CLI_OBJ) $(NETCONF_OBJ) +OBJS += $(RESTCONF_OBJ) + +clean: + rm -f $(PLUGINS) $(OBJS) + +distclean: clean + rm -f Makefile *~ .depend + + +install: $(YANGSPECS) $(CLISPECS) $(PLUGINS) $(APPNAME).xml + install -m 0644 $(APPNAME).xml $(DESTDIR)$(CLIXON_DEFAULT_CONFIG) + install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME) + install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/clispec + install -m 0644 $(CLISPECS) $(DESTDIR)$(libdir)/$(APPNAME)/clispec + install -d -m 0755 $(DESTDIR)$(datarootdir)/$(APPNAME)/yang + install -m 0644 $(YANGSPECS) $(DESTDIR)$(DESTDIR)$(CLIXON_DATADIR) + install -d -m 0755 $(DESTDIR)$(localstatedir)/$(APPNAME) + +# Uncomment for installing config file in /usr/local/etc instead +# install -d -m 0755 $(DESTDIR)$(sysconfdir) +# install -m 0644 $(APPNAME).xml $(DESTDIR)$(sysconfdir) + +# Uncomment for installing cli plugin (see CLI_SRC) +# install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/cli +# install -m 0644 $(INSTALLFLAGS) $(CLI_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/cli + +# Uncomment for installing backend plugin (see BE_SRC) +# install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/backend +# install -m 0644 $(INSTALLFLAGS) $(BE_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/backend + +# Uncomment for installing netconf plugin (see NETCONF_SRC) +# install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/netconf +# install -m 0644 $(INSTALLFLAGS) $(NETCONF_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/netconf + +# Uncomment for installing restconf plugin (see RESTCONF_SRC). The conditional is +# because it is an option to disable restconf +#ifeq ($(with_restconf),yes) +# install -d -m 0755 $(DESTDIR)$(libdir)/$(APPNAME)/restconf +# install -m 0644 $(INSTALLFLAGS) $(RESTCONF_PLUGIN) $(DESTDIR)$(libdir)/$(APPNAME)/restconf +#endif + +uninstall: + rm -rf $(DESTDIR)$(CLIXON_DEFAULT_CONFIG) + rm -rf $(DESTDIR)$(libdir)/$(APPNAME) + rm -rf $(DESTDIR)$(localstatedir)/$(APPNAME) + rm -rf $(DESTDIR)$(datarootdir)/$(APPNAME) + + +install-include: + +depend: + $(CC) $(DEPENDFLAGS) $(INCLUDES) $(CFLAGS) -MM $(SRC) > .depend + +#include .depend + diff --git a/example/hello/README.md b/example/hello/README.md new file mode 100644 index 00000000..0c7c3a0b --- /dev/null +++ b/example/hello/README.md @@ -0,0 +1,111 @@ +# Clixon hello world example + + * [Content](#content) + * [Compile and run](#compile) + * [Using the CLI](#using-the-cli) + * [Netconf](#netconf) + * [Restconf](#restconf) + * [Next steps](#next-steps) + +## Content + +This directory contains a Clixon example which includes a simple example. It contains the following files: +* `hello.xml` The configuration file. See [yang/clixon-config@.yang](../../yang/clixon-config@2019-03-05.yang) for the documentation of all available fields. +* `clixon-hello@2019-04-17.yang` The yang spec of the example. +* `hello_cli.cli` CLIgen specification. +* `Makefile.in` Example makefile where plugins are built and installed +* `README.md` This file + + +## Compile and run + +Before you start, +* Make [group setup](../../doc/FAQ.md#do-i-need-to-setup-anything-important) + +``` + make && sudo make install +``` +Start backend in the background: +``` + sudo clixon_backend +``` +Start cli: +``` + clixon_cli +``` + +## Using the CLI + +The example CLI allows you to modify and view the data model using `set`, `delete` and `show` via generated code. + +The following example shows how to add a very simple configuration `hello world` using the generated CLI. The config is added to the candidate database, shown, committed to running, and then deleted. +``` + olof@vandal> clixon_cli + cli> set + hello + cli> set hello world + cli> show configuration + hello world; + cli> commit + cli> delete + all Delete whole candidate configuration + hello + cli> delete hello + cli> show configuration + cli> commit + cli> quit + olof@vandal> +``` + +## Netconf + +Clixon also provides a Netconf interface. The following example starts a netconf client form the shell, adds the hello world config, commits it, and shows it: +``` + olof@vandal> clixon_netconf -q + ]]>]]> + ]]>]]> + ]]>]]> + ]]>]]> + ]]>]]> + ]]>]]> +olof@vandal> +``` + +## Restconf + +Clixon also provides a Restconf interface. A reverse proxy needs to be configured. There are [instructions how to setup Nginx](../../doc/FAQ.md#how-do-i-use-restconf) for Clixon. + +Start restconf daemon +``` + sudo su -c "/www-data/clixon_restconf" -s /bin/sh www-data & +``` + +Start sending restconf commands (using Curl): +``` + olof@vandal> curl -X POST http://localhost/restconf/data -d '{"clixon-hello:hello":{"world":null}}' + olof@vandal> curl -X GET http://localhost/restconf/data + { + "data": { + "clixon-hello:hello": { + "world": null + } + } + } +``` + +## Next steps + +The hello world example only has a Yang spec and a template CLI +spec. For more advanced applications, customized backend, cli, netconf +and restconf code callbacks becomes necessary. + +Further, you may want to add upgrade, RPC:s, state data, notification +streams, authentication and authorization. The [main example](../main) +contains examples for such capabilities. + +There are also [container examples](../../docker) and lots more. + + + + + diff --git a/example/hello/clixon-hello@2019-04-17.yang b/example/hello/clixon-hello@2019-04-17.yang new file mode 100644 index 00000000..89c30acd --- /dev/null +++ b/example/hello/clixon-hello@2019-04-17.yang @@ -0,0 +1,14 @@ +module clixon-hello { + yang-version 1.1; + namespace "urn:example:hello"; + prefix he; + revision 2019-04-17 { + description + "Clixon hello world example"; + } + container hello{ + container world{ + presence true; + } + } +} diff --git a/example/hello/hello.xml b/example/hello/hello.xml new file mode 100644 index 00000000..1210e1f2 --- /dev/null +++ b/example/hello/hello.xml @@ -0,0 +1,13 @@ + + /usr/local/etc/example.xml + *:* + /usr/local/share/clixon + clixon-hello + hello + /usr/local/lib/hello/clispec + /usr/local/var/hello.sock + /usr/local/var/hello.pidfile + /usr/local/var/hello + init + false + diff --git a/example/hello/hello_cli.cli b/example/hello/hello_cli.cli new file mode 100644 index 00000000..a3b469f5 --- /dev/null +++ b/example/hello/hello_cli.cli @@ -0,0 +1,54 @@ +# Common CLI syntax for both server and PMNode operatio mode +CLICON_MODE="hello"; +CLICON_PROMPT="cli> "; + +# Reference generated data model +set @datamodel, cli_set(); +merge @datamodel, cli_merge(); +create @datamodel, cli_create(); +delete("Delete a configuration item") @datamodel, cli_del(); + +validate("Validate changes"), cli_validate(); +commit("Commit the changes"), cli_commit(); +quit("Quit"), cli_quit(); +delete("Delete a configuration item") all("Delete whole candidate configuration"), delete_all("candidate"); + +startup("Store running as startup config"), db_copy("running", "startup"); +no("Negate or remove") debug("Debugging parts of the system"), cli_debug_cli((int32)0); +debug("Debugging parts of the system"), cli_debug_cli((int32)1);{ + level("Set debug level: 1..n") ("Set debug level (0..n)"), cli_debug_backend(); +} +discard("Discard edits (rollback 0)"), discard_changes(); +compare("Compare running and candidate"), compare_dbs((int32)1); + +show("Show a particular state of the system"){ + xpath("Show configuration") ("XPATH expression"), show_conf_xpath("candidate"); + version("Show version"), cli_show_version("candidate", "text", "/"); + compare("Compare candidate and running databases"), compare_dbs((int32)0);{ + xml("Show comparison in xml"), compare_dbs((int32)0); + text("Show comparison in text"), compare_dbs((int32)1); + } + configuration("Show configuration"), cli_show_config("candidate", "text", "/");{ + xml("Show configuration as XML"), cli_show_config("candidate", "xml", "/");{ + @datamodel, cli_show_auto("candidate", "xml"); + } + cli("Show configuration as CLI commands"), cli_show_config("candidate", "cli", "/");{ + @datamodel, cli_show_auto("candidate", "cli"); + } + netconf("Show configuration as netconf edit-config operation"), cli_show_config("candidate", "netconf", "/");{ + @datamodel, cli_show_auto("candidate", "netconf"); + } + text("Show configuration as text"), cli_show_config("candidate","text","/");{ + @datamodel, cli_show_auto("candidate", "text"); + } + json("Show configuration as JSON"), cli_show_config("candidate", "json", "/");{ + @datamodel, cli_show_auto("candidate", "json"); + } + } +} + +save("Save candidate configuration to XML file") ("Filename (local filename)"), save_config_file("candidate","filename"); +load("Load configuration from XML file") ("Filename (local filename)"),load_config_file("filename", "replace");{ + replace("Replace candidate with file contents"), load_config_file("filename", "replace"); + merge("Merge file with existent candidate"), load_config_file("filename", "merge"); +} diff --git a/example/main/README.md b/example/main/README.md index 76f9eb02..16215237 100644 --- a/example/main/README.md +++ b/example/main/README.md @@ -15,7 +15,7 @@ ## Content This directory contains a Clixon example which includes a simple example. It contains the following files: -* `example.xml` The configuration file. See [yang/clixon-config@.yang](../../yang/clixon-config@2018-10-21.yang) for the documentation of all available fields. +* `example.xml` The configuration file. See [yang/clixon-config@.yang](../../yang/clixon-config@2019-03-05.yang) for the documentation of all available fields. * `clixon-example@2019-01-13.yang` The yang spec of the example. * `example_cli.cli` CLIgen specification. * `example_cli.c` CLI callback plugin containing functions called in the cli file above: a generic callback (`mycallback`) and an example RPC call (`example_client_rpc`). diff --git a/include/clixon_custom.h b/include/clixon_custom.h index d06a9e50..c302d0ed 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -41,4 +41,6 @@ */ #undef RPC_USERNAME_ASSERT - +/* Use new xml_insert code on sorted xml lists + */ +#define USE_XML_INSERT diff --git a/lib/clixon/clixon_string.h b/lib/clixon/clixon_string.h index f78ddbc9..eb106c10 100644 --- a/lib/clixon/clixon_string.h +++ b/lib/clixon/clixon_string.h @@ -86,6 +86,7 @@ int xml_chardata_encode(char **escp, char *fmt, ...); int uri_percent_decode(char *enc, char **str); const char *clicon_int2str(const map_str2int *mstab, int i); int clicon_str2int(const map_str2int *mstab, char *str); +int clicon_str2int_search(const map_str2int *mstab, char *str, int upper); int nodeid_split(char *nodeid, char **prefix, char **id); char *clixon_trim(char *str); int regexp_xsd2posix(char *xsd, char **posix); diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index ef6395a7..879aa453 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -118,8 +118,9 @@ cxobj *xml_child_i_type(cxobj *xn, int i, enum cxobj_type type); cxobj *xml_child_i_set(cxobj *xt, int i, cxobj *xc); cxobj *xml_child_each(cxobj *xparent, cxobj *xprev, enum cxobj_type type); -cxobj **xml_childvec_get(cxobj *x); +int xml_child_insert_pos(cxobj *x, cxobj *xc, int i); int xml_childvec_set(cxobj *x, int len); +cxobj **xml_childvec_get(cxobj *x); cxobj *xml_new(char *name, cxobj *xn_parent, yang_stmt *spec); yang_stmt *xml_spec(cxobj *x); int xml_spec_set(cxobj *x, yang_stmt *spec); @@ -130,7 +131,6 @@ cxobj *xml_find(cxobj *xn_parent, char *name); int xml_addsub(cxobj *xp, cxobj *xc); cxobj *xml_wrap_all(cxobj *xp, char *tag); cxobj *xml_wrap(cxobj *xc, char *tag); -#define xml_insert(x,t) xml_wrap_all((x),(t)) int xml_purge(cxobj *xc); int xml_child_rm(cxobj *xp, int i); int xml_rm(cxobj *xc); diff --git a/lib/clixon/clixon_xml_sort.h b/lib/clixon/clixon_xml_sort.h index 558d3c11..a75c1358 100644 --- a/lib/clixon/clixon_xml_sort.h +++ b/lib/clixon/clixon_xml_sort.h @@ -40,8 +40,10 @@ * Prototypes */ int xml_child_spec(cxobj *x, cxobj *xp, yang_stmt *yspec, yang_stmt **yp); +int xml_cmp(cxobj *x1, cxobj *x2, int enm); int xml_sort(cxobj *x0, void *arg); -int xml_sort_verify(cxobj *x, void *arg); -int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp); +int xml_insert(cxobj *xp, cxobj *xc); +int xml_sort_verify(cxobj *x, void *arg); +int match_base_child(cxobj *x0, cxobj *x1c, yang_stmt *yc, cxobj **x0cp); #endif /* _CLIXON_XML_SORT_H */ diff --git a/lib/clixon/clixon_yang.h b/lib/clixon/clixon_yang.h index 44919010..944e9a8f 100644 --- a/lib/clixon/clixon_yang.h +++ b/lib/clixon/clixon_yang.h @@ -161,6 +161,9 @@ enum rfc_6020 yang_keyword_get(yang_stmt *ys); char *yang_argument_get(yang_stmt *ys); cg_var *yang_cv_get(yang_stmt *ys); cvec *yang_cvec_get(yang_stmt *ys); +int yang_cvec_set(yang_stmt *ys, cvec *cvv); +void *yang_regex_cache_get(yang_stmt *ys); +int yang_regex_cache_set(yang_stmt *ys, void *regex); /* Other functions */ yang_stmt *yspec_new(void); diff --git a/lib/src/clixon_datastore.c b/lib/src/clixon_datastore.c index 149cf49d..278a0023 100644 --- a/lib/src/clixon_datastore.c +++ b/lib/src/clixon_datastore.c @@ -188,14 +188,15 @@ xmldb_copy(clicon_handle h, int retval = -1; char *fromfile = NULL; char *tofile = NULL; - db_elmnt *de1 = NULL; - db_elmnt *de2 = NULL; + db_elmnt *de1 = NULL; /* from */ + db_elmnt *de2 = NULL; /* to */ db_elmnt de0 = {0,}; - cxobj *x1 = NULL; - cxobj *x2 = NULL; + cxobj *x1 = NULL; /* from */ + cxobj *x2 = NULL; /* to */ /* XXX lock */ if (clicon_option_bool(h, "CLICON_XMLDB_CACHE")){ + /* Copy in-memory cache */ /* 1. "to" xml tree in x1 */ if ((de1 = clicon_db_elmnt_get(h, from)) != NULL) x1 = de1->de_xml; @@ -208,7 +209,7 @@ xmldb_copy(clicon_handle h, xml_free(x2); x2 = NULL; } - else if (x2 == NULL){ /* create x2 and copy x1 to it */ + else if (x2 == NULL){ /* create x2 and copy from x1 */ if ((x2 = xml_new(xml_name(x1), NULL, xml_spec(x1))) == NULL) goto done; if (xml_copy(x1, x2) < 0) @@ -221,12 +222,13 @@ xmldb_copy(clicon_handle h, if (xml_copy(x1, x2) < 0) goto done; } - if (x1 || x2){ - if (de2) - de0 = *de2; - de0.de_xml = x2; /* The new tree */ - clicon_db_elmnt_set(h, to, &de0); - } + /* always set cache although not strictly necessary in case 1 + * above, but logic gets complicated due to differences with + * de and de->de_xml */ + if (de2) + de0 = *de2; + de0.de_xml = x2; /* The new tree */ + clicon_db_elmnt_set(h, to, &de0); } /* Copy the files themselves (above only in-memory cache) */ if (xmldb_db2file(h, from, &fromfile) < 0) diff --git a/lib/src/clixon_datastore_write.c b/lib/src/clixon_datastore_write.c index 13fa6440..63d2958f 100644 --- a/lib/src/clixon_datastore_write.c +++ b/lib/src/clixon_datastore_write.c @@ -128,9 +128,7 @@ text_modify(clicon_handle h, int i; int ret; int changed = 0; /* Only if x0p's children have changed-> sort is necessary */ - - assert(x1 && xml_type(x1) == CX_ELMNT); - assert(y0); + /* Check for operations embedded in tree according to netconf */ if ((opstr = xml_find_value(x1, "operation")) != NULL) if (xml_operation(opstr, &op) < 0) @@ -157,8 +155,16 @@ text_modify(clicon_handle h, permit = 1; } // int iamkey=0; - if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) + +#ifdef USE_XML_INSERT + /* Add new xml node but without parent - insert when node fully + copied (see changed conditional below) */ + if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL) goto done; +#else + if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) + goto done; +#endif changed++; /* Copy xmlns attributes */ @@ -204,6 +210,12 @@ text_modify(clicon_handle h, } } } +#ifdef USE_XML_INSERT + if (changed){ + if (xml_insert(x0p, x0) < 0) + goto done; + } +#endif break; case OP_DELETE: if (x0==NULL){ @@ -283,8 +295,15 @@ text_modify(clicon_handle h, goto fail; permit = 1; } +#ifdef USE_XML_INSERT + /* Add new xml node but without parent - insert when node fully + copied (see changed conditional below) */ + if ((x0 = xml_new(x1name, NULL, (yang_stmt*)y0)) == NULL) + goto done; +#else if ((x0 = xml_new(x1name, x0p, (yang_stmt*)y0)) == NULL) goto done; +#endif changed++; /* Copy xmlns attributes */ x1a = NULL; @@ -346,6 +365,12 @@ text_modify(clicon_handle h, if (ret == 0) goto fail; } +#ifdef USE_XML_INSERT + if (changed){ + if (xml_insert(x0p, x0) < 0) + goto done; + } +#endif break; case OP_DELETE: if (x0==NULL){ @@ -363,15 +388,16 @@ text_modify(clicon_handle h, } if (xml_purge(x0) < 0) goto done; - changed++; } break; default: break; } /* CONTAINER switch op */ } /* else Y_CONTAINER */ +#ifndef USE_XML_INSERT if (changed) xml_sort(x0p, NULL); +#endif retval = 1; done: if (x0vec) @@ -418,8 +444,8 @@ text_modify_top(clicon_handle h, int ret; /* Assure top-levels are 'config' */ - assert(x0 && strcmp(xml_name(x0),"config")==0); - assert(x1 && strcmp(xml_name(x1),"config")==0); + // assert(x0 && strcmp(xml_name(x0),"config")==0); + // assert(x1 && strcmp(xml_name(x1),"config")==0); /* Check for operations embedded in tree according to netconf */ if ((opstr = xml_find_value(x1, "operation")) != NULL) @@ -584,11 +610,11 @@ xml_container_presence(cxobj *x, */ int xmldb_put(clicon_handle h, - const char *db, - enum operation_type op, - cxobj *x1, - char *username, - cbuf *cbret) + const char *db, + enum operation_type op, + cxobj *x1, + char *username, + cbuf *cbret) { int retval = -1; char *dbfile = NULL; diff --git a/lib/src/clixon_hash.c b/lib/src/clixon_hash.c index fa47bfcc..b14e10ad 100644 --- a/lib/src/clixon_hash.c +++ b/lib/src/clixon_hash.c @@ -112,6 +112,7 @@ hash_bucket(const char *str) * * @retval hash Pointer to new hash table. * @retval NULL Error + * @see hash_free For freeing the hash-table */ clicon_hash_t * hash_init(void) diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index 530b342c..ceaf290a 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -915,79 +915,4 @@ json_parse_file(int fd, return retval; } -/* - * Turn this on to get a json parse and pretty print test program - * Usage: json - * read json from input - * Example compile: - gcc -g -o json -I. -I../clixon ./clixon_json.c -lclixon -lcligen - * Example run: - echo '{"foo": -23}' | ./json -*/ -#if 0 /* Test program */ - -static int -usage(char *argv0) -{ - fprintf(stderr, "usage:%s.\n\tInput on stdin\n", argv0); - exit(0); -} - -int -main(int argc, - char **argv) -{ - cxobj *xt; - cxobj *xc; - cbuf *cb = cbuf_new(); - char *buf = NULL; - int i; - int c; - int len; - FILE *f = stdin; - - if (argc != 1){ - usage(argv[0]); - return 0; - } - clicon_log_init(__FILE__, LOG_INFO, CLICON_LOG_STDERR); - len = 1024; /* any number is fine */ - if ((buf = malloc(len)) == NULL){ - perror("malloc"); - return -1; - } - memset(buf, 0, len); - - i = 0; /* position in buf */ - while (1){ /* read the whole file */ - if ((c = fgetc(f)) == EOF) - break; - if (len==i){ - if ((buf = realloc(buf, 2*len)) == NULL){ - fprintf(stderr, "%s: realloc: %s\n", __FUNCTION__, strerror(errno)); - goto done; - } - memset(buf+len, 0, len); - len *= 2; - } - buf[i++] = (char)(c&0xff); - } /* read a line */ - - if (json_parse_str(buf, &xt) < 0) - return -1; - xc = NULL; - while ((xc = xml_child_each(xt, xc, -1)) != NULL) { - xmltree2cbuf(cb, xc, 0); /* dump data structures */ - //clicon_xml2cbuf(cb, xc, 0, 1); /* print xml */ - } - fprintf(stdout, "%s", cbuf_get(cb)); - if (xt) - xml_free(xt); - if (cb) - cbuf_free(cb); - done: - return 0; -} - -#endif /* Test program */ diff --git a/lib/src/clixon_string.c b/lib/src/clixon_string.c index ffbc68ff..bf25f146 100644 --- a/lib/src/clixon_string.c +++ b/lib/src/clixon_string.c @@ -552,7 +552,7 @@ clicon_int2str(const map_str2int *mstab, * @param[in] str Input string * @retval int Value * @retval -1 Error, not found - * @note linear search + * @see clicon_str2int_search for optimized lookup, but strings must be sorted */ int clicon_str2int(const map_str2int *mstab, @@ -566,6 +566,65 @@ clicon_str2int(const map_str2int *mstab, return -1; } +/*! Map from string to int using binary (alphatical) search + * @param[in] ms String, integer map + * @param[in] str Input string + * @param[in] low Lower bound index + * @param[in] upper Upper bound index + * @param[in] len Length of array (max) + * @param[out] found Integer found (can also be negative) + * @retval 0 Not found + * @retval 1 Found with "found" value set. + * @note Assumes sorted strings, tree search + */ +static int +str2int_search1(const map_str2int *mstab, + char *str, + int low, + int upper, + int len, + int *found) +{ + const struct map_str2int *ms; + int mid; + int cmp; + + if (upper < low) + return 0; /* not found */ + mid = (low + upper) / 2; + if (mid >= len) /* beyond range */ + return 0; /* not found */ + ms = &mstab[mid]; + if ((cmp = strcmp(str, ms->ms_str)) == 0){ + *found = ms->ms_int; + return 1; /* found */ + } + else if (cmp < 0) + return str2int_search1(mstab, str, low, mid-1, len, found); + else + return str2int_search1(mstab, str, mid+1, upper, len, found); +} + +/*! Map from string to int using str2int map + * @param[in] ms String, integer map + * @param[in] str Input string + * @retval int Value + * @retval -1 Error, not found + * @note Assumes sorted strings, tree search + * @note -1 can not be value + */ +int +clicon_str2int_search(const map_str2int *mstab, + char *str, + int len) +{ + int found; + + if (str2int_search1(mstab, str, 0, len, len, &found)) + return found; + return -1; /* not found */ +} + /*! Split colon-separated node identifier into prefix and name * @param[in] node-id * @param[out] prefix Malloced string. May be NULL. diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index 1fdfab50..1735df14 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -127,6 +127,7 @@ struct xml{ reference, dont free */ cg_var *x_cv; /* Cached value as cligen variable (eg xml_cmp) */ + char *x_ns_cache; /* Cached namespace */ int _x_vector_i; /* internal use: xml_child_each */ int _x_i; /* internal use for sorting: see xml_enumerate and xml_cmp */ @@ -234,6 +235,7 @@ xml_prefix_set(cxobj *xn, * @retval 0 OK * @retval -1 Error * @see xmlns_check XXX can these be merged? + * @note, this function uses a cache. Any case where cache should be cleared? */ int xml2ns(cxobj *x, @@ -241,9 +243,11 @@ xml2ns(cxobj *x, char **namespace) { int retval = -1; - char *ns; + char *ns = NULL; cxobj *xp; + if ((ns = x->x_ns_cache) != NULL) + goto ok; if (prefix != NULL) /* xmlns:="" */ ns = xml_find_type_value(x, "xmlns", prefix, CX_ATTR); else /* xmlns="" */ @@ -261,6 +265,11 @@ xml2ns(cxobj *x, ns = DEFAULT_XML_RPC_NAMESPACE; #endif } + if (ns && (x->x_ns_cache = strdup(ns)) == NULL){ + clicon_err(OE_XML, errno, "strdup"); + goto done; + } + ok: if (namespace) *namespace = ns; retval = 0; @@ -562,6 +571,7 @@ xml_child_nr_type(cxobj *xn, * @param[in] i the number of the child, eg order in children vector * @retval xml The child xml node * @retval NULL if no such child, or empty child + * @see xml_child_i_type */ cxobj * xml_child_i(cxobj *xn, @@ -653,7 +663,7 @@ xml_child_each(cxobj *xparent, } /*! Extend child vector with one and insert xml node there - * Note: does not do anything with child, you may need to set its parent, etc + * @note does not do anything with child, you may need to set its parent, etc */ static int xml_child_append(cxobj *x, @@ -669,7 +679,31 @@ xml_child_append(cxobj *x, return 0; } -/*! Set a a childvec to a specific size, fill with children after +/*! Insert child xc at position i under parent xp + * + * @see xml_child_append + * @note does not do anything with child, you may need to set its parent, etc + */ +int +xml_child_insert_pos(cxobj *xp, + cxobj *xc, + int i) +{ + size_t size; + + xp->x_childvec_len++; + xp->x_childvec = realloc(xp->x_childvec, xp->x_childvec_len*sizeof(cxobj*)); + if (xp->x_childvec == NULL){ + clicon_err(OE_XML, errno, "realloc"); + return -1; + } + size = (xml_child_nr(xp) - i - 1)*sizeof(cxobj *); + memmove(&xp->x_childvec[i+1], &xp->x_childvec[i], size); + xp->x_childvec[i] = xc; + return 0; +} + +/*! Set a childvec to a specific size, fill with children after * @code * xml_childvec_set(x, 2); * xml_child_i_set(x, 0, xc0) @@ -710,8 +744,10 @@ xml_childvec_get(cxobj *x) * ... * xml_free(x); * @endcode - * @note yspec may be NULL either because it is not known or it is irrelevant, - * eg for body or attribute + * @note As a rule, yspec should be given in normal Clixon calls to enable + * proper sorting and insert functionality. Except as follows: + * - type is body or attribute + * - Yang is unknown * @see xml_sort_insert */ cxobj * @@ -732,6 +768,7 @@ xml_new(char *name, xml_parent_set(x, xp); if (xml_child_append(xp, x) < 0) return NULL; + x->_x_i = xml_child_nr(xp)-1; } x->x_spec = yspec; /* Can be NULL */ return x; @@ -788,11 +825,12 @@ xml_cv_set(cxobj *x, * name "name". * * @param[in] x_up Base XML object - * @param[in] name shell wildcard pattern to match with node name + * @param[in] name Node name * * @retval xmlobj if found. * @retval NULL if no such node found. * @see xml_find_type A more generic function + * @note Linear scalability and relies on strcmp */ cxobj * xml_find(cxobj *x_up, @@ -967,15 +1005,14 @@ xml_child_rm(cxobj *xp, int xml_rm(cxobj *xc) { - int retval = 0; + int retval = -1; cxobj *xp; cxobj *x; int i; if ((xp = xml_parent(xc)) == NULL) - goto done; - retval = -1; - /* Find child in parent */ + goto ok; + /* Find child in parent XXX: search? */ x = NULL; i = 0; while ((x = xml_child_each(xp, x, -1)) != NULL) { if (x == xc) @@ -983,7 +1020,10 @@ xml_rm(cxobj *xc) i++; } if (x != NULL) - retval = xml_child_rm(xp, i); + if (xml_child_rm(xp, i) < 0) + goto done; + ok: + retval = 0; done: return retval; } @@ -1331,6 +1371,8 @@ xml_free(cxobj *x) free(x->x_childvec); if (x->x_cv) cv_free(x->x_cv); + if (x->x_ns_cache) + free(x->x_ns_cache); free(x); return 0; } diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index 98c649f2..ac43a616 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -1073,6 +1073,13 @@ cvec2xml_1(cvec *cvv, * @param[out] changed_x0 Pointervector to XML nodes changed orig value * @param[out] changed_x1 Pointervector to XML nodes changed wanted value * @param[out] changedlen Length of changed vector + * Algorithm to compare two sorted lists A, B: + * A 0 1 2 3 5 6 + * B 0 2 4 5 6 + * Let a,b be first elements of A,B respectively + * a = b : recurse; get next a,b + * a < b : add a in x0, get next a + * a > b : add b in x1, get next b */ static int xml_diff1(yang_stmt *ys, @@ -1092,35 +1099,54 @@ xml_diff1(yang_stmt *ys, yang_stmt *yc; char *b1; char *b2; + int eq; - clicon_debug(2, "%s: %s", __FUNCTION__, ys->ys_argument?ys->ys_argument:"yspec"); - /* Check nodes present in x0 and x1 + nodes only in x0 - * Loop over x0 - * XXX: room for improvement. Compare with match_base_child() - */ - x0c = NULL; - while ((x0c = xml_child_each(x0, x0c, CX_ELMNT)) != NULL){ - if ((yc = xml_spec(x0c)) == NULL){ - clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x0c)); - goto done; + /* Traverse x0 and x1 in lock-step */ + x0c = x1c = NULL; + x0c = xml_child_each(x0, x0c, CX_ELMNT); + x1c = xml_child_each(x1, x1c, CX_ELMNT); + for (;;){ + if (x0c == NULL && x1c == NULL) + goto ok; + else if (x0c == NULL){ + if (cxvec_append(x1c, x1vec, x1veclen) < 0) + goto done; + x1c = xml_child_each(x1, x1c, CX_ELMNT); + continue; } - /* Does x1 have a child matching x0c? */ - if (match_base_child(x1, x0c, yc, &x1c) < 0) - goto done; - if (x1c == NULL){ + else if (x1c == NULL){ if (cxvec_append(x0c, x0vec, x0veclen) < 0) goto done; + x0c = xml_child_each(x0, x0c, CX_ELMNT); + continue; } - else if (yang_choice(yc)){ - /* if x0c and x1c are choice/case, then they are changed */ - if (cxvec_append(x0c, changed_x0, changedlen) < 0) - goto done; - (*changedlen)--; /* append two vectors */ - if (cxvec_append(x1c, changed_x1, changedlen) < 0) + /* Both x0c and x1c exists, check if they are equal. */ + eq = xml_cmp(x0c, x1c, 0); + if (eq < 0){ + if (cxvec_append(x0c, x0vec, x0veclen) < 0) goto done; + x0c = xml_child_each(x0, x0c, CX_ELMNT); } - else{ /* if x0c and x1c are leafs w bodies, then they are changed */ - if (yc->ys_keyword == Y_LEAF){ + else if (eq > 0){ + if (cxvec_append(x1c, x1vec, x1veclen) < 0) + goto done; + x1c = xml_child_each(x1, x1c, CX_ELMNT); + } + else{ /* equal */ + if ((yc = xml_spec(x0c)) == NULL){ + clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x0c)); + goto done; + } + if (yang_choice(yc)){ + /* if x0c and x1c are choice/case, then they are changed */ + if (cxvec_append(x0c, changed_x0, changedlen) < 0) + goto done; + (*changedlen)--; /* append two vectors */ + if (cxvec_append(x1c, changed_x1, changedlen) < 0) + goto done; + } + else if (yc->ys_keyword == Y_LEAF){ + /* if x0c and x1c are leafs w bodies, then they are changed */ if ((b1 = xml_body(x0c)) == NULL) /* empty type */ break; if ((b2 = xml_body(x1c)) == NULL) /* empty type */ @@ -1133,29 +1159,16 @@ xml_diff1(yang_stmt *ys, goto done; } } - if (xml_diff1(yc, x0c, x1c, - x0vec, x0veclen, - x1vec, x1veclen, - changed_x0, changed_x1, changedlen)< 0) + else if (xml_diff1(yc, x0c, x1c, + x0vec, x0veclen, + x1vec, x1veclen, + changed_x0, changed_x1, changedlen)< 0) goto done; } - } /* while x0 */ - /* Check nodes present only in x1 - * Loop over x1 - */ - x1c = NULL; - while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL){ - if ((yc = xml_spec(x1c)) == NULL){ - clicon_err(OE_UNIX, errno, "Unknown element: %s", xml_name(x1c)); - goto done; - } - /* Does x0 have a child matching x1c? */ - if (match_base_child(x0, x1c, yc, &x0c) < 0) - goto done; - if (x0c == NULL) - if (cxvec_append(x1c, x1vec, x1veclen) < 0) - goto done; - } /* while x0 */ + x0c = xml_child_each(x0, x0c, CX_ELMNT); + x1c = xml_child_each(x1, x1c, CX_ELMNT); + } + ok: retval = 0; done: return retval; @@ -1635,7 +1648,8 @@ xml_default(cxobj *xt, cxobj *xc; cxobj *xb; char *str; - + int added=0; + if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){ retval = 0; goto done; @@ -1650,8 +1664,14 @@ xml_default(cxobj *xt, assert(y->ys_cv); if (!cv_flag(y->ys_cv, V_UNSET)){ /* Default value exists */ if (!xml_find(xt, y->ys_argument)){ + +#ifdef USE_XML_INSERT + if ((xc = xml_new(y->ys_argument, NULL, y)) == NULL) + goto done; +#else if ((xc = xml_new(y->ys_argument, xt, y)) == NULL) goto done; +#endif xml_flag_set(xc, XML_FLAG_DEFAULT); if ((xb = xml_new("body", xc, NULL)) == NULL) goto done; @@ -1663,11 +1683,19 @@ xml_default(cxobj *xt, if (xml_value_set(xb, str) < 0) goto done; free(str); + added++; +#ifdef USE_XML_INSERT + if (xml_insert(xt, xc) < 0) + goto done; +#endif } } } } - xml_sort(xt, NULL); +#ifndef USE_XML_INSERT + if (added) + xml_sort(xt, NULL); +#endif retval = 0; done: return retval; @@ -1959,8 +1987,8 @@ api_path2xpath(yang_stmt *yspec, * @param[out] xpathp Resulting xml tree * @param[out] ypathp Yang spec matching xpathp * @retval 1 OK - * @retval 0 Invalid api_path or associated XML, clicon_err called - * @retval -1 Fatal error, clicon_err called + * @retval 0 Invalid api_path or associated XML, clicon_err called + * @retval -1 Fatal error, clicon_err called * * @note both retval 0 and -1 set clicon_err, but the later is fatal * @see api_path2xpath For api-path to xml xpath translation @@ -1994,6 +2022,7 @@ api_path2xml_vec(char **vec, cxobj *x = NULL; yang_stmt *y = NULL; yang_stmt *ymod; + yang_stmt *ykey; char *namespace = NULL; if ((nodeid = vec[0]) == NULL || strlen(nodeid)==0){ @@ -2077,7 +2106,12 @@ api_path2xml_vec(char **vec, /* Create keys */ while ((cvi = cvec_each(cvk, cvi)) != NULL) { keyname = cv_string_get(cvi); - if ((xn = xml_new(keyname, x, NULL)) == NULL) + if ((ykey = yang_find(y, Y_LEAF, keyname)) == NULL){ + clicon_err(OE_XML, 0, "List statement \"%s\" has no key leaf \"%s\"", + yang_argument_get(y), keyname); + goto done; + } + if ((xn = xml_new(keyname, x, ykey)) == NULL) goto done; xml_type_set(xn, CX_ELMNT); if ((xb = xml_new("body", xn, NULL)) == NULL) @@ -2174,8 +2208,8 @@ api_path2xml(char *api_path, goto fail; } nvec--; /* NULL-terminated */ - if ((retval = api_path2xml_vec(vec+1, nvec, - xtop, yspec, nodeclass, strict, + if ((retval = api_path2xml_vec(vec+1, nvec, + xtop, yspec, nodeclass, strict, xbotp, ybotp)) < 1) goto done; xml_yang_root(*xbotp, &xroot); diff --git a/lib/src/clixon_xml_parse.l b/lib/src/clixon_xml_parse.l index 35a9d5af..8d1826c2 100644 --- a/lib/src/clixon_xml_parse.l +++ b/lib/src/clixon_xml_parse.l @@ -132,10 +132,10 @@ ncname {namestart}{namechar}* "\< { BEGIN(START); return *clixon_xml_parsetext; } & { _YA->ya_lex_state =STATEA;BEGIN(AMPERSAND);} -[ \t] { clixon_xml_parselval.string = yytext;return WHITESPACE; } -\r\n { clixon_xml_parselval.string = "\n";return WHITESPACE; } +[ \t]+ { clixon_xml_parselval.string = yytext;return WHITESPACE; } +\r\n { clixon_xml_parselval.string = "\n"; _YA->ya_linenum++; return WHITESPACE; } \r { clixon_xml_parselval.string = "\n";return WHITESPACE; } -\n { clixon_xml_parselval.string = yytext; _YA->ya_linenum++;return WHITESPACE; } +\n { clixon_xml_parselval.string = "\n"; _YA->ya_linenum++;return WHITESPACE; } . { clixon_xml_parselval.string = yytext; return CHARDATA; } /* @see xml_chardata_encode */ diff --git a/lib/src/clixon_xml_parse.y b/lib/src/clixon_xml_parse.y index b5fcbc05..96876021 100644 --- a/lib/src/clixon_xml_parse.y +++ b/lib/src/clixon_xml_parse.y @@ -112,6 +112,44 @@ xml_parse_content(struct xml_parse_yacc_arg *ya, return retval; } +/*! Add whitespace + * If text, ie only body, keep as is. + * But if there is an element, then skip all whitespace. + */ +static int +xml_parse_whitespace(struct xml_parse_yacc_arg *ya, + char *str) +{ + cxobj *xn = ya->ya_xelement; + cxobj *xp = ya->ya_xparent; + int retval = -1; + int i; + + ya->ya_xelement = NULL; /* init */ + /* If there is an element already, only add one whitespace child + * otherwise, keep all whitespace. + */ +#if 1 + for (i=0; iya_xelement = xn; + ok: + retval = 0; + done: + return retval; +} + + static int xml_parse_version(struct xml_parse_yacc_arg *ya, char *ver) @@ -243,11 +281,17 @@ xml_parse_bslash1(struct xml_parse_yacc_arg *ya, while ((xc = xml_child_each(x, xc, CX_ELMNT)) != NULL) break; if (xc != NULL){ /* at least one element */ - xc = NULL; - while ((xc = xml_child_each(x, xc, CX_BODY)) != NULL) { - xml_purge(xc); - xc = NULL; /* reset iterator */ - } + int i; + for (i=0; i element"); } | pi { clicon_debug(2, "content -> pi"); } | CHARDATA { if (xml_parse_content(_YA, $1) < 0) YYABORT; clicon_debug(2, "content -> CHARDATA %s", $1); } - | WHITESPACE { if (xml_parse_content(_YA, $1) < 0) YYABORT; + | WHITESPACE { if (xml_parse_whitespace(_YA, $1) < 0) YYABORT; clicon_debug(2, "content -> WHITESPACE %s", $1); } | { clicon_debug(2, "content -> "); } ; diff --git a/lib/src/clixon_xml_sort.c b/lib/src/clixon_xml_sort.c index 5c0dd389..fb063250 100644 --- a/lib/src/clixon_xml_sort.c +++ b/lib/src/clixon_xml_sort.c @@ -89,7 +89,8 @@ xml_cv_cache(cxobj *x, uint8_t fraction = 0; char *body; - body = xml_body(x); + if ((body = xml_body(x)) == NULL) + body=""; if ((cv = xml_cv(x)) != NULL) goto ok; if ((y = xml_spec(x)) == NULL) @@ -187,19 +188,30 @@ xml_child_spec(cxobj *x, } /*! Help function to qsort for sorting entries in xml child vector same parent - * @param[in] xml object 1 - * @param[in] xml object 2 - * @retval 0 If equal - * @retval <0 if x1 is less than x2 - * @retval >0 if x1 is greater than x2 + * @param[in] x1 object 1 + * @param[in] x2 object 2 + * @param[in] same If set, x1 and x2 are member of same parent & enumeration + * is used (see explanation below) + * @retval 0 If equal + * @retval <0 If x1 is less than x2 + * @retval >0 If x1 is greater than x2 * @see xml_cmp1 Similar, but for one object + * + * There are distinct calls for this function: + * 1. For sorting in an existing list of XML children + * 2. For searching of an existing element in a list + * In the first case, there is a special case for "ordered-by-user", where + * if they have the same yang-spec, the existing order is used as tie-breaker. + * In other words, if order-by-system, or if the case (2) above, the existing + * order is ignored and the actual xml element contents is examined. * @note empty value/NULL is smallest value - * @note xml_enumerate_children must have been called prior to this call + * @note some error cases return as -1 (qsort cant handle errors) * @note some error cases return as -1 (qsort cant handle errors) */ -static int +int xml_cmp(cxobj *x1, - cxobj *x2) + cxobj *x2, + int same) { yang_stmt *y1; yang_stmt *y2; @@ -217,18 +229,18 @@ xml_cmp(cxobj *x1, int nr2 = 0; cxobj *x1b; cxobj *x2b; - int e; - e=0; if (x1==NULL || x2==NULL) goto done; /* shouldnt happen */ - e=1; y1 = xml_spec(x1); y2 = xml_spec(x2); - nr1 = xml_enumerate_get(x1); - nr2 = xml_enumerate_get(x2); + if (same){ + nr1 = xml_enumerate_get(x1); + nr2 = xml_enumerate_get(x2); + } if (y1==NULL && y2==NULL){ - equal = nr1-nr2; + if (same) + equal = nr1-nr2; goto done; } if (y1==NULL){ @@ -239,24 +251,24 @@ xml_cmp(cxobj *x1, equal = 1; goto done; } - e=2; if (y1 != y2){ yi1 = yang_order(y1); yi2 = yang_order(y2); if ((equal = yi1-yi2) != 0) goto done; } - e=3; /* Now y1==y2, same Yang spec, can only be list or leaf-list, * But first check exceptions, eg config false or ordered-by user * otherwise sort according to key + * If the two elements are in the same list, and they are ordered-by user + * then do not look more into equivalence, use the enumeration in the + * existing list. */ - if (yang_config(y1)==0 || - yang_find(y1, Y_ORDERED_BY, "user") != NULL){ - equal = nr1-nr2; - goto done; /* Ordered by user or state data : maintain existing order */ - } - e=4; + if (same && + (yang_config(y1)==0 || yang_find(y1, Y_ORDERED_BY, "user") != NULL)){ + equal = nr1-nr2; + goto done; /* Ordered by user or state data : maintain existing order */ + } switch (yang_keyword_get(y1)){ case Y_LEAF_LIST: /* Match with name and value */ if ((b1 = xml_body(x1)) == NULL) @@ -287,8 +299,10 @@ xml_cmp(cxobj *x1, else{ if (xml_cv_cache(x1b, &cv1) < 0) /* error case */ goto done; + assert(cv1); if (xml_cv_cache(x2b, &cv2) < 0) /* error case */ goto done; + assert(cv2); if ((equal = cv_cmp(cv1, cv2)) != 0) goto done; } @@ -298,9 +312,8 @@ xml_cmp(cxobj *x1, default: break; } - e=5; done: - clicon_debug(2, "%s %s %s %d %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, e, nr1, nr2, yi1, yi2); + clicon_debug(2, "%s %s %s %d nr: %d %d yi: %d %d", __FUNCTION__, xml_name(x1), xml_name(x2), equal, nr1, nr2, yi1, yi2); return equal; } @@ -311,95 +324,9 @@ static int xml_cmp_qsort(const void* arg1, const void* arg2) { - return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2); + return xml_cmp(*(struct xml**)arg1, *(struct xml**)arg2, 1); } -/*! Compare xml object - * @param[in] x XML node to compare with - * @param[in] y The yang spec of x - * @param[in] name Name to compare with x - * @param[in] keyword Yang keyword (stmt type) to compare w x/y - * @param[in] keynr Length of keyvec/keyval vector when applicable - * @param[in] keyvec Array of of yang key identifiers - * @param[in] keyval Array of of yang key values - * @param[out] userorder If set, this yang order is user ordered, linear search - * @retval 0 If equal (or userorder set) - * @retval <0 if arg1 is less than arg2 - * @retval >0 if arg1 is greater than arg2 - * @see xml_cmp Similar, but for two objects - * @note Does not care about y type of value as xml_cmp - */ -static int -xml_cmp1(cxobj *x, - yang_stmt *y, - char *name, - enum rfc_6020 keyword, - int keynr, - char **keyvec, - char **keyval, - cg_var **keycvec, - int *userorder) -{ - char *b; - cxobj *xb; - int i; - char *keyname; - char *key; - int match = 0; - cg_var *cv; - - /* state data = userorder */ - if (userorder && yang_config(y)==0) - *userorder=1; - /* Check if same yang spec (order in yang stmt list) */ - switch (keyword){ - case Y_CONTAINER: /* Match with name */ - case Y_LEAF: /* Match with name */ - match = strcmp(name, xml_name(x)); - break; - case Y_LEAF_LIST: /* Match with name and value */ - if (userorder && yang_find(y, Y_ORDERED_BY, "user") != NULL) - *userorder=1; - if ((b=xml_body(x)) == NULL) - match = 1; - else{ - if (keycvec[0]){ - if (xml_cv_cache(x, &cv) < 0) /* error case */ - goto done; - match = cv_cmp(keycvec[0], cv); - } - else - match = strcmp(keyval[0], b); - } - break; - case Y_LIST: /* Match with array of key values */ - if (userorder && yang_find(y, Y_ORDERED_BY, "user") != NULL) - *userorder=1; - /* All must match */ - for (i=0; i=0; i--){ /* Then decrement */ - xc = xml_child_i(x0, i); + xc = xml_child_i(xp, i); y = xml_spec(xc); if (yangi!=yang_order(y)) break; - if (xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, NULL) == 0) + if (xml_cmp(xc, x1, 0) == 0) return xc; } return NULL; /* Not found */ } /*! + * @param[in] xp Parent xml node. * @param[in] yangi Yang order * @param[in] keynr Length of keyvec/keyval vector when applicable * @param[in] keyvec Array of of yang key identifiers @@ -469,14 +393,10 @@ xml_search_userorder(cxobj *x0, * @param[in] upper Lower bound of childvec search interval */ static cxobj * -xml_search1(cxobj *x0, - char *name, +xml_search1(cxobj *xp, + cxobj *x1, + int userorder, int yangi, - enum rfc_6020 keyword, - int keynr, - char **keyvec, - char **keyval, - cg_var **keycvec, int low, int upper) { @@ -484,122 +404,200 @@ xml_search1(cxobj *x0, int cmp; cxobj *xc; yang_stmt *y; - int userorder= 0; - + if (upper < low) return NULL; /* not found */ mid = (low + upper) / 2; - if (mid >= xml_child_nr(x0)) /* beyond range */ + if (mid >= xml_child_nr(xp)) /* beyond range */ return NULL; - xc = xml_child_i(x0, mid); + xc = xml_child_i(xp, mid); if ((y = xml_spec(xc)) == NULL) return NULL; cmp = yangi-yang_order(y); /* Here is right yang order == same yang? */ if (cmp == 0){ - cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, &userorder); - if (userorder && cmp) /* Look inside this yangi order */ - return xml_search_userorder(x0, y, name, yangi, mid, keyword, keynr, keyvec, keyval, keycvec); + if (userorder){ + return xml_search_userorder(xp, x1, y, yangi, mid); + } + else /* Ordered by system */ + cmp = xml_cmp(x1, xc, 0); } if (cmp == 0) return xc; else if (cmp < 0) - return xml_search1(x0, name, yangi, keyword, - keynr, keyvec, keyval, keycvec, low, mid-1); + return xml_search1(xp, x1, userorder, yangi, low, mid-1); else - return xml_search1(x0, name, yangi, keyword, - keynr, keyvec, keyval, keycvec, mid+1, upper); + return xml_search1(xp, x1, userorder, yangi, mid+1, upper); return NULL; } -/*! Find XML children using binary search - * @param[in] yangi yang child order +/*! Find XML child under xp matching x1 using binary search + * @param[in] xp Parent xml node. + * @param[in] yangi Yang child order * @param[in] keynr Length of keyvec/keyval vector when applicable * @param[in] keyvec Array of of yang key identifiers * @param[in] keyval Array of of yang key values */ static cxobj * -xml_search(cxobj *x0, - char *name, - int yangi, - enum rfc_6020 keyword, - int keynr, - char **keyvec, - char **keyval, - cg_var **keycvec) +xml_search(cxobj *xp, + cxobj *x1, + yang_stmt *yc) { - cxobj *xa; - int low = 0; - int high = xml_child_nr(x0); - + cxobj *xa; + int low = 0; + int upper = xml_child_nr(xp); + int userorder=0; + cxobj *xret = NULL; + int yangi; + /* Assume if there are any attributes, they are first in the list, mask them by raising low to skip them */ - for (low=0; low upper){ /* beyond range */ + clicon_err(OE_XML, 0, "low>upper %d %d", low, upper); + goto done; + } + if (low == upper){ + retval = low; + goto done; + } + mid = (low + upper) / 2; + if (mid >= xml_child_nr(xp)){ /* beyond range */ + clicon_err(OE_XML, 0, "Beyond range %d %d %d", low, mid, upper); + goto done; + } + xc = xml_child_i(xp, mid); + if ((yc = xml_spec(xc)) == NULL){ + clicon_err(OE_XML, 0, "No spec found %s", xml_name(xc)); + goto done; + } + if (yc == yn){ /* Same yang */ + if (userorder){ /* append: increment linearly until no longer equal */ + for (i=mid+1; i, xn = + * same order but different yang spec + */ + } + if (low +1 == upper){ /* termination criterium */ + if (cmp<0) { + retval = mid; + goto done; + } + retval = mid+1; + goto done; + } + if (cmp == 0){ + retval = mid; + goto done; + } + else if (cmp < 0) + return xml_insert2(xp, xn, yn, yni, userorder, low, mid); + else + return xml_insert2(xp, xn, yn, yni, userorder, mid+1, upper); + done: + return retval; +} + +/*! Insert xc as child to xp in sorted place. Remove xc from previous parent. + * @param[in] xp Parent xml node. If NULL just remove from old parent. + * @param[in] x Child xml node to insert under xp + * @retval 0 OK + * @retval -1 Error + * @see xml_addsub where xc is appended. xml_insert is xml_addsub();xml_sort() */ int -xml_insert_pos(cxobj *x0, - char *name, - int yangi, - enum rfc_6020 keyword, - int keynr, - char **keyvec, - char **keyval, - int low, - int upper) +xml_insert(cxobj *xp, + cxobj *xi) { - int mid; - cxobj *xc; + int retval = -1; + cxobj *xa; + int low = 0; + int upper; yang_stmt *y; - int cmp; - int i; int userorder= 0; - - if (upper < low) - return low; /* not found */ - mid = (low + upper) / 2; - if (mid >= xml_child_nr(x0)) - return xml_child_nr(x0); /* upper range */ - xc = xml_child_i(x0, mid); - y = xml_spec(xc); - cmp = yangi-yang_order(y); - if (cmp == 0){ - cmp = xml_cmp1(xc, y, name, keyword, keynr, keyvec, keyval, keycvec, &userorder); - if (userorder){ /* Look inside this yangi order */ - /* Special case: append last of equals if ordered by user */ - for (i=mid+1;i 0) + if (xml_cmp(xprev, x, 1) > 0) goto done; } xprev = x; @@ -652,15 +650,8 @@ match_base_child(cxobj *x0, int retval = -1; cvec *cvk = NULL; /* vector of index keys */ cg_var *cvi; - char *b; cxobj *xb; char *keyname; - char keynr = 0; - char **keyval = NULL; - char **keyvec = NULL; - cg_var **keycvec = NULL; - int i; - int yorder; cxobj *x0c = NULL; yang_stmt *y0c; yang_stmt *y0p; @@ -686,19 +677,10 @@ match_base_child(cxobj *x0, case Y_LEAF: /* Equal regardless */ break; case Y_LEAF_LIST: /* Match with name and value */ - keynr = 1; - if ((keyval = calloc(keynr+1, sizeof(char*))) == NULL){ - clicon_err(OE_UNIX, errno, "calloc"); - goto done; - } - if ((keyval[0] = xml_body(x1c)) == NULL) + if (xml_body(x1c) == NULL){ /* Treat as empty string */ + // assert(0); goto ok; - if ((keycvec = calloc(keynr+1, sizeof(cg_var*))) == NULL){ - clicon_err(OE_UNIX, errno, "calloc"); - goto done; } - if (xml_cv_cache(x1c, &keycvec[0]) < 0) /* error case */ - goto done; break; case Y_LIST: /* Match with key values */ cvk = yang_cvec_get(yc); /* Use Y_LIST cache, see ys_populate_list() */ @@ -706,51 +688,22 @@ match_base_child(cxobj *x0, * Then create two vectors one with names and one with values of x1c, * ec: keyvec: [a,b,c] keyval: [1,2,3] */ - cvi = NULL; keynr = 0; - while ((cvi = cvec_each(cvk, cvi)) != NULL) - keynr++; - if ((keyval = calloc(keynr+1, sizeof(char*))) == NULL){ - clicon_err(OE_UNIX, errno, "calloc"); - goto done; - } - if ((keyvec = calloc(keynr+1, sizeof(char*))) == NULL){ - clicon_err(OE_UNIX, errno, "calloc"); - goto done; - } - if ((keycvec = calloc(keynr+1, sizeof(char*))) == NULL){ - clicon_err(OE_UNIX, errno, "calloc"); - goto done; - } - cvi = NULL; i = 0; + cvi = NULL; while ((cvi = cvec_each(cvk, cvi)) != NULL) { keyname = cv_string_get(cvi); - keyvec[i] = keyname; - if ((xb = xml_find(x1c, keyname)) == NULL) + // keyvec[i] = keyname; + if ((xb = xml_find(x1c, keyname)) == NULL){ goto ok; - if ((b = xml_body(xb)) == NULL) - goto ok; - keyval[i] = b; - if (xml_cv_cache(xb, &keycvec[i]) < 0) /* error case */ - goto done; - i++; + } } - break; default: break; } /* Get match. */ - yorder = yang_order(yc); - x0c = xml_search(x0, xml_name(x1c), yorder, yang_keyword_get(yc), keynr, keyvec, keyval, keycvec); + x0c = xml_search(x0, x1c, yc); ok: *x0cp = x0c; retval = 0; - done: - if (keyval) - free(keyval); - if (keyvec) - free(keyvec); - if (keycvec) - free(keycvec); return retval; } diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index b6706087..25722f11 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -69,6 +69,7 @@ #include #include #include +#include /* cligen */ #include @@ -223,6 +224,46 @@ yang_cvec_get(yang_stmt *ys) return ys->ys_cvec; } +/*! Set yang statement CLIgen variable vector + * @param[in] ys Yang statement node + * @param[in] cvec CLIgen vector + * @retval 0 OK + * @retval -1 Error + */ +int +yang_cvec_set(yang_stmt *ys, + cvec *cvv) +{ + if (ys->ys_cvec) + cvec_free(ys->ys_cvec); + ys->ys_cvec = cvv; + return 0; +} + +/*! Get regular expression cache - the compiled regex + * @param[in] ys Yang statement node + * @retval re Compiled regex + * @see regcomp + */ +void* +yang_regex_cache_get(yang_stmt *ys) +{ + return ys->ys_regex_cache; +} + +/*! Set regular expression cache - the compiled regex + * @param[in] ys Yang statement node + * @param[in] re Compiled regex + * @see regcomp + */ +int +yang_regex_cache_set(yang_stmt *ys, + void *regex) +{ + ys->ys_regex_cache = regex; + return 0; +} + /* End access functions */ /*! Create new yang specification @@ -267,6 +308,17 @@ ys_new(enum rfc_6020 keyw) return ys; } + +static int +yang_regex_cache_free(yang_stmt *ys) +{ + if (ys->ys_regex_cache){ + regfree(ys->ys_regex_cache); + free(ys->ys_regex_cache); + } + return 0; +} + /*! Free a single yang statement */ static int ys_free1(yang_stmt *ys) @@ -281,6 +333,7 @@ ys_free1(yang_stmt *ys) cvec_free(ys->ys_cvec); if (ys->ys_typecache) yang_type_cache_free(ys->ys_typecache); + yang_regex_cache_free(ys); free(ys); return 0; } @@ -720,6 +773,30 @@ yang_choice(yang_stmt *y) return NULL; } +static int +order1_choice(yang_stmt *yp, + yang_stmt *y) +{ + yang_stmt *ys; + yang_stmt *yc; + int i; + int j; + + for (i=0; iys_len; i++){ + ys = yp->ys_stmt[i]; + if (ys->ys_keyword == Y_CASE){ + for (j=0; jys_len; j++){ + yc = ys->ys_stmt[j]; + if (yang_datanode(yc) && yc == y) + return 1; + } + } + else if (yang_datanode(ys) && ys == y) + return 1; + } + return 0; +} + /*! Find matching y in yp:s children, return 0 and index or -1 if not found. * @param[in] yp Parent * @param[in] y Yang datanode to find @@ -737,10 +814,16 @@ order1(yang_stmt *yp, for (i=0; iys_len; i++){ ys = yp->ys_stmt[i]; - if (!yang_datanode(ys)) - continue; - if (ys==y) - return 1; + if (ys->ys_keyword == Y_CHOICE){ + if (order1_choice(ys, y) == 1) /* If one of the choices is "y" */ + return 1; + } + else { + if (!yang_datanode(ys)) + continue; + if (ys==y) + return 1; + } (*index)++; } return 0; @@ -763,7 +846,14 @@ yang_order(yang_stmt *y) int j=0; int tot = 0; + /* Some special handling if yp is choice (or case) and maybe union? + * if so, the real parent (from an xml point of view) is the parents + * parent. + */ yp = y->ys_parent; + while (yp->ys_keyword == Y_CASE || yp->ys_keyword == Y_CHOICE) + yp = yp->ys_parent; + /* XML nodes with yang specs that are children of modules are special - * In clixon, they are seen as an "implicit" container where the XML can come from different * modules. The order must therefore be global among yang top-symbols to be unique. @@ -1950,6 +2040,7 @@ yang_parse_str(char *str, * @param[in] ysp Yang specification. Should have been created by caller using yspec_new * @retval ymod Top-level yang (sub)module * @retval NULL Error + * @note this function simply parse a yang spec, no dependencies or checks */ yang_stmt * yang_parse_file(int fd, diff --git a/lib/src/clixon_yang_internal.h b/lib/src/clixon_yang_internal.h index e554e6ef..be561cc7 100644 --- a/lib/src/clixon_yang_internal.h +++ b/lib/src/clixon_yang_internal.h @@ -95,6 +95,7 @@ struct yang_stmt{ Y_TYPE & identity: store all derived types */ yang_type_cache *ys_typecache; /* If ys_keyword==Y_TYPE, cache all typedef data except unions */ + void *ys_regex_cache; /* regex cache */ int _ys_vector_i; /* internal use: yn_each */ }; diff --git a/lib/src/clixon_yang_module.c b/lib/src/clixon_yang_module.c index 3bfea1cb..96593246 100644 --- a/lib/src/clixon_yang_module.c +++ b/lib/src/clixon_yang_module.c @@ -160,7 +160,8 @@ yang_modules_revision(clicon_handle h) } /*! Actually build the yang modules state XML tree -*/ + * @see RFC7895 + */ static int yms_build(clicon_handle h, yang_stmt *yspec, @@ -198,8 +199,11 @@ yms_build(clicon_handle h, cprintf(cb,"%s", ymod->ys_argument); if ((ys = yang_find(ymod, Y_REVISION, NULL)) != NULL) cprintf(cb,"%s", ys->ys_argument); - else + else{ + /* RFC7895 1 If no (such) revision statement exists, the module's or + submodule's revision is the zero-length string. */ cprintf(cb,""); + } if ((ys = yang_find(ymod, Y_NAMESPACE, NULL)) != NULL) cprintf(cb,"%s", ys->ys_argument); else diff --git a/lib/src/clixon_yang_type.c b/lib/src/clixon_yang_type.c index 34664487..5ef07be6 100644 --- a/lib/src/clixon_yang_type.c +++ b/lib/src/clixon_yang_type.c @@ -49,6 +49,7 @@ #include #include #include +#include #include /* cligen */ @@ -80,19 +81,19 @@ static const map_str2int ytmap[] = { {"int32", CGV_INT32}, /* NOTE, first match on right is significant, dont move */ {"string", CGV_STRING}, /* NOTE, first match on right is significant, dont move */ {"string", CGV_REST}, /* For cv -> yang translation of rest */ - {"binary", CGV_STRING}, - {"bits", CGV_STRING}, + {"binary", CGV_STRING}, + {"bits", CGV_STRING}, {"boolean", CGV_BOOL}, - {"decimal64", CGV_DEC64}, + {"decimal64", CGV_DEC64}, {"empty", CGV_VOID}, /* May not include any content */ - {"enumeration", CGV_STRING}, + {"enumeration", CGV_STRING}, {"identityref", CGV_STRING}, /* XXX */ {"instance-identifier", CGV_STRING}, /* XXX */ - {"int8", CGV_INT8}, - {"int16", CGV_INT16}, + {"int8", CGV_INT8}, + {"int16", CGV_INT16}, {"int64", CGV_INT64}, {"leafref", CGV_STRING}, /* XXX */ - {"uint8", CGV_UINT8}, + {"uint8", CGV_UINT8}, {"uint16", CGV_UINT16}, {"uint32", CGV_UINT32}, {"uint64", CGV_UINT64}, @@ -100,11 +101,106 @@ static const map_str2int ytmap[] = { {NULL, -1} }; +/*! Mapping from yang string types --> cligen types + * @note not 100% same as map_str2int since it has significant order AND + * string->CGV_REST entry removed + */ +static const map_str2int ytmap2[] = { + {"binary", CGV_STRING}, + {"bits", CGV_STRING}, + {"boolean", CGV_BOOL}, + {"decimal64", CGV_DEC64}, + {"empty", CGV_VOID}, /* May not include any content */ + {"enumeration", CGV_STRING}, + {"identityref", CGV_STRING}, /* XXX */ + {"instance-identifier", CGV_STRING}, /* XXX */ + {"int16", CGV_INT16}, + {"int32", CGV_INT32}, + {"int64", CGV_INT64}, + {"int8", CGV_INT8}, + {"leafref", CGV_STRING}, /* XXX */ + {"string", CGV_STRING}, + {"uint16", CGV_UINT16}, + {"uint32", CGV_UINT32}, + {"uint64", CGV_UINT64}, + {"uint8", CGV_UINT8}, + {"union", CGV_REST}, /* Is replaced by actual type */ + {NULL, -1} +}; + +/*! Regular expression compiling + * @retval -1 Error + * @retval 0 regex problem (no match?) + * @retval 1 OK Match + * @see match_regexp the CLIgen original composite function + */ +static int +regex_compile(char *pattern0, + regex_t *re) +{ + int retval = -1; + char pattern[1024]; + // char errbuf[1024]; + int len0; + int status; + + len0 = strlen(pattern0); + if (len0 > sizeof(pattern)-5){ + clicon_err(OE_XML, EINVAL, "pattern too long"); + goto done; + } + strncpy(pattern, "^(", 2); + strncpy(pattern+2, pattern0, sizeof(pattern)-2); + strncat(pattern, ")$", sizeof(pattern)-len0-1); + if ((status = regcomp(re, pattern, REG_NOSUB|REG_EXTENDED)) != 0) { +#if 0 /* ignore error msg for now */ + regerror(status, re, errbuf, sizeof(errbuf)); +#endif + goto fail; + } + retval = 1; + done: + return retval; + fail: + retval = 0; + goto done; +} + +/*! Regular expression execution + * @retval -1 Error + * @retval 0 regex problem (no match?) + * @retval 1 OK Match + * @see match_regexp the CLIgen original composite function + */ +static int +regex_exec(regex_t *re, + char *string) +{ + int retval = -1; + int status; + // char errbuf[1024]; + + status = regexec(re, string, (size_t) 0, NULL, 0); + if (status != 0) { +#if 0 /* ignore error msg for now */ + regerror(status, re, errbuf, sizeof(errbuf)); +#endif + goto fail; + } + retval = 1; + done: + return retval; + fail: + retval = 0; + goto done; +} + + /* return 1 if built-in, 0 if not */ static int yang_builtin(char *type) { - if (clicon_str2int(ytmap, type) != -1) + if (clicon_str2int_search(ytmap2, type, (sizeof(ytmap)/sizeof(map_str2int))-2) != -1) return 1; return 0; } @@ -250,7 +346,7 @@ yang2cv_type(char *ytype, *cv_type = CGV_ERR; /* built-in types */ - if ((ret = clicon_str2int(ytmap, ytype)) != -1){ + if ((ret = clicon_str2int_search(ytmap2, ytype, (sizeof(ytmap)/sizeof(map_str2int))-2)) != -1){ *cv_type = ret; return 0; } @@ -526,14 +622,41 @@ cv_validate1(cg_var *cv, } if ((options & YANG_OPTIONS_PATTERN) != 0){ char *posix = NULL; - if (regexp_xsd2posix(pattern, &posix) < 0) - goto done; - if ((retval2 = match_regexp(str?str:"", posix)) < 0){ - clicon_err(OE_DB, 0, "match_regexp: %s", pattern); - return -1; + regex_t *re = NULL; + + if ((re = yang_regex_cache_get(yrestype)) == NULL){ + /* Transform to posix regex */ + if (regexp_xsd2posix(pattern, &posix) < 0) + goto done; + /* Create regex cache */ + if ((re = malloc(sizeof(*re))) == NULL){ + clicon_err(OE_UNIX, errno, "malloc"); + goto done; + } + memset(re, 0, sizeof(*re)); + /* Compute regex pattern for use in patterns */ + if ((retval2 = regex_compile(posix, re)) < 0) + goto done; + if (retval2 == 0){ + if (reason) + *reason = cligen_reason("regexp match fail: \"%s\" does not match %s", + str, pattern); + goto fail; + break; + } + yang_regex_cache_set(yrestype, re); + if (posix) + free(posix); + } + if ((retval2 = regex_exec(re, str?str:"")) < 0) + goto done; + if (retval2 == 0){ + if (reason) + *reason = cligen_reason("regexp match fail: \"%s\" does not match %s", + str, pattern); + goto fail; + break; } - if (posix) - free(posix); if (retval2 == 0){ if (reason) *reason = cligen_reason("regexp match fail: \"%s\" does not match %s", @@ -668,7 +791,7 @@ ys_cv_validate_union(yang_stmt *ys, /*! Validate cligen variable cv using yang statement as spec * * @param[in] cv A cligen variable to validate. This is a correctly parsed cv. - * @param[in] ys A yang statement, must be leaf of leaf-list. + * @param[in] ys A yang statement, must be leaf or leaf-list. * @param[out] reason If given, and if return value is 0, contains a malloced string * describing the reason why the validation failed. Must be freed. * @retval -1 Error (fatal), with errno set to indicate error diff --git a/test/README.md b/test/README.md index d11a62ef..809f00db 100644 --- a/test/README.md +++ b/test/README.md @@ -60,6 +60,10 @@ The `mem.sh` runs memory checks using valgrind. Start it with no arguments to te mem.sh restconf backend # Only backend and cli ``` +## Performance plots + +The script `plot_perf.sh` produces gnuplots for some testcases. + ## Site.sh You may add your site-specific modifications in a `site.sh` file. Example: ``` diff --git a/test/lib.sh b/test/lib.sh index 38a01380..f2a9d32c 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -201,6 +201,7 @@ new(){ # - expected command return value (0 if OK) # - expected stdout outcome, # - expected2 stdout outcome, +# Example: expectfn "$clixon_cli -1 -f $cfg show conf cli" 0 "^$" expectfn(){ cmd=$1 retval=$2 @@ -212,7 +213,7 @@ expectfn(){ expect2= fi ret=$($cmd) - r=$? + r=$? # echo "cmd:\"$cmd\"" # echo "retval:\"$retval\"" # echo "expect:\"$expect\"" @@ -221,7 +222,7 @@ expectfn(){ if [ $r != $retval ]; then echo -e "\e[31m\nError ($r != $retval) in Test$testnr [$testname]:" echo -e "\e[0m:" - return + exit -1 fi # if [ $r != 0 ]; then # return diff --git a/test/plot_perf.sh b/test/plot_perf.sh index dc565817..9d394746 100755 --- a/test/plot_perf.sh +++ b/test/plot_perf.sh @@ -1,13 +1,48 @@ #!/bin/bash -# Transactions per second for large lists read/write plotter using gnuplot -# WORK IN PROGRESS -. ./lib.sh -max=2000 # Nr of db entries -step=200 -reqs=500 +# Performance of large lists. See large-lists.md +# The parameters are shown below (under Default values) +# Examples +# 1. run all measurements up to 10000 entris collect all results in /tmp/plots +# run=true plot=false to=10000 resdir=/tmp/plots ./plot_perf.sh +# 2. Use existing data plot and show on X11 +# run=false plot=true resdir=/tmp/plots term=x11 ./plot_perf.sh +# 3. Use existing data plot i686 and armv7l data as png +# archs="i686 armv7l" run=false plot=true resdir=/tmp/plots term=png ./plot_perf.sh + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +arch=$(arch) +# Default values +: ${to:=5000} # Max N +: ${step=1000} # Iterate in steps (also starting point) +: ${reqs=100} # Number of requests in each burst +: ${run:=true} # run tests (or skip them). If false just plot +: ${term:=x11} # x11 interactive, alt: png +: ${resdir=$dir} # Result dir (both data and gnuplot) +: ${plot=false} # Result dir (both data and gnuplot) +: ${archs=$arch} # Plotting can be made for many architectures (not run) + +# 0 prefix to protect against shell dynamic binding) +to0=$to +step0=$step +reqs0=$reqs + +ext=$term # gnuplot output file extenstion + +# Global variables +APPNAME=example cfg=$dir/plot-conf.xml fyang=$dir/plot.yang -fconfig=$dir/config +fxml=$dir/data.xml +fjson=$dir/data.json + +# Resultdir - if different from $dir that gets erased +#resdir=$dir + +if [ ! -d $resdir ]; then + mkdir $resdir +fi # For memcheck # clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" @@ -15,149 +50,479 @@ fconfig=$dir/config clixon_netconf=clixon_netconf cat < $fyang -module ietf-ip{ +module scaling{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix sc; container x { - list y { - key "a"; - leaf a { - type string; + description "top-level container"; + list y { + description "List with potential large number of elements"; + key "a"; + leaf a { + description "key in list"; + type int32; + } + leaf b { + description "payload data"; + type string; + } } - leaf b { - type string; - } - } - leaf-list c { - type string; - } - } + } } EOF cat < $cfg - + $cfg - $fyang - ietf-ip - /usr/local/var/routing/routing.sock - /usr/local/var/routing/routing.pidfile -false - /usr/local/var/routing - + $dir + /usr/local/share/clixon + scaling + /usr/local/var/example/example.sock + /usr/local/var/example/example.pidfile + false + $dir + false + EOF -run(){ - nr=$1 # Number of entries in DB - reqs=$2 - mode=$3 - - echo -n "replace" > $fconfig - for (( i=0; i<$nr; i++ )); do - case $mode in - readlist|writelist|restreadlist|restwritelist) - echo -n "$i$i" >> $fconfig - ;; - writeleaflist) - echo -n "$i" >> $fconfig - ;; - esac - done +# Generate file with n entries +# argument: +genfile(){ + if [ $2 = netconf ]; then + echo -n "replace" > $fxml + for (( i=0; i<$1; i++ )); do + echo -n "$i$i" >> $fxml + done + echo "]]>]]>" >> $fxml + else # restconf + echo -n '{"scaling:x":{"y":[' > $fjson + for (( i=0; i<$1; i++ )); do + if [ $i -ne 0 ]; then + echo -n ',' >> $fjson + fi + echo -n "{\"a\":$i,\"b\":\"$i\"}" >> $fjson + done + echo ']}}' >> $fjson + fi +} - echo "]]>]]>" >> $fconfig +# Run netconffunction +# args: +# where proto is one of: +# netconf, restconf +# where op is one of: +# get put delete commit +runnet(){ + op=$1 + nr=$2 # Number of entries in DB (keep diff from n due to shell dynamic binding) + reqs=$3 - expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" - - case $mode in - readlist) - time -p for (( i=0; i<$reqs; i++ )); do - rnd=$(( ( RANDOM % $nr ) )) - echo "]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null - ;; - writelist) - time -p for (( i=0; i<$reqs; i++ )); do - rnd=$(( ( RANDOM % $nr ) )) - echo "$rnd$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null - ;; - restreadlist) - time -p for (( i=0; i<$reqs; i++ )); do - rnd=$(( ( RANDOM % $nr ) )) - curl -sSG http://localhost/restconf/data/x/y=$rnd,$rnd > /dev/null -done - ;; - writeleaflist) - time -p for (( i=0; i<$reqs; i++ )); do - rnd=$(( ( RANDOM % $nr ) )) - echo "$rnd]]>]]>" -done | $clixon_netconf -qf $cfg -y $fyang > /dev/null + file=$resdir/$op-netconf-$reqs-$arch + echo -n "$nr " >> $file + case $op in + put) + if [ $reqs = 0 ]; then # Write all in one go + genfile $nr netconf; + { time -p cat $fxml | $clixon_netconf -qf $cfg -y $fyang ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + else # reqs != 0 + { time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )); + echo "$rnd$rnd]]>]]>"; + done | $clixon_netconf -qf $cfg -y $fyang > /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + fi ;; + get) + if [ $reqs = 0 ]; then # Read all in one go + { time -p echo "]]>]]>" | $clixon_netconf -qf $cfg -y $fyang > /dev/null ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + else # reqs != 0 + { time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "$rnd$rnd]]>]]>" + done | $clixon_netconf -qf $cfg -y $fyang > /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + fi + ;; + delete) + { time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )) + echo "$rnd$rnd]]>]]>" +done | $clixon_netconf -qf $cfg -y $fyang; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + ;; + commit) + { time -p echo "]]>]]>" | $clixon_netconf -qf $cfg -y $fyang > /dev/null ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + ;; + *) + err "Operation not supported" "$op" + exit + ;; esac - expecteof "$clixon_netconf -qf $cfg -y $fyang" "]]>]]>" "^]]>]]>$" - } -step(){ - i=$1 - mode=$2 - echo -n "" > $fconfig - t=$(TEST=%e run $i $reqs $mode 2>&1 | awk '/real/ {print $2}') - #TEST=%e run $i $reqs $mode 2>&1 - # t is time in secs of $reqs -> transactions per second. $reqs - p=$(echo "$reqs/$t" | bc -lq) - # p is transactions per second. - echo "$i $p" >> $dir/$mode -# echo "m:$mode i:$i t=$t p=$p" +# Run restconf function +# args: +# where proto is one of: +# netconf, restconf +# where op is one of: +# get put delete +runrest(){ + op=$1 + nr=$2 # Number of entries in DB + reqs=$3 + + file=$resdir/$op-restconf-$reqs-$arch + echo -n "$nr " >> $file + case $op in + put) + if [ $reqs = 0 ]; then # Write all in one go + genfile $nr restconf + # restconf @- means from stdin + { time -p curl -sS -X PUT -d @$fjson http://localhost/restconf/data/scaling:x ; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + else # Small requests + { time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )); + curl -sS -X PUT http://localhost/restconf/data/scaling:x/y=$rnd -d "{\"scaling:y\":{\"a\":$rnd,\"b\":\"$rnd\"}}" + done ; } 2>&1 | awk '/real/ {print $2}' | tr , .>> $file + # + fi + ;; + get) + if [ $reqs = 0 ]; then # Read all in one go + { time -p curl -sS -X GET http://localhost/restconf/data/scaling:x > /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $file + else # Small requests + { time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )); + curl -sS -X GET http://localhost/restconf/data/scaling:x/y=$rnd + done ; } 2>&1 | awk '/real/ {print $2}' | tr , .>> $file + fi + ;; + delete) + { time -p for (( i=0; i<$reqs; i++ )); do + rnd=$(( ( RANDOM % $nr ) )); + curl -sS -X GET http://localhost/restconf/data/scaling:x/y=$rnd + done ; } 2>&1 | awk '/real/ {print $2}' | tr , .>> $file + ;; + *) + err "Operation not supported" "$op" + exit + ;; + esac } -once(){ - # kill old backend (if any) - sudo clixon_backend -zf $cfg -y $fyang - if [ $? -ne 0 ]; then - err - fi - # start new backend - sudo clixon_backend -s init -f $cfg -y $fyang - if [ $? -ne 0 ]; then - err - fi +commit(){ + # commit to running + expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" +} - # Always as a start - for (( i=10; i<=$step; i=i+10 )); do - step $i readlist - step $i writelist - step $i restreadlist - step $i writeleaflist +reset(){ + # delete all in candidate + expecteof "$clixon_netconf -qf $cfg -y $fyang" 0 "none]]>]]>" '^]]>]]>$' + # commit to running + commit +} + +# Load n entries into candidate +# Args: +load(){ + # Generate file ($fxml) + genfile $1 netconf + # Write it to backend in one chunk + expecteof_file "$clixon_netconf -qf $cfg -y $fyang" "$fxml" "^]]>]]>$" +} + +# Run an operation, iterate from to in increment of +# Each operation do times +# args: +# =0 means all in one go +# means a priori loaded into datastore +plot(){ + op=$1 + proto=$2 + from=$3 + step=$4 + to=$5 + reqs=$6 + can=$7 + run=$8 + + if [ $# -ne 8 ]; then + exit "plot should be called with 8 arguments, got $#" + fi + + # reset file + new "Create file $resdir/$op-$proto-$reqs-$arch" + echo -n "" > $resdir/$op-$proto-$reqs-$arch + for (( n=$from; n<=$to; n=$n+$step )); do + reset + if [ $can = n ]; then + load $n + if [ $run = n ]; then + commit + fi + fi + new "$op-$proto-$reqs-$arch $n" + if [ $proto = netconf ]; then + runnet $op $n $reqs + else + runrest $op $n $reqs + fi done - # Actual steps - for (( i=$step; i<=$max; i=i+$step )); do - step $i readlist - step $i writelist - step $i restreadlist - step $i writeleaflist + echo # newline +} + +# Run an operation, iterate from to in increment of +# Each operation do times +# args: +# =0 means all in one go +startup(){ + from=$1 + step=$2 + to=$3 + mode=startup + + if [ $# -ne 3 ]; then + exit "plot should be called with 3 arguments, got $#" + fi + + # gnuplot file + gfile=$resdir/startup-$arch + new "Create file $gfile" + echo -n "" > $gfile + + # Startup db: load with n entries + dbfile=$dir/${mode}_db + sudo touch $dbfile + sudo chmod 666 $dbfile + for (( n=$from; n<=$to; n=$n+$step )); do + new "startup-$arch $n" + new "Generate $n entries to $dbfile" + echo -n "" > $dbfile + for (( i=0; i<$n; i++ )); do + echo -n "$i$i" >> $dbfile + done + echo "" >> $dbfile + + new "Startup backend once -s $mode -f $cfg -y $fyang" + echo -n "$n " >> $gfile + { time -p sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang 2> /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $gfile + + done + echo # newline +} + +if $run; then + + # Startup test before regular backend/restconf start since we only start + # backend a single time + startup $step $step $to + + new "test params: -f $cfg -y $fyang" + if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg -y $fyang" + start_backend -s init -f $cfg -y $fyang + fi + + new "kill old restconf daemon" + sudo pkill -u www-data -f "/www-data/clixon_restconf" + + new "start restconf daemon" + start_restconf -f $cfg -y $fyang + + new "waiting" + sleep $RCWAIT + + + to=$to0 + step=$step0 + reqs=$reqs0 + + + # Put all tests + for proto in netconf restconf; do + new "$proto put all entries to candidate (restconf:running)" + plot put $proto $step $step $to 0 0 0 # all candidate 0 running 0 done - # Check if still alive - pid=`pgrep clixon_backend` - if [ -z "$pid" ]; then - err "backend already dead" - fi - # kill backend - sudo clixon_backend -zf $cfg - if [ $? -ne 0 ]; then - err "kill backend" - fi -} + # Get all tests + for proto in netconf restconf; do + new "$proto get all entries from running" + plot get $proto $step $step $to 0 n n # start w full datastore + done -once + # Netconf commit all + new "Netconf commit all entries from candidate to running" + plot commit netconf $step $step $to 0 n 0 # candidate full running empty + + # Transactions get/put/delete + reqs=$reqs0 + for proto in netconf restconf; do + new "$proto get $reqs from full database" + plot get $proto $step $step $to $reqs n n + + new "$proto put $reqs to full database(replace / alter values)" + plot put $proto $step $step $to $reqs n n + + new "$proto delete $reqs from full database(replace / alter values)" + plot delete $proto $step $step $to $reqs n n + done + + new "Kill restconf daemon" + stop_restconf + + if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg + fi +fi # if run + +if $plot; then + +# 0. Startup +gplot="" +for a in $archs; do + gplot="$gplot \"$resdir/startup-$a\" title \"startup-$a\"," +done gnuplot -persist < $cfg + + $cfg + $dir + /usr/local/share/clixon + /usr/local/lib/$APPNAME/clispec + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/$APPNAME/$APPNAME.pidfile + $dir + true + +EOF + +cat < $fyang +module example { + yang-version 1.1; + namespace "urn:example:example"; + prefix ex; + revision 2019-01-13; + container c{ + list a{ + key x; + leaf x{ + type int32; + } + } + } +} +EOF + +testrun(){ + x0=$1 + xi="$2" + xp=c + + new "random list add leaf-list" + # First run sorted (assume this is the refernce == correct) + rs=$($clixon_util_insert -y $fyang -x "$xi" -b "$x0" -p $xp $OPTS -s) + # Then run actual insert + r0=$($clixon_util_insert -y $fyang -x "$xi" -b "$x0" -p $xp $OPTS) + # If both are null something is amiss + if [ -z "$r0" -a -z "$rs" ]; then + err "length of retval is zero" + fi + # echo "rs:$rs" +# echo "r0:$r0" + # Check they are equal + if [[ "$r0" != "$rs" ]]; then + err "$rs" "$r0" + fi +} + +new "test params: -y $fyang $OPTS" + +# Empty element base list +x0='' +new "empty list" +testrun "$x0" "1" + +# One element base list +x0='99' +new "one element list first" +testrun "$x0" "1" + +new "one element list last" +testrun "$x0" "100" + +# Two element base list +x0='299' +new "two element list first" +testrun "$x0" "1" + +new "two element list mid" +testrun "$x0" "12" + +new "two element list last" +testrun "$x0" "3000" + +# Three element base list +x0='299101' +new "three element list first" +testrun "$x0" "1" + +new "three element list second" +testrun "$x0" "10" + +new "three element list third" +testrun "$x0" "100" + +new "three element list last" +testrun "$x0" "1000" + +# Four element base list +x0='299101200' + +new "four element list first" +testrun "$x0" "1" + +new "four element list second" +testrun "$x0" "10" + +new "four element list third" +testrun "$x0" "100" + +new "four element list fourth" +testrun "$x0" "102" + +new "four element list last" +testrun "$x0" "1000" + +# Five element base list +x0='299101200300' + +new "five element list first" +testrun "$x0" "1" + +new "five element list mid" +testrun "$x0" "100" + +new "five element list last" +testrun "$x0" "1000" + +cat < $fyang +module example { + yang-version 1.1; + namespace "urn:example:example"; + prefix ex; + revision 2019-01-13; + container c{ + leaf a{ + type string; + } + container b{ + leaf a { + type string; + } + } + choice c1{ + case a{ + leaf x{ + type string; + } + } + case b{ + leaf y{ + type int32; + } + } + } + choice c2{ + leaf z{ + type string; + } + leaf t{ + type int32; + } + } + list d{ + key x; + leaf x{ + type int32; + } + ordered-by user; + } + leaf-list e{ + type int32; + } + } +} +EOF + +# Advanced list +# Empty base list +x0='' +xp=c +new "adv empty list add leaf" +testrun "$x0" "leaf" + +new "adv empty list add choice c1" +testrun "$x0" "choice1" + +xi='33' +new "adv empty list add leaf-list" +testrun "$x0" "33" + +# base list +x0='leafchoice133' + +new "adv list add leaf-list" +testrun "$x0" "32" + +new "adv list add leaf-list" +testrun "$x0" "32" + +rm -rf $dir diff --git a/test/test_perf.sh b/test/test_perf.sh index 596d7b70..c3df0aa3 100755 --- a/test/test_perf.sh +++ b/test/test_perf.sh @@ -5,7 +5,7 @@ s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi # Number of list/leaf-list entries in file -: ${perfnr:=1000} +: ${perfnr:=20000} # Number of requests made get/put : ${perfreq:=100} @@ -15,6 +15,7 @@ APPNAME=example cfg=$dir/scaling-conf.xml fyang=$dir/scaling.yang fconfig=$dir/large.xml +fconfig2=$dir/large2.xml cat < $fyang module scaling{ @@ -43,16 +44,49 @@ cat < $cfg $cfg $dir /usr/local/share/clixon - $IETFRFC scaling /usr/local/var/$APPNAME/$APPNAME.sock - /usr/local/var/$APPNAME/$APPNAME.pidfile + /usr/local/var/example/$APPNAME.pidfile false - /usr/local/var/$APPNAME + $dir false + example + /usr/local/lib/example/cli + /usr/local/lib/example/clispec + 1 + VARS + 0 EOF +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi +fi +# Try startup mode w startup +for mode in startup running; do + file=$dir/${mode}_db + sudo touch $file + sudo chmod 666 $file + new "generate large startup config ($file) with $perfnr list entries in mode $mode" + echo -n "" > $file + for (( i=0; i<$perfnr; i++ )); do + echo -n "$i$i" >> $file + done + echo "" >> $file + + new "Startup backend once -s $mode -f $cfg -y $fyang" + # Cannot use start_backend here due to expected error case + time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null +done + +new "Startup backend once -s $mode -f $cfg -y $fyang" +# Cannot use start_backend here due to expected error case +time sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang # 2> /dev/null + new "test params: -f $cfg -y $fyang" if [ $BE -ne 0 ]; then new "kill old backend" @@ -86,13 +120,9 @@ new "netconf write large config" expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" # Here, there are $perfnr entries in candidate - new "netconf write large config again" expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" -# Remove the file, its used for different purposes further down -rm $fconfig - # Now commit it from candidate to running new "netconf commit large config" expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" @@ -119,7 +149,7 @@ done | $clixon_netconf -qf $cfg -y $fyang > /dev/null new "restconf get $perfreq small config" time -p for (( i=0; i<$perfreq; i++ )); do rnd=$(( ( RANDOM % $perfnr ) )) - curl -sG http://localhost/restconf/data/scaling:x/y=$rnd,$rnd > /dev/null + curl -sG http://localhost/restconf/data/scaling:x/y=$rnd > /dev/null done new "restconf add $perfreq small config" @@ -135,19 +165,23 @@ expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" '^{"data": {"scaling:x": {"y": \[{"a": 0,"b": 0},{ "a": 1,"b": 1},{ "a": 2,"b": 2},{ "a": 3,"b": 3},' +new "restconf delete $perfreq small config" +time -p for (( i=0; i<$perfreq; i++ )); do + rnd=$(( ( RANDOM % $perfnr ) )) + curl -s -X DELETE http://localhost/restconf/data/scaling:x/y=$rnd +done + # Now do leaf-lists istead of leafs new "generate large leaf-list config" -echo -n "replace" > $fconfig +echo -n "replace" > $fconfig2 for (( i=0; i<$perfnr; i++ )); do - echo -n "$i" >> $fconfig + echo -n "$i" >> $fconfig2 done -echo "]]>]]>" >> $fconfig +echo "]]>]]>" >> $fconfig2 new "netconf replace large list-leaf config" -expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" - -rm $fconfig +expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig2" "^]]>]]>$" new "netconf commit large leaf-list config" expecteof "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" 0 "]]>]]>" "^]]>]]>$" @@ -183,4 +217,5 @@ fi # kill backend stop_backend -f $cfg + rm -rf $dir diff --git a/test/test_restconf.sh b/test/test_restconf.sh index 4b3ab3dc..1d71e78b 100755 --- a/test/test_restconf.sh +++ b/test/test_restconf.sh @@ -58,7 +58,7 @@ sleep $RCWAIT new "restconf tests" new "restconf root discovery. RFC 8040 3.1 (xml+xrd)" -expecteq "$(curl -s -X GET http://localhost/.well-known/host-meta)" 0 " +expecteq "$(curl -s -X GET http://localhost/.well-known/host-meta)" 0 " " diff --git a/test/test_restconf2.sh b/test/test_restconf2.sh index 543a9799..0ec36b11 100755 --- a/test/test_restconf2.sh +++ b/test/test_restconf2.sh @@ -95,7 +95,7 @@ expectfn 'curl -s -X POST -d {"example:cont1":{"interface":{"type":"regular"}}} new "restconf POST initial tree" expectfn 'curl -s -X POST -d {"example:cont1":{"interface":{"name":"local0","type":"regular"}}} http://localhost/restconf/data' 0 "" -new "restconf GET datastore intial" +new "restconf GET datastore initial" expectfn "curl -s -X GET http://localhost/restconf/data/example:cont1" 0 '{"example:cont1": {"interface": \[{"name": "local0","type": "regular"}\]}}' new "restconf GET interface subtree" diff --git a/test/test_startup.sh b/test/test_startup.sh index 9457e4ea..ac4be92f 100755 --- a/test/test_startup.sh +++ b/test/test_startup.sh @@ -6,6 +6,9 @@ # - An extra xml configuration file starts with an "extra" interface # - running db starts with a "run" interface # - startup db starts with a "start" interface +# There is also an "invalid" XML and a "broken" XML +# There are two steps, first run through everything OK +# Then try with invalid and borken XML and ensure the backend quits and all is untouched # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi @@ -36,27 +39,37 @@ cat < $cfg EOF -# Create running-db containin the interface "run" +# Create running-db containin the interface "run" OK runvar='runex:ethtrue' -# Create startup-db containing the interface "startup" +# Create startup-db containing the interface "startup" OK startvar='startupex:ethtrue' -# extra +# extra OK extravar='extraex:ethtrue' +# invalid (contains ), but OK XML syntax +invalidvar='invalidex:ethtrue' + +# Broken XML (contains ) +brokenvar='brokenex:ethtrue' + # Create a pre-set running, startup and (extra) config. # The configs are identified by an interface called run, startup, extra. # Depending on startup mode (init, none, running, or startup) # expect different output of an initial get-config of running testrun(){ mode=$1 - exprun=$2 # expected running_db after startup + rdb=$2 # running db at start + sdb=$3 # startup db at start + edb=$4 # extradb at start + exprun=$5 # expected running_db after startup + sudo rm -f $dir/*_db - echo "$runvar" > $dir/running_db - echo "$startvar" > $dir/startup_db - echo "$extravar" > $dir/extra_db + echo "$rdb" > $dir/running_db + echo "$sdb" > $dir/startup_db + echo "$edb" > $dir/extra_db if [ $BE -ne 0 ]; then # Bring your own backend # kill old backend (if any) @@ -81,7 +94,7 @@ testrun(){ expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' "^$exprun]]>]]>$" new "Startup test for $mode mode, check startup is untouched" - expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' "^$startvar]]>]]>$" + expecteof "$clixon_netconf -qf $cfg" 0 ']]>]]>' "^$sdb]]>]]>$" new "Kill backend" # Check if premature kill @@ -93,17 +106,79 @@ testrun(){ stop_backend -f $cfg } # testrun + +# The backend should fail with 255 and all db:s should be unaffected +testfail(){ + mode=$1 + rdb=$2 # running db at start + sdb=$3 # startup db at start + edb=$4 # extradb at start + + sudo rm -f $dir/*_db + + echo "$rdb" > $dir/running_db + echo "$sdb" > $dir/startup_db + echo "$edb" > $dir/extra_db + + # kill old backend (if any) + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -f $cfg -s $mode -c $dir/extra_db" + ret=$(start_backend -1 -s $mode -f $cfg -c $dir/extra_db 2> /dev/null) + r=$? + if [ $r -ne 255 ]; then + err "Unexpected retval" $r + fi + # permission kludges + sudo chmod 666 $dir/running_db + sudo chmod 666 $dir/startup_db + new "Checking running unchanged" + ret=$(diff $dir/running_db <(echo "$rdb")) + if [ $? -ne 0 ]; then + err "$rdb" "$ret" + fi + new "Checking startup unchanged" + ret=$(diff $dir/startup_db <(echo "$sdb")) + if [ $? -ne 0 ]; then + err "$sdb" "$ret" + fi + + new "Checking extra unchanged" + ret=$(diff $dir/extra_db <(echo "$edb")) + if [ $? -ne 0 ]; then + err "$edb" "$ret" + fi +} + +# 1. Try different modes on OK running/startup/extra # Init mode: delete running and reload from scratch (just extra) -testrun init "$extravar" +testrun init "$runvar" "$startvar" "$extravar" "$extravar" # None mode: do nothing, running remains -testrun none "$runvar" +testrun none "$runvar" "$startvar" "$extravar" "$runvar" # Running mode: keep running but load also extra -testrun running 'extraex:ethtruerunex:ethtrue' +testrun running "$runvar" "$startvar" "$extravar" 'extraex:ethtruerunex:ethtrue' # Startup mode: scratch running, load startup with extra on top -testrun startup 'extraex:ethtruestartupex:ethtrue' +testrun startup "$runvar" "$startvar" "$extravar" 'extraex:ethtruestartupex:ethtrue' -echo $dir -#rm -rf $dir +# 2. Try different modes on Invalid running/startup/extra WITHOUT failsafe +# ensure all db:s are unchanged after failure. + +new "Test invalid running in running mode" +testfail running "$invalidvar" "$startvar" "$extravar" + +new "Run invalid startup in startup mode" +testfail startup "$runvar" "$invalidvar" "$extravar" + +new "Test broken running in running mode" +testfail running "$brokenvar" "$startvar" "$extravar" + +new "Run broken startup in startup mode" +testfail startup "$runvar" "$brokenvar" "$extravar" + +rm -rf $dir diff --git a/util/Makefile.in b/util/Makefile.in index a4ef9f67..a7eacd4c 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -73,6 +73,7 @@ APPSRC += clixon_util_json.c APPSRC += clixon_util_yang.c APPSRC += clixon_util_xpath.c APPSRC += clixon_util_datastore.c +APPSRC += clixon_util_insert.c ifeq ($(with_restconf),yes) APPSRC += clixon_util_stream.c # Needs curl endif @@ -107,6 +108,9 @@ clixon_util_stream: clixon_util_stream.c $(LIBDEPS) clixon_util_datastore: clixon_util_datastore.c $(LIBDEPS) $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ +clixon_util_insert: clixon_util_insert.c $(LIBDEPS) + $(CC) $(INCLUDES) $(CPPFLAGS) @CFLAGS@ $(LDFLAGS) $^ $(LIBS) -o $@ + distclean: clean rm -f Makefile *~ .depend diff --git a/util/clixon_util_insert.c b/util/clixon_util_insert.c new file mode 100644 index 00000000..1edf35f7 --- /dev/null +++ b/util/clixon_util_insert.c @@ -0,0 +1,215 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + +See https://www.w3.org/TR/xpath/ + + * Turn this on to get an xpath test program + * Usage: xpath [] + * read xpath on first line and xml on rest of lines from input + * Example compile: + gcc -g -o xpath -I. -I../clixon ./clixon_xsl.c -lclixon -lcligen + * Example run: +echo "a\n" | xpath +*/ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* cligen */ +#include + +/* clixon */ +#include "clixon/clixon.h" + +static int +usage(char *argv0) +{ + fprintf(stderr, "usage:%s [options]\n" + "where options are\n" + "\t-h \t\tHelp\n" + "\t-D \tDebug\n" + "\t-y \tYANG spec file (or stdin)\n" + "\t-b \tXML base expression\n" + "\t-x \tXML to insert\n" + "\t-p \tXpath to where in base and XML\n" + "\t-s \tSort output after insert\n" + "Assume insert xml is first child of xpath. Ie if xml=23 and xpath=a, then inserted element is 23\n", + argv0 + ); + exit(0); +} + +int +main(int argc, char **argv) +{ + int retval = -1; + char *argv0 = argv[0]; + int c; + char *filename = NULL; + int fd = 0; /* unless overriden by argv[1] */ + char *x0str = NULL; + char *xistr = NULL; + char *xpath = NULL; + yang_stmt *yspec; + cxobj *x0 = NULL; + cxobj *xb; + cxobj *xi = NULL; + int sort = 0; + clicon_handle h; + + clicon_log_init("clixon_insert", LOG_DEBUG, CLICON_LOG_STDERR); + if ((h = clicon_handle_init()) == NULL) + goto done; + optind = 1; + opterr = 0; + while ((c = getopt(argc, argv, "hD:y:b:x:p:s")) != -1) + switch (c) { + case 'h': + usage(argv0); + break; + case 'D': + if (sscanf(optarg, "%d", &debug) != 1) + usage(argv0); + break; + case 'y': /* YANG spec file */ + filename = optarg; + if (0 && (fd = open(filename, O_RDONLY)) < 0){ + clicon_err(OE_UNIX, errno, "open(%s)", argv[1]); + goto done; + } + break; + case 'b': /* Base XML expression */ + x0str = optarg; + break; + case 'x': /* XML to insert */ + xistr = optarg; + break; + case 'p': /* XPATH base */ + xpath = optarg; + break; + case 's': /* sort output after insert */ + sort++; + break; + default: + usage(argv[0]); + break; + } + if (xistr == NULL || x0str == NULL) + usage(argv0); + if (xpath == NULL) + usage(argv0); + if (filename == NULL) + usage(argv0); + clicon_debug(1, "xistr:%s", xistr); + clicon_debug(1, "x0str:%s", x0str); + clicon_debug(1, "xpath:%s", xpath); + if ((yspec = yspec_new()) == NULL) + goto done; +#if 1 + if (yang_spec_parse_file(h, filename, yspec) < 0) + goto done; +#else + if (yang_parse_file(fd, "yang test", yspec) == NULL) + goto done; +#endif + /* Parse base XML */ + if (xml_parse_string(x0str, yspec, &x0) < 0){ + clicon_err(OE_XML, 0, "Parsing base xml: %s", x0str); + goto done; + } + if (xml_apply(x0, CX_ELMNT, xml_spec_populate, yspec) < 0) + goto done; + if ((xb = xpath_first(x0, "%s", xpath)) == NULL){ + clicon_err(OE_XML, 0, "xpath: %s not found in x0", xpath); + goto done; + } + if (debug){ + clicon_debug(1, "xb:"); + xml_print(stderr, xb); + } + /* Parse insert XML */ + if (xml_parse_string(xistr, yspec, &xi) < 0){ + clicon_err(OE_XML, 0, "Parsing insert xml: %s", xistr); + goto done; + } + if (xml_apply(xi, CX_ELMNT, xml_spec_populate, yspec) < 0) + goto done; + if ((xi = xpath_first(xi, "%s", xpath)) == NULL){ + clicon_err(OE_XML, 0, "xpath: %s not found in xi", xpath); + goto done; + } + /* Find first element child */ + if ((xi = xml_child_i_type(xi, 0, CX_ELMNT)) == NULL){ + clicon_err(OE_XML, 0, "xi has no element child"); + goto done; + } + /* Remove it from parent */ + if (xml_rm(xi) < 0) + goto done; + if (debug){ + clicon_debug(1, "xi:"); + xml_print(stderr, xi); + } + if (xml_insert(xb, xi) < 0) + goto done; + if (debug){ + clicon_debug(1, "x0:"); + xml_print(stderr, x0); + } + if (sort) + xml_sort(xb, NULL); + clicon_xml2file(stdout, xb, 0, 0); + + retval = 0; + done: + if (x0) + xml_free(x0); + if (yspec) + yspec_free(yspec); + if (fd > 0) + close(fd); + return retval; +} diff --git a/util/clixon_util_xpath.c b/util/clixon_util_xpath.c index b711b072..198cdb8a 100644 --- a/util/clixon_util_xpath.c +++ b/util/clixon_util_xpath.c @@ -33,13 +33,7 @@ See https://www.w3.org/TR/xpath/ - * Turn this on to get an xpath test program - * Usage: xpath [] - * read xpath on first line and xml on rest of lines from input - * Example compile: - gcc -g -o xpath -I. -I../clixon ./clixon_xsl.c -lclixon -lcligen - * Example run: -echo "a\n" | xpath +* */ #ifdef HAVE_CONFIG_H