diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ff47bdb..1450f6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,13 @@ ## 6.3.0 Expected: July 2023 +### New features +* Output pipes + * Building on a new CLIgen feature + * See https://clixon-docs.readthedocs.io/en/latest/cli.html#output-pipes + ### API changes on existing protocol/config features +Users may have to change how they access the system * New `clixon-config@2023-05-01.yang` revision * Added options: `CLICON_CONFIG_EXTEND` @@ -89,7 +95,6 @@ schema mount and other features required by the clixon controller project, along with minor improvements and bugfixes. ### API changes on existing protocol/config features - Users may have to change how they access the system * New `clixon-config@2023-03-01.yang` revision diff --git a/apps/cli/Makefile.in b/apps/cli/Makefile.in index 9d073507..01f9825a 100644 --- a/apps/cli/Makefile.in +++ b/apps/cli/Makefile.in @@ -107,6 +107,7 @@ LIBSRC += cli_plugin.c LIBSRC += cli_auto.c LIBSRC += cli_generate.c LIBSRC += cli_autocli.c +LIBSRC += cli_pipe.c LIBOBJ = $(LIBSRC:.c=.o) # Name of lib diff --git a/apps/cli/cli_main.c b/apps/cli/cli_main.c index 9b73ce53..b4293ae6 100644 --- a/apps/cli/cli_main.c +++ b/apps/cli/cli_main.c @@ -308,7 +308,6 @@ autocli_trees_default(clicon_handle h) goto done; if (cligen_ph_parsetree_set(ph, pt) < 0) goto done; - /* Create backward compatible tree: @datamodelstate */ if ((ph = cligen_ph_add(cli_cligen(h), "datamodelstate")) == NULL) goto done; @@ -403,7 +402,7 @@ autocli_start(clicon_handle h) if (yang2cli_init(h) < 0) goto done; yspec = clicon_dbspec_yang(h); - /* The actual generating call from yang to clispec for the complete yang spec */ + /* The actual generating call from yang to clispec for the complete yang spec, @basemodel */ if (yang2cli_yspec(h, yspec, AUTOCLI_TREENAME) < 0) goto done; /* XXX Create pre-5.5 tree-refs for backward compatibility */ diff --git a/apps/cli/cli_pipe.c b/apps/cli/cli_pipe.c new file mode 100644 index 00000000..e4269e6f --- /dev/null +++ b/apps/cli/cli_pipe.c @@ -0,0 +1,123 @@ +/* + * + ***** BEGIN LICENSE BLOCK ***** + + Copyright (C) 2023 Olof Hagsand + + This file is part of CLIXON. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Alternatively, the contents of this file may be used under the terms of + the GNU General Public License Version 3 or later (the "GPL"), + in which case the provisions of the GPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of the GPL, and not to allow others to + use your version of this file under the terms of Apache License version 2, + indicate your decision by deleting the provisions above and replace them with + the notice and other provisions required by the GPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the Apache License version 2 or the GPL. + + ***** END LICENSE BLOCK ***** + * + */ + +#ifdef HAVE_CONFIG_H +#include "clixon_config.h" /* generated by config & autoconf */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* cligen */ +#include + +/* clicon */ +#include + +#include "clixon_cli_api.h" + +/* Grep pipe output function + */ +int +grep_fn(cligen_handle h, + cvec *cvv, + cvec *argv) +{ + int retval = -1; + cbuf *cb = NULL; + cg_var *av; + cg_var *cv; + char *name; + + if ((cb = cbuf_new()) == NULL){ + perror("cbuf_new"); + goto done; + } + if (argv && (av = cvec_i(argv, 0)) != NULL){ + /* First arg is command */ + // cprintf(cb, "%s", cv_string_get(av)); + /* Rest are names of parameters from cvv */ + av = NULL; + while ((av = cvec_each1(argv, av)) != NULL){ + name = cv_string_get(av); + if ((cv = cvec_find_var(cvv, name)) != NULL) + cprintf(cb, "%s", cv_string_get(cv)); + } + retval = execl("/usr/bin/grep", "grep", "-e", cbuf_get(cb), (char *) NULL); + } + done: + if (cb) + cbuf_free(cb); + return retval; +} + +/*! Test cli callback calls cligen_output with output lines as given by function arguments + * + * This is to generate output to eg cligen_output scrolling + * Example: + * a, printlines_fn("line1 abc", "line2 def"); + */ +int +output_fn(cligen_handle handle, + cvec *cvv, + cvec *argv) +{ + cg_var *cv; + + cv = NULL; + while ((cv = cvec_each(argv, cv)) != NULL){ + cligen_output(stdout, "%s\n", cv_string_get(cv)); + } + return 0; +} diff --git a/apps/cli/cli_plugin.c b/apps/cli/cli_plugin.c index c762361e..a9afd4b1 100644 --- a/apps/cli/cli_plugin.c +++ b/apps/cli/cli_plugin.c @@ -32,7 +32,7 @@ the terms of any one of the Apache License version 2 or the GPL. ***** END LICENSE BLOCK ***** - + * */ #ifdef HAVE_CONFIG_H @@ -88,6 +88,7 @@ static int gen_parse_tree(clicon_handle h, char *name, parse_tree *pt, + char *pipetree, pt_head **php) { int retval = -1; @@ -101,6 +102,9 @@ gen_parse_tree(clicon_handle h, clicon_err(OE_UNIX, errno, "cligen_ph_prompt_set"); goto done; } + if (pipetree && + cligen_ph_pipe_set(ph, pipetree) < 0) + goto done; *php = ph; retval = 0; done: @@ -163,6 +167,25 @@ clixon_str2fn(char *name, return NULL; } +/*! Set output pipe flag in all callbacks + * + * @param[in] co CLIgen parse-tree object + * @param[in] arg Argument, cast to application-specific info + * @retval 1 OK and return (abort iteration) + * @retval 0 OK and continue + * @retval -1 Error: break and return + */ +static int +cli_mark_output_pipes(cg_obj *co, + void *arg) +{ + cg_callback *cc; + + for (cc = co->co_callbacks; cc; cc = co_callback_next(cc)) + cc->cc_flags |= CC_FLAGS_PIPE_FUNCTION; + return 0; +} + /*! Load a file containing clispec syntax and append to specified modes, also load C plugin * * First load CLIgen file, @@ -195,6 +218,7 @@ clispec_load_file(clicon_handle h, int i; int nvec; char *plgnam; + char *pipetree; pt_head *ph; #ifndef CLIXON_STATIC_PLUGINS clixon_plugin_t *cp; @@ -238,6 +262,7 @@ clispec_load_file(clicon_handle h, mode = cvec_find_str(cvv, "CLICON_MODE"); prompt = cvec_find_str(cvv, "CLICON_PROMPT"); plgnam = cvec_find_str(cvv, "CLICON_PLUGIN"); + pipetree = cvec_find_str(cvv, "CLICON_PIPETREE"); #ifndef CLIXON_STATIC_PLUGINS if (plgnam != NULL) { /* Find plugin for callback resolving */ @@ -252,6 +277,13 @@ clispec_load_file(clicon_handle h, } #endif + /* Algorithm to find output pipe trees: first mode is on the form | + */ + if (mode && strlen(mode) && IS_PIPE_TREE(mode)){ + if (pt_apply(pt, cli_mark_output_pipes, -1, NULL) < 0) + goto done; + } + /* Resolve callback names to function pointers. */ if (cligen_callbackv_str2fn(pt, (cgv_str2fn_t*)clixon_str2fn, handle) < 0){ clicon_err(OE_PLUGIN, 0, "Mismatch between CLIgen file '%s' and CLI plugin file '%s'. Some possible errors:\n\t1. A function given in the CLIgen file does not exist in the plugin (ie link error)\n\t2. The CLIgen spec does not point to the correct plugin .so file (CLICON_PLUGIN=\"%s\" is wrong)", @@ -296,7 +328,7 @@ clispec_load_file(clicon_handle h, clicon_err(OE_UNIX, errno, "pt_new"); goto done; } - if (gen_parse_tree(h, name, ptnew, &ph) < 0) + if (gen_parse_tree(h, name, ptnew, pipetree, &ph) < 0) goto done; if (ph == NULL) goto done; diff --git a/doc/CLI.md b/doc/CLI.md index 282727d3..09b7906a 100644 --- a/doc/CLI.md +++ b/doc/CLI.md @@ -3,7 +3,7 @@ * [CLIgen](#cligen) * [Command history](#history) * [Large spec designs](#large-specs) - +* [Output pipes](#output-pipes) ## CLIgen @@ -93,3 +93,159 @@ You can also add the C preprocessor as a first step. You can then define macros, $(CPP) -P -x assembler-with-cpp $(INCLUDES) -o $@ $< ``` +## Output pipes + +This section describes implementation aspects of Clixon output pipes. + +Output pipes resemble UNIX shell pipes and are useful to filter or modify CLI output. Example: + ``` + cli> show config | grep parameter + 5 + x + cli> + ``` + +Clixon and CLIgen implements a limited variant of output pipes using a set of mechanisms, as follows: + +* Pipe trees, name starts with vertical bar +* Pipe functions, marked with flag +* Explicit pipe reference, use tree reference mechanism with appended function +* Default pipe reference, dynamic expansion of parse-tree +* Callback evaluation + +Note that `cligen_output` must be used for all output to use output pipes. This is already true for scrolling. + +Further, multiple pipe functions are not (yet) supported, such as: `fn | tail | count`. There are no fundamental obstacles to implement them. + +### Pipe trees + +Clixon uses the CLIgen `tree` mechanism to specify a set of output +pipes. A pipe tree is similar to other trees, but is distinguished +using a vertical bar as the first character in its name. + +For example, the name of a pipe tree could be `|mypipe` and a reference to such a pipe would be: `@|mypipe`. + +A pipetree is declared in Clixon assigning the `CLICON_MODE` variable. The tree itself usually starts with the escaped vertical bar character (but could be something else), followed by a set of pipe functions. Example: +``` + CLICON_MODE="|mypipe"; + \| { + grep , grep_fn("grep -e", "arg"); + tail, tail_fn(); + } +``` + +The CLIgen method uses the treename assignment instead, but is otherwise similar: +``` + treename="|mypipe"; +``` + +Callbacks referenced in a pipe tree, such as `grep_fn`, are marked +with a `CC_FLAGS_PIPE_FUNCTION` flag, to distingusih them from regular +callbacks. All callbacks in a pipe tree are marked as pipe +functions. This is done when parsing (or just after). Clixon and +CLIgen does it slightly different: +* CLIgen: If the "active" tree is a pipe-tree (starts with '|') then a parsed callback is a pipe function +* Clixon: After parsing, if the 'mode' is a pipe-tree, the traverse all callbacks and mark them + +### Pipe functions + +The pipe callback functions example, are called with the same arguments as regular CLIgen callbacks +``` + int tail_fn(cligen_handle h, cvec *cvv, cvec *argv) +``` +where `cvv` is the command line and `argv` are the arguments in the call. + +However, the difference is that a pipe function is expected to receive +input on stdin and produce output to stdout. + +This can be done by making an `exec` call to a UNIX command, as follows: +``` + execl("/usr/bin/tail", (char *) NULL); +``` + +Another way to write a pipe function is just by using stdin and stdout directly, without an exec. + +The clixon system itself arranges the input and output to be +redirected properly, if the pipe function is a part of a pipe tree as described below. + +### Explicit pipe references + +A straightforward way to reference a pipe tree is by using an explicit pipe-tree reference as follows: +``` + set { + @|mypipe, regular_cb(); + } +``` +Note that the `regular_cb()` is stated as an argument to the mypipe reference. This means it will be preended to each callback in 'mypipe'. + +For example, in the following CLI call: +``` + cli> set | tail +``` +Two callbacks will be evaluated as follows: +``` + regular_cb() | tail_fn() +``` +where the stdout of `regular_cb()` is redirected to the stdin of `tail_fn()`. + +If a call without pipe is wanted, the CLI explicit reference can be extended as follows: +``` + set, regular_cb(); { + @|mypipe, regular_cb(); + } +``` + +### Default pipe references + +Instead of explicitly stating a pipe tree after each command, it is +possible to make a default pipe-tree rule, which uses dynamic expansion to add pipe-trees automatically. + +In Clixon: +``` + CLICON_PIPETREE="|mypipe"; # in Clixon + pipetree="|mypipe"; # in CLIgen +``` + +This autoamtically expands each terminal command into a pipe-tree reference in a dynamic way. For example, assume the command `set, regular_cb();` is specified and a user types `set `. + +This expands the syntax by adding the `@|mypipe, regular_cb()` which is in turn expanded to: +``` + set, regular_cb(); { + \| { + grep , grep_fn("grep -e", "arg"); + tail, tail_fn(); + } + } +``` + +### Callback evaluation + +When a callback is evaluated, a pipe function, if present, is run in a forked sub-process and a +'pipe-socket' is registered as input to that process. + +Then, the regular callback is called. When the callback calls `cligen_output()`, the output is redirected to the pipe-socket (sock) to the pipe-function, then back to `cligen_output_basic()`, as shown in the following sketch: +``` + regular_cb() --> cligen_output() --> (sock) --> tail_fn() --> (sock) --> cligen_output_basic() +``` + +The `cligen_output_basic()` makes no redirection to avoid recursion. + +### Discussion + +The explicit syntax may seem counterintuitive, since the pipe-tree reference is made *before* the original call: `@|mypipe, regular_cb()`. + +It would be nicer to place the pipe-tree reference *after* the original call, something like: +``` +regular_cb() |mypipe; +``` + +Further, the default pipe references rely on a statement (`pipetree=` +or `CLICON_PIPETREE=`) in the *top-level* tree. This means that any +different setting in a referenced tree is not significant. + +It can be discussed whether this is good or bad. The advantage is that +it is easy to make a default statement in a single place. It could be +a lot of work to find out which sub-trees are referenced and change +them all. + + diff --git a/example/main/Makefile.in b/example/main/Makefile.in index 0eb9c72d..995a397a 100644 --- a/example/main/Makefile.in +++ b/example/main/Makefile.in @@ -97,7 +97,7 @@ all: $(PLUGINS) .c.o: $(CC) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -c $< -CLISPECS = $(APPNAME)_cli.cli +CLISPECS = $(APPNAME)_cli.cli YANGSPECS = clixon-example@2022-11-01.yang diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index 6326abb8..da6cc480 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -415,7 +415,7 @@ clixon_plugins_load(clicon_handle h, int ret; plugin_module_struct *ms = plugin_module_struct_get(h); - clicon_debug(1, "%s", __FUNCTION__); + clicon_debug(CLIXON_DBG_DETAIL, "%s", __FUNCTION__); if (ms == NULL){ clicon_err(OE_PLUGIN, EINVAL, "plugin module not initialized"); goto done; @@ -426,8 +426,7 @@ clixon_plugins_load(clicon_handle h, /* Load all plugins */ for (i = 0; i < ndp; i++) { snprintf(filename, MAXPATHLEN-1, "%s/%s", dir, dp[i].d_name); - clicon_debug(1, "DEBUG: Loading plugin '%.*s' ...", - (int)strlen(filename), filename); + clicon_debug(CLIXON_DBG_DEFAULT, "Loading plugin '%s'", filename); if ((ret = plugin_load_one(h, filename, function, RTLD_NOW, &cp)) < 0) goto done; if (ret == 0) diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index d9b3d9ed..38ccf7b8 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -2212,7 +2212,7 @@ ys_populate_leaf(clicon_handle h, cv_dec64_n_set(cv, fraction_digits); if (cv_name_set(cv, ys->ys_argument) == NULL){ - clicon_err(OE_YANG, errno, "cv_new_set"); + clicon_err(OE_YANG, errno, "cv_name_set"); goto done; } /* get parent of where type is defined, can be original object */ diff --git a/test/test_cli_pipe.sh b/test/test_cli_pipe.sh new file mode 100755 index 00000000..c30394cc --- /dev/null +++ b/test/test_cli_pipe.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# CLIgen output pipe functions +# Note, | must be escaped as \| otherwise shell's pipe is used (w possibly same result) +# XXX Autocli does not work + +# Magic line must be first in script (see README.md) +s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi + +APPNAME=example + +# include err() and new() functions and creates $dir + +cfg=$dir/conf_yang.xml +fyang=$dir/clixon-example.yang +clidir=$dir/clidir + +if [ ! -d $clidir ]; then + mkdir $clidir +fi + +cat < $cfg + + $cfg + ${YANG_INSTALLDIR} + $fyang + /usr/local/lib/$APPNAME/backend + $clidir + /usr/local/lib/$APPNAME/cli + $APPNAME + /usr/local/var/$APPNAME/$APPNAME.sock + /usr/local/var/$APPNAME/$APPNAME.pidfile + /usr/local/var/$APPNAME + +EOF + +cat < $fyang +module clixon-example{ + yang-version 1.1; + namespace "urn:example:clixon"; + prefix ex; + /* Generic config data */ + container table{ + list parameter{ + key name; + leaf name{ + type string; + } + leaf value{ + type string; + } + } + } +} +EOF + +cat < $clidir/example.cli +CLICON_MODE="example"; +CLICON_PROMPT="%U@%H %W> "; + +# Autocli syntax tree operations +edit @datamodel, cli_auto_edit("datamodel"); +up, cli_auto_up("datamodel"); +top, cli_auto_top("datamodel"); +set @datamodel, cli_auto_set(); +delete("Delete a configuration item") { + @datamodel, cli_auto_del(); + all("Delete whole candidate configuration"), delete_all("candidate"); +} +commit("Commit the changes"), cli_commit(); + +EOF + +cat < $clidir/nodefault.cli +CLICON_MODE="nodefault"; +CLICON_PROMPT="%U@%H %W> "; + +show("Show a particular state of the system"){ + implicit("No pipe function") { + configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false); + } + explicit("Explicit pipe function") { + configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{ + @|mypipe, cli_show_auto_mode("candidate", "xml", true, false); + } + } + autocli("Generated tree") @datamodelshow, cli_show_auto("candidate", "xml", true, false, "report-all"); + treeref @treeref; +} +EOF + +cat < $clidir/default.cli +CLICON_MODE="default"; +CLICON_PROMPT="%U@%H %W> "; +CLICON_PIPETREE="|mypipe"; # Only difference from nodefault + +show("Show a particular state of the system"){ + implicit("No pipe function") { + configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false); + } + explicit("Explicit pipe function") { + configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{ + @|mypipe, cli_show_auto_mode("candidate", "xml", true, false); + } + } + autocli("Generated tree") @datamodelshow, cli_show_auto("candidate", "xml", true, false, "report-all"); + treeref @treeref; +} +EOF + +cat < $clidir/treeref.cli +CLICON_MODE="treeref"; +CLICON_PIPETREE="|mypipe"; + +implicit("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false); +explicit("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{ + @|mypipe, cli_show_auto_mode("candidate", "xml", true, false); +} + +EOF + +cat < $clidir/clipipe.cli +CLICON_MODE="|mypipe"; # Must start with | +#CLICON_PIPETREE="|mypipe"; +\| { + grep , grep_fn("grep -e", "arg"); +} +EOF + +new "test params: -f $cfg" +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -z -f $cfg + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg" + start_backend -s init -f $cfg +fi + +new "wait backend" +wait_backend + +new "Add entry x" +expectpart "$($clixon_cli -1 -f $cfg set table parameter x value a)" 0 "^$" + +new "Add entry y" +expectpart "$($clixon_cli -1 -f $cfg set table parameter y value b)" 0 "^$" + +new "Commit" +expectpart "$($clixon_cli -1 -f $cfg commit)" 0 "^$" + +new "Pipes with no default rule" +mode=nodefault + +new "$mode show implicit" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config)" 0 "" "" "table" "value" + +new "$mode show explicit" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config)" 0 "" "" "table" "value" + +new "$mode show implicit | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config \| grep par 2>&1)" 255 "Unknown command" + +new "$mode show explicit | grep par , expect fail" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config \| grep par)" 0 "" "" --not-- "table" "value" + +new "$mode show treeref explicit | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref explicit \| grep par)" 0 "" "" --not-- "table" "value" + +# No-default top-level rule also applies to sub-tree, feature or bug? +new "$mode show treeref implicit | grep par, expect error" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref implicit \| grep par 2>&1)" 255 "Unknown command" + +# No-default top-level rule also applies to sub-tree, feature or bug? +new "$mode show autocli table | grep par, expect error" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table \| grep par 2>&1)" 255 "Unknown command" + +new "Pipes with default rule" +mode=default + +new "$mode show implicit" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config)" 0 "" "" "table" "value" + +new "$mode show explicit" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config)" 0 "" "" "table" "value" + +new "$mode show implicit | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config \| grep par 2>&1)" 0 "" "" --not-- "table" "value" + +new "$mode show explicit | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config \| grep par)" 0 "" "" --not-- "table" "value" + +new "$mode show treeref explicit | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref explicit \| grep par)" 0 "" "" --not-- "table" "value" + +new "$mode show treeref implicit | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref implicit \| grep par)" 0 "" "" --not-- "table" "value" + +new "$mode show autocli table | grep par" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table \| grep par 2>&1)" 0 "" "" --not-- "table" "value" + +new "$mode show autocli table parameter x value | grep value" +expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table parameter x value \| grep value)" 0 "a" --not-- "table" "parameter" + +if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=$(pgrep -u root -f clixon_backend) + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg +fi + +rm -rf $dir + +unset mode + +new "endtest" +endtest