experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Output pipe functionality

Browse source 
Based on output pipe code in CLIgen
Clixon adaptions include `CLICON_PIPETREE=` variable and a new cli_pipe.c callback file
This commit is contained in:
Olof hagsand 2023-06-28 14:40:10 +02:00
parent e498e09570
commit b33603107d
10 changed files with 547 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

7
CHANGELOG.md
View file

@ -43,7 +43,13 @@
## 6.3.0
Expected: July 2023
### New features
* Output pipes
* Building on a new CLIgen feature
* See https://clixon-docs.readthedocs.io/en/latest/cli.html#output-pipes
### API changes on existing protocol/config features
Users may have to change how they access the system
* New `clixon-config@2023-05-01.yang` revision
* Added options: `CLICON_CONFIG_EXTEND`
@ -89,7 +95,6 @@ schema mount and other features required by the clixon controller
project, along with minor improvements and bugfixes.
### API changes on existing protocol/config features
Users may have to change how they access the system
* New `clixon-config@2023-03-01.yang` revision

1
apps/cli/Makefile.in
View file

@ -107,6 +107,7 @@ LIBSRC += cli_plugin.c
LIBSRC += cli_auto.c
LIBSRC += cli_generate.c
LIBSRC += cli_autocli.c
LIBSRC += cli_pipe.c
LIBOBJ = $(LIBSRC:.c=.o)
# Name of lib

3
apps/cli/cli_main.c
View file

@ -308,7 +308,6 @@ autocli_trees_default(clicon_handle h)
goto done;
if (cligen_ph_parsetree_set(ph, pt) < 0)
goto done;
/* Create backward compatible tree: @datamodelstate */
if ((ph = cligen_ph_add(cli_cligen(h), "datamodelstate")) == NULL)
goto done;
@ -403,7 +402,7 @@ autocli_start(clicon_handle h)
if (yang2cli_init(h) < 0)
goto done;
yspec = clicon_dbspec_yang(h);
/* The actual generating call from yang to clispec for the complete yang spec */
/* The actual generating call from yang to clispec for the complete yang spec, @basemodel */
if (yang2cli_yspec(h, yspec, AUTOCLI_TREENAME) < 0)
goto done;
/* XXX Create pre-5.5 tree-refs for backward compatibility */

123
apps/cli/cli_pipe.c Normal file
View file

@ -0,0 +1,123 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2023 Olof Hagsand
This file is part of CLIXON.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Alternatively, the contents of this file may be used under the terms of
the GNU General Public License Version 3 or later (the "GPL"),
in which case the provisions of the GPL are applicable instead
of those above. If you wish to allow use of your version of this file only
under the terms of the GPL, and not to allow others to
use your version of this file under the terms of Apache License version 2,
indicate your decision by deleting the provisions above and replace them with
the notice and other provisions required by the GPL. If you do not delete
the provisions above, a recipient may use your version of this file under
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*
*/
#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <signal.h>
#include <errno.h>
#include <stdarg.h>
#include <time.h>
#include <ctype.h>
#include <unistd.h>
#include <dirent.h>
#include <syslog.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/wait.h>
#include <sys/param.h>
#include <sys/mount.h>
#include <pwd.h>
/* cligen */
#include <cligen/cligen.h>
/* clicon */
#include <clixon/clixon.h>
#include "clixon_cli_api.h"
/* Grep pipe output function
*/
int
grep_fn(cligen_handle h,
cvec *cvv,
cvec *argv)
{
int retval = -1;
cbuf *cb = NULL;
cg_var *av;
cg_var *cv;
char *name;
if ((cb = cbuf_new()) == NULL){
perror("cbuf_new");
goto done;
}
if (argv && (av = cvec_i(argv, 0)) != NULL){
/* First arg is command */
// cprintf(cb, "%s", cv_string_get(av));
/* Rest are names of parameters from cvv */
av = NULL;
while ((av = cvec_each1(argv, av)) != NULL){
name = cv_string_get(av);
if ((cv = cvec_find_var(cvv, name)) != NULL)
cprintf(cb, "%s", cv_string_get(cv));
}
retval = execl("/usr/bin/grep", "grep", "-e", cbuf_get(cb), (char *) NULL);
}
done:
if (cb)
cbuf_free(cb);
return retval;
}
/*! Test cli callback calls cligen_output with output lines as given by function arguments
*
* This is to generate output to eg cligen_output scrolling
* Example:
* a, printlines_fn("line1 abc", "line2 def");
*/
int
output_fn(cligen_handle handle,
cvec *cvv,
cvec *argv)
{
cg_var *cv;
cv = NULL;
while ((cv = cvec_each(argv, cv)) != NULL){
cligen_output(stdout, "%s\n", cv_string_get(cv));
}
return 0;
}

36
apps/cli/cli_plugin.c
View file

@ -32,7 +32,7 @@
the terms of any one of the Apache License version 2 or the GPL.
***** END LICENSE BLOCK *****
*
*/
#ifdef HAVE_CONFIG_H
@ -88,6 +88,7 @@ static int
gen_parse_tree(clicon_handle h,
char *name,
parse_tree *pt,
char *pipetree,
pt_head **php)
{
int retval = -1;
@ -101,6 +102,9 @@ gen_parse_tree(clicon_handle h,
clicon_err(OE_UNIX, errno, "cligen_ph_prompt_set");
goto done;
}
if (pipetree &&
cligen_ph_pipe_set(ph, pipetree) < 0)
goto done;
*php = ph;
retval = 0;
done:
@ -163,6 +167,25 @@ clixon_str2fn(char *name,
return NULL;
}
/*! Set output pipe flag in all callbacks
*
* @param[in] co CLIgen parse-tree object
* @param[in] arg Argument, cast to application-specific info
* @retval 1 OK and return (abort iteration)
* @retval 0 OK and continue
* @retval -1 Error: break and return
*/
static int
cli_mark_output_pipes(cg_obj *co,
void *arg)
{
cg_callback *cc;
for (cc = co->co_callbacks; cc; cc = co_callback_next(cc))
cc->cc_flags |= CC_FLAGS_PIPE_FUNCTION;
return 0;
}
/*! Load a file containing clispec syntax and append to specified modes, also load C plugin
*
* First load CLIgen file,
@ -195,6 +218,7 @@ clispec_load_file(clicon_handle h,
int i;
int nvec;
char *plgnam;
char *pipetree;
pt_head *ph;
#ifndef CLIXON_STATIC_PLUGINS
clixon_plugin_t *cp;
@ -238,6 +262,7 @@ clispec_load_file(clicon_handle h,
mode = cvec_find_str(cvv, "CLICON_MODE");
prompt = cvec_find_str(cvv, "CLICON_PROMPT");
plgnam = cvec_find_str(cvv, "CLICON_PLUGIN");
pipetree = cvec_find_str(cvv, "CLICON_PIPETREE");
#ifndef CLIXON_STATIC_PLUGINS
if (plgnam != NULL) { /* Find plugin for callback resolving */
@ -252,6 +277,13 @@ clispec_load_file(clicon_handle h,
}
#endif
/* Algorithm to find output pipe trees: first mode is on the form |<treename>
*/
if (mode && strlen(mode) && IS_PIPE_TREE(mode)){
if (pt_apply(pt, cli_mark_output_pipes, -1, NULL) < 0)
goto done;
}
/* Resolve callback names to function pointers. */
if (cligen_callbackv_str2fn(pt, (cgv_str2fn_t*)clixon_str2fn, handle) < 0){
clicon_err(OE_PLUGIN, 0, "Mismatch between CLIgen file '%s' and CLI plugin file '%s'. Some possible errors:\n\t1. A function given in the CLIgen file does not exist in the plugin (ie link error)\n\t2. The CLIgen spec does not point to the correct plugin .so file (CLICON_PLUGIN=\"%s\" is wrong)",
@ -296,7 +328,7 @@ clispec_load_file(clicon_handle h,
clicon_err(OE_UNIX, errno, "pt_new");
goto done;
}
if (gen_parse_tree(h, name, ptnew, &ph) < 0)
if (gen_parse_tree(h, name, ptnew, pipetree, &ph) < 0)
goto done;
if (ph == NULL)
goto done;

158
doc/CLI.md
View file

@ -3,7 +3,7 @@
* [CLIgen](#cligen)
* [Command history](#history)
* [Large spec designs](#large-specs)
* [Output pipes](#output-pipes)
## CLIgen
@ -93,3 +93,159 @@ You can also add the C preprocessor as a first step. You can then define macros,
$(CPP) -P -x assembler-with-cpp $(INCLUDES) -o $@ $<
```
## Output pipes
This section describes implementation aspects of Clixon output pipes.
Output pipes resemble UNIX shell pipes and are useful to filter or modify CLI output. Example:
```
cli> show config | grep parameter
<parameter>5</parameter>
<parameter>x</parameter>
cli>
```
Clixon and CLIgen implements a limited variant of output pipes using a set of mechanisms, as follows:
* Pipe trees, name starts with vertical bar
* Pipe functions, marked with flag
* Explicit pipe reference, use tree reference mechanism with appended function
* Default pipe reference, dynamic expansion of parse-tree
* Callback evaluation
Note that `cligen_output` must be used for all output to use output pipes. This is already true for scrolling.
Further, multiple pipe functions are not (yet) supported, such as: `fn | tail | count`. There are no fundamental obstacles to implement them.
### Pipe trees
Clixon uses the CLIgen `tree` mechanism to specify a set of output
pipes. A pipe tree is similar to other trees, but is distinguished
using a vertical bar as the first character in its name.
For example, the name of a pipe tree could be `|mypipe` and a reference to such a pipe would be: `@|mypipe`.
A pipetree is declared in Clixon assigning the `CLICON_MODE` variable. The tree itself usually starts with the escaped vertical bar character (but could be something else), followed by a set of pipe functions. Example:
```
CLICON_MODE="|mypipe";
\| {
grep <arg:rest>, grep_fn("grep -e", "arg");
tail, tail_fn();
}
```
The CLIgen method uses the treename assignment instead, but is otherwise similar:
```
treename="|mypipe";
```
Callbacks referenced in a pipe tree, such as `grep_fn`, are marked
with a `CC_FLAGS_PIPE_FUNCTION` flag, to distingusih them from regular
callbacks. All callbacks in a pipe tree are marked as pipe
functions. This is done when parsing (or just after). Clixon and
CLIgen does it slightly different:
* CLIgen: If the "active" tree is a pipe-tree (starts with '|') then a parsed callback is a pipe function
* Clixon: After parsing, if the 'mode' is a pipe-tree, the traverse all callbacks and mark them
### Pipe functions
The pipe callback functions example, are called with the same arguments as regular CLIgen callbacks
```
int tail_fn(cligen_handle h, cvec *cvv, cvec *argv)
```
where `cvv` is the command line and `argv` are the arguments in the call.
However, the difference is that a pipe function is expected to receive
input on stdin and produce output to stdout.
This can be done by making an `exec` call to a UNIX command, as follows:
```
execl("/usr/bin/tail", (char *) NULL);
```
Another way to write a pipe function is just by using stdin and stdout directly, without an exec.
The clixon system itself arranges the input and output to be
redirected properly, if the pipe function is a part of a pipe tree as described below.
### Explicit pipe references
A straightforward way to reference a pipe tree is by using an explicit pipe-tree reference as follows:
```
set {
@|mypipe, regular_cb();
}
```
Note that the `regular_cb()` is stated as an argument to the mypipe reference. This means it will be preended to each callback in 'mypipe'.
For example, in the following CLI call:
```
cli> set | tail
```
Two callbacks will be evaluated as follows:
```
regular_cb() | tail_fn()
```
where the stdout of `regular_cb()` is redirected to the stdin of `tail_fn()`.
If a call without pipe is wanted, the CLI explicit reference can be extended as follows:
```
set, regular_cb(); {
@|mypipe, regular_cb();
}
```
### Default pipe references
Instead of explicitly stating a pipe tree after each command, it is
possible to make a default pipe-tree rule, which uses dynamic expansion to add pipe-trees automatically.
In Clixon:
```
CLICON_PIPETREE="|mypipe"; # in Clixon
pipetree="|mypipe"; # in CLIgen
```
This autoamtically expands each terminal command into a pipe-tree reference in a dynamic way. For example, assume the command `set, regular_cb();` is specified and a user types `set `.
This expands the syntax by adding the `@|mypipe, regular_cb()` which is in turn expanded to:
```
set, regular_cb(); {
\| {
grep <arg:rest>, grep_fn("grep -e", "arg");
tail, tail_fn();
}
}
```
### Callback evaluation
When a callback is evaluated, a pipe function, if present, is run in a forked sub-process and a
'pipe-socket' is registered as input to that process.
Then, the regular callback is called. When the callback calls `cligen_output()`, the output is redirected to the pipe-socket (sock) to the pipe-function, then back to `cligen_output_basic()`, as shown in the following sketch:
```
regular_cb() --> cligen_output() --> (sock) --> tail_fn() --> (sock) --> cligen_output_basic()
```
The `cligen_output_basic()` makes no redirection to avoid recursion.
### Discussion
The explicit syntax may seem counterintuitive, since the pipe-tree reference is made *before* the original call: `@|mypipe, regular_cb()`.
It would be nicer to place the pipe-tree reference *after* the original call, something like:
```
regular_cb() |mypipe;
```
Further, the default pipe references rely on a statement (`pipetree=`
or `CLICON_PIPETREE=`) in the *top-level* tree. This means that any
different setting in a referenced tree is not significant.
It can be discussed whether this is good or bad. The advantage is that
it is easy to make a default statement in a single place. It could be
a lot of work to find out which sub-trees are referenced and change
them all.

5
lib/src/clixon_plugin.c
View file

@ -415,7 +415,7 @@ clixon_plugins_load(clicon_handle h,
int ret;
plugin_module_struct *ms = plugin_module_struct_get(h);
clicon_debug(1, "%s", __FUNCTION__);
clicon_debug(CLIXON_DBG_DETAIL, "%s", __FUNCTION__);
if (ms == NULL){
clicon_err(OE_PLUGIN, EINVAL, "plugin module not initialized");
goto done;
@ -426,8 +426,7 @@ clixon_plugins_load(clicon_handle h,
/* Load all plugins */
for (i = 0; i < ndp; i++) {
snprintf(filename, MAXPATHLEN-1, "%s/%s", dir, dp[i].d_name);
clicon_debug(1, "DEBUG: Loading plugin '%.*s' ...",
(int)strlen(filename), filename);
clicon_debug(CLIXON_DBG_DEFAULT, "Loading plugin '%s'", filename);
if ((ret = plugin_load_one(h, filename, function, RTLD_NOW, &cp)) < 0)
goto done;
if (ret == 0)

2
lib/src/clixon_yang.c
View file

@ -2212,7 +2212,7 @@ ys_populate_leaf(clicon_handle h,
cv_dec64_n_set(cv, fraction_digits);
if (cv_name_set(cv, ys->ys_argument) == NULL){
clicon_err(OE_YANG, errno, "cv_new_set");
clicon_err(OE_YANG, errno, "cv_name_set");
goto done;
}
/* get parent of where type is defined, can be original object */

221
test/test_cli_pipe.sh Executable file
View file

@ -0,0 +1,221 @@
#!/usr/bin/env bash
# CLIgen output pipe functions
# Note, | must be escaped as \| otherwise shell's pipe is used (w possibly same result)
# XXX Autocli does not work
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
APPNAME=example
# include err() and new() functions and creates $dir
cfg=$dir/conf_yang.xml
fyang=$dir/clixon-example.yang
clidir=$dir/clidir
if [ ! -d $clidir ]; then
mkdir $clidir
fi
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
<CLICON_YANG_DIR>${YANG_INSTALLDIR}</CLICON_YANG_DIR>
<CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
<CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
<CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
<CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
<CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
<CLICON_SOCK>/usr/local/var/$APPNAME/$APPNAME.sock</CLICON_SOCK>
<CLICON_BACKEND_PIDFILE>/usr/local/var/$APPNAME/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>/usr/local/var/$APPNAME</CLICON_XMLDB_DIR>
</clixon-config>
EOF
cat <<EOF > $fyang
module clixon-example{
yang-version 1.1;
namespace "urn:example:clixon";
prefix ex;
/* Generic config data */
container table{
list parameter{
key name;
leaf name{
type string;
}
leaf value{
type string;
}
}
}
}
EOF
cat <<EOF > $clidir/example.cli
CLICON_MODE="example";
CLICON_PROMPT="%U@%H %W> ";
# Autocli syntax tree operations
edit @datamodel, cli_auto_edit("datamodel");
up, cli_auto_up("datamodel");
top, cli_auto_top("datamodel");
set @datamodel, cli_auto_set();
delete("Delete a configuration item") {
@datamodel, cli_auto_del();
all("Delete whole candidate configuration"), delete_all("candidate");
}
commit("Commit the changes"), cli_commit();
EOF
cat <<EOF > $clidir/nodefault.cli
CLICON_MODE="nodefault";
CLICON_PROMPT="%U@%H %W> ";
show("Show a particular state of the system"){
implicit("No pipe function") {
configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);
}
explicit("Explicit pipe function") {
configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{
@|mypipe, cli_show_auto_mode("candidate", "xml", true, false);
}
}
autocli("Generated tree") @datamodelshow, cli_show_auto("candidate", "xml", true, false, "report-all");
treeref @treeref;
}
EOF
cat <<EOF > $clidir/default.cli
CLICON_MODE="default";
CLICON_PROMPT="%U@%H %W> ";
CLICON_PIPETREE="|mypipe"; # Only difference from nodefault
show("Show a particular state of the system"){
implicit("No pipe function") {
configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);
}
explicit("Explicit pipe function") {
configuration("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{
@|mypipe, cli_show_auto_mode("candidate", "xml", true, false);
}
}
autocli("Generated tree") @datamodelshow, cli_show_auto("candidate", "xml", true, false, "report-all");
treeref @treeref;
}
EOF
cat <<EOF > $clidir/treeref.cli
CLICON_MODE="treeref";
CLICON_PIPETREE="|mypipe";
implicit("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);
explicit("Show configuration"), cli_show_auto_mode("candidate", "xml", true, false);{
@|mypipe, cli_show_auto_mode("candidate", "xml", true, false);
}
EOF
cat <<EOF > $clidir/clipipe.cli
CLICON_MODE="|mypipe"; # Must start with |
#CLICON_PIPETREE="|mypipe";
\| {
grep <arg:rest>, grep_fn("grep -e", "arg");
}
EOF
new "test params: -f $cfg"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -z -f $cfg
if [ $? -ne 0 ]; then
err
fi
new "start backend -s init -f $cfg"
start_backend -s init -f $cfg
fi
new "wait backend"
wait_backend
new "Add entry x"
expectpart "$($clixon_cli -1 -f $cfg set table parameter x value a)" 0 "^$"
new "Add entry y"
expectpart "$($clixon_cli -1 -f $cfg set table parameter y value b)" 0 "^$"
new "Commit"
expectpart "$($clixon_cli -1 -f $cfg commit)" 0 "^$"
new "Pipes with no default rule"
mode=nodefault
new "$mode show implicit"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config)" 0 "<parameter>" "</parameter>" "table" "value"
new "$mode show explicit"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config)" 0 "<parameter>" "</parameter>" "table" "value"
new "$mode show implicit | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config \| grep par 2>&1)" 255 "Unknown command"
new "$mode show explicit | grep par , expect fail"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config \| grep par)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
new "$mode show treeref explicit | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref explicit \| grep par)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
# No-default top-level rule also applies to sub-tree, feature or bug?
new "$mode show treeref implicit | grep par, expect error"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref implicit \| grep par 2>&1)" 255 "Unknown command"
# No-default top-level rule also applies to sub-tree, feature or bug?
new "$mode show autocli table | grep par, expect error"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table \| grep par 2>&1)" 255 "Unknown command"
new "Pipes with default rule"
mode=default
new "$mode show implicit"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config)" 0 "<parameter>" "</parameter>" "table" "value"
new "$mode show explicit"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config)" 0 "<parameter>" "</parameter>" "table" "value"
new "$mode show implicit | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config \| grep par 2>&1)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
new "$mode show explicit | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config \| grep par)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
new "$mode show treeref explicit | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref explicit \| grep par)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
new "$mode show treeref implicit | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show treeref implicit \| grep par)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
new "$mode show autocli table | grep par"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table \| grep par 2>&1)" 0 "<parameter>" "</parameter>" --not-- "table" "value"
new "$mode show autocli table parameter x value | grep value"
expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table parameter x value \| grep value)" 0 "<value>a</value>" --not-- "table" "parameter"
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill
pid=$(pgrep -u root -f clixon_backend)
if [ -z "$pid" ]; then
err "backend already dead"
fi
# kill backend
stop_backend -f $cfg
fi
rm -rf $dir
unset mode
new "endtest"
endtest