From c24c38dbb5aaab826f069ffc30da5daaa1753aa8 Mon Sep 17 00:00:00 2001
From: Olof hagsand <olof@hagsand.se>
Date: Mon, 2 Dec 2024 16:24:34 +0100
Subject: [PATCH] CLI generic pipe callbacks

---
 CHANGELOG.md                              |  3 +
 apps/cli/cli_auto.c                       |  2 -
 apps/cli/cli_common.c                     |  6 +-
 apps/cli/cli_handle.c                     |  1 -
 apps/cli/cli_main.c                       |  1 -
 apps/cli/cli_pipe.c                       | 82 ++++++++++++++++++++---
 apps/cli/cli_pipe.h                       | 48 +++++++++++++
 apps/cli/cli_show.c                       | 69 ++++++++++++++++++-
 apps/cli/clixon_cli_api.h                 |  2 +-
 test/test_cli_pipe.sh                     | 28 +++++++-
 yang/clixon/clixon-config@2024-11-01.yang | 12 ++++
 11 files changed, 234 insertions(+), 20 deletions(-)
 create mode 100644 apps/cli/cli_pipe.h

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5bcaaad..b5d72f14 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@ Expected: January 2025
 
 ### Features
 
+* New: CLI generic pipe callbacks
+  * Add scripts in `CLICON_CLI_PIPE_DIR`
 * New: [feature request: support xpath functions for strings](https://github.com/clicon/clixon/issues/556)
   * Added: re-match, substring, string, string-length, translate, substring-before, substring-after, starts-with
 * Added support for system-only-config data
@@ -26,6 +28,7 @@ Expected: January 2025
   * New `ca_system_only` backend callback for reading system-only data
 * New `clixon-config@2024-11-01.yang` revision
   * Changed: `CLICON_NETCONF_DUPLICATE_ALLOW` to not only check but remove duplicates
+  * Added: `CLICON_CLI_PIPE_DIR`
   * Added: `CLICON_XMLDB_SYSTEM_ONLY_CONFIG`
 
 ### C/CLI-API changes on existing features
diff --git a/apps/cli/cli_auto.c b/apps/cli/cli_auto.c
index 7d1cbc3d..594951e4 100644
--- a/apps/cli/cli_auto.c
+++ b/apps/cli/cli_auto.c
@@ -52,9 +52,7 @@
 #include <stdarg.h>
 #include <time.h>
 #include <ctype.h>
-
 #include <unistd.h>
-#include <dirent.h>
 #include <syslog.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
diff --git a/apps/cli/cli_common.c b/apps/cli/cli_common.c
index 590a8b85..22a7c249 100644
--- a/apps/cli/cli_common.c
+++ b/apps/cli/cli_common.c
@@ -48,7 +48,6 @@
 #include <time.h>
 #include <ctype.h>
 #include <unistd.h>
-#include <dirent.h>
 #include <syslog.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
@@ -807,7 +806,8 @@ cli_set_mode(clixon_handle h,
  * @param[in] argv Function arguments
  * @retval   0 OK, returns the exit code of the program
  * @retval  -1 Error
- *
+ * @note utility, should probably not be in lib
+ * @see cli_start_shell
  */
 int
 cli_start_program(clixon_handle h,
@@ -926,6 +926,8 @@ done:
  * @param[in]   argv  [<shell>], defaults to "sh"
  * @retval      0     OK
  * @retval     -1     Error
+ * @note utility, should probably not be in lib
+ * @see cli_start_program
  */
 int
 cli_start_shell(clixon_handle h,
diff --git a/apps/cli/cli_handle.c b/apps/cli/cli_handle.c
index c72637a7..fec17dfb 100644
--- a/apps/cli/cli_handle.c
+++ b/apps/cli/cli_handle.c
@@ -49,7 +49,6 @@
 #include <errno.h>
 #include <assert.h>
 #include <dlfcn.h>
-#include <dirent.h>
 #include <grp.h>
 #include <signal.h>
 #include <sys/types.h>
diff --git a/apps/cli/cli_main.c b/apps/cli/cli_main.c
index 623ccd21..0bc3b7f0 100644
--- a/apps/cli/cli_main.c
+++ b/apps/cli/cli_main.c
@@ -45,7 +45,6 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <dlfcn.h>
-#include <dirent.h>
 #include <unistd.h>
 #include <errno.h>
 #include <signal.h>
diff --git a/apps/cli/cli_pipe.c b/apps/cli/cli_pipe.c
index 7e35b812..304cee83 100644
--- a/apps/cli/cli_pipe.c
+++ b/apps/cli/cli_pipe.c
@@ -50,9 +50,7 @@
 #include <stdarg.h>
 #include <time.h>
 #include <ctype.h>
-
 #include <unistd.h>
-#include <dirent.h>
 #include <syslog.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
@@ -71,24 +69,25 @@
 #include <clixon/clixon.h>
 
 #include "clixon_cli_api.h"
+#include "cli_pipe.h"
 
 /* General-purpose pipe output function
  *
  * @param[in]  h      Clixon handle
- * @param[in]  cmd    Command to exec
+ * @param[in]  cmd    Command/file to exec
  * @param[in]  option Option to command (or NULL)
  * @param[in]  value  Command argument value (or NULL)
  */
-int
+static int
 pipe_arg_fn(clixon_handle h,
             char         *cmd,
             char         *option,
             char         *value)
 {
-    int          retval = -1;
-    struct stat  fstat;
-    char       **argv = NULL;
-    int          i;
+    int         retval = -1;
+    struct stat fstat;
+    char      **argv = NULL;
+    int         i;
 
     if (cmd == NULL || strlen(cmd) == 0){
         clixon_err(OE_PLUGIN, EINVAL, "cmd '%s' NULL or empty", cmd);
@@ -108,8 +107,10 @@ pipe_arg_fn(clixon_handle h,
     }
     i = 0;
     argv[i++] = cmd;
-    argv[i++] = option;
-    argv[i++] = value;
+    if (option) {
+        argv[i++] = option;
+        argv[i++] = value;
+    }
     argv[i++] = NULL;
     retval = execv(cmd, argv);
  done:
@@ -198,6 +199,7 @@ pipe_wc_fn(clixon_handle h,
     if (cvec_len(argv) != 1){
         clixon_err(OE_PLUGIN, EINVAL, "Received %d arguments. Expected: <NUM>", cvec_len(argv));
         goto done;
+
     }
     if ((cv = cvec_i(argv, 0)) != NULL &&
         (str = cv_string_get(cv)) != NULL &&
@@ -392,6 +394,66 @@ pipe_save_file(clixon_handle h,
     return retval;
 }
 
+/*! pipe function: start script
+ *
+ * @param[in]  h     Clixon handle
+ * @param[in]  cvv   Vector of cli string and instantiated variables
+ * @param[in]  argv  String vector of options. Format:
+ *                     name     Name of cv containing script filename
+ * @retval     0     OK
+ * @retval    -1     Error
+ * @code
+ *  format("Generic format by callback") <callback:string>("Name of generic format"),
+ *         pipe_generic_callback("callback");
+ * @endcode
+ */
+int
+pipe_generic(clixon_handle h,
+             cvec         *cvv,
+             cvec         *argv)
+{
+    int     retval = -1;
+    char   *cvname;
+    cg_var *cv;
+    char   *dir;
+    char   *filename;
+    char   *script = NULL;
+    cbuf   *cb = NULL;
+
+    if (cvec_len(argv) != 1) {
+        clixon_err(OE_PLUGIN, EINVAL, "Received %d arguments. Expected: <script>", cvec_len(argv));
+        goto done;
+    }
+    if ((cvname = cv_string_get(cvec_i(argv, 0))) != NULL) {
+        if ((cv = cvec_find(cvv, cvname)) != NULL){
+            if ((script = cv_string_get(cv)) == NULL){
+                clixon_err(OE_PLUGIN, EINVAL, "cv name is empty");
+                goto done;
+            }
+        }
+    }
+    if (script == NULL || strlen(script) == 0)
+        goto ok;
+    if ((dir = clicon_option_str(h, "CLICON_CLI_PIPE_DIR")) == NULL){
+        clixon_err(OE_UNIX, 0, "CLICON_CLI_PIPE_DIR not set");
+        goto done;
+    }
+    if ((cb = cbuf_new()) == NULL){
+        clixon_err(OE_UNIX, errno, "cbuf_new");
+        goto done;
+    }
+    cprintf(cb, "%s/%s", dir, script);
+    filename = cbuf_get(cb);
+    if (pipe_arg_fn(h, filename, NULL, NULL) < 0)
+        goto done;
+ ok:
+    retval = 0;
+ done:
+    if (cb)
+        cbuf_free(cb);
+    return retval;
+}
+
 /*! Test cli callback calls cligen_output with output lines as given by function arguments
  *
  * Only for test or debugging to generate output to eg cligen_output scrolling
diff --git a/apps/cli/cli_pipe.h b/apps/cli/cli_pipe.h
new file mode 100644
index 00000000..8355d7ea
--- /dev/null
+++ b/apps/cli/cli_pipe.h
@@ -0,0 +1,48 @@
+/*
+ *
+  ***** BEGIN LICENSE BLOCK *****
+ 
+  Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
+  Copyright (C) 2017-2019 Olof Hagsand
+  Copyright (C) 2020-2022 Olof Hagsand and Rubicon Communications, LLC (Netgate)
+
+  This file is part of CLIXON.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  Alternatively, the contents of this file may be used under the terms of
+  the GNU General Public License Version 3 or later (the "GPL"),
+  in which case the provisions of the GPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of the GPL, and not to allow others to
+  use your version of this file under the terms of Apache License version 2, 
+  indicate your decision by deleting the provisions above and replace them with
+  the  notice and other provisions required by the GPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the Apache License version 2 or the GPL.
+
+  ***** END LICENSE BLOCK *****
+
+ */
+
+#ifndef _CLI_PIPE_H_
+#define _CLI_PIPE_H_
+
+int pipe_grep_fn(clixon_handle h, cvec *cvv, cvec *argv);
+int pipe_wc_fn(clixon_handle h, cvec *cvv, cvec *argv);
+int pipe_tail_fn(clixon_handle h, cvec *cvv, cvec *argv);
+int pipe_showas_fn(clixon_handle h, cvec *cvv, cvec *argv);
+int pipe_save_file(clixon_handle h, cvec *cvv, cvec *argv);
+int pipe_generic(clixon_handle h, cvec *cvv, cvec *argv);
+
+#endif /* _CLI_PIPE_H_ */
diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c
index 585e108d..dd121b52 100644
--- a/apps/cli/cli_show.c
+++ b/apps/cli/cli_show.c
@@ -49,7 +49,6 @@
 #include <time.h>
 #include <ctype.h>
 #include <unistd.h>
-#include <dirent.h>
 #include <syslog.h>
 #include <pwd.h>
 #include <inttypes.h>
@@ -611,6 +610,72 @@ expand_yang_list(void   *h,
     return retval;
 }
 
+/*! Completion callback of variable for file directory
+ *
+ * Returns an expand-type list of commands as used by cligen 'expand'
+ * functionality.
+ *
+ * Assume callback given in a cligen spec: a <x:int expand_dbvar("db" "<xmlkeyfmt>")
+ * @param[in]   h        clicon handle
+ * @param[in]   name     Name of this function (eg "expand_dbvar")
+ * @param[in]   cvv      The command so far. Eg: cvec [0]:"a 5 b"; [1]: x=5;
+ * @param[in]   argv     Arguments given at the callback:
+ *   <dir>    File directory
+ *   <regex>  Regexp of files to show
+ * @param[out]  commands vector of function pointers to callback functions
+ * @param[out]  helptxt  vector of pointers to helptexts
+ * @retval      0        OK
+ * @retval     -1        Error
+ * @code
+ *    <callback:string expand_dir("/usr/local/var/pipedir", "\.sh$")>("comment"), Command;
+ * @endcode
+ * @see expand_dbvar
+ */
+int
+expand_dir(void   *h,
+           char   *name,
+           cvec   *cvv,
+           cvec   *argv,
+           cvec   *commands,
+           cvec   *helptexts)
+{
+    int            retval = -1;
+    int            argc = 0;
+    cg_var        *cv;
+    char          *dir;
+    char          *regexp = NULL;
+    struct dirent *dp;
+    int            ndp;
+    int            i;
+
+    if (argv == NULL || cvec_len(argv) < 1 || cvec_len(argv) > 2){
+        clixon_err(OE_PLUGIN, EINVAL, "requires arguments: <dir> [<regexp>]");
+        goto done;
+    }
+    if ((cv = cvec_i(argv, argc++)) == NULL){
+        clixon_err(OE_PLUGIN, 0, "Error when accessing argument <schemanode>");
+        goto done;
+    }
+    dir = cv_string_get(cv);
+    if (cvec_len(argv) > argc){
+        if ((cv = cvec_i(argv, argc++)) == NULL){
+            clixon_err(OE_PLUGIN, 0, "Error when accessing argument <schemanode>");
+            goto done;
+        }
+        regexp = cv_string_get(cv);
+    }
+    if ((ndp = clicon_file_dirent(dir, &dp, regexp, S_IFREG)) < 0)
+        goto done;
+    for (i = 0; i < ndp; i++) {
+        cvec_add_string(commands, NULL, dp[i].d_name);
+    }
+    retval = 0;
+ done:
+    if (dp)
+        free(dp);
+    return retval;
+}
+
 /*! CLI callback show yang spec. If arg given matches yang argument string 
  *
  * @param[in]  h     Clixon handle
@@ -1080,7 +1145,7 @@ cli_show_version(clixon_handle h,
  *  [<mt-point>]     Optional YANG path-arg/xpath from mount-point
  *   <dbname>        Name of datastore, such as "running"
  * -- from here optional:
- *   <format>        "text"|"xml"|"json"|"cli"|"netconf" (see format_enum), default: xml
+ *   <format>        text|xml|json|cli|netconf|default (see format_enum), default: xml
  *   <pretty>        true|false: pretty-print or not
  *   <state>         true|false: also print state
  *   <default>       Retrieval mode: report-all, trim, explicit, report-all-tagged, 
diff --git a/apps/cli/clixon_cli_api.h b/apps/cli/clixon_cli_api.h
index f079962d..e4251ff1 100644
--- a/apps/cli/clixon_cli_api.h
+++ b/apps/cli/clixon_cli_api.h
@@ -33,7 +33,7 @@
 
   ***** END LICENSE BLOCK *****
 
-  * Note, this is a CLICON API file, only exprorted function prototypes should appear here
+  * Note, this is a CLICON API file, only exported function prototypes should appear here
  */
 
 #ifndef _CLIXON_CLI_API_H_
diff --git a/test/test_cli_pipe.sh b/test/test_cli_pipe.sh
index b4316f99..9509de2f 100755
--- a/test/test_cli_pipe.sh
+++ b/test/test_cli_pipe.sh
@@ -12,10 +12,14 @@ APPNAME=example
 cfg=$dir/conf_yang.xml
 fyang=$dir/clixon-example.yang
 clidir=$dir/clidir
+pipedir=$dir/pipedir
 
 if [ ! -d $clidir ]; then
     mkdir $clidir
 fi
+if [ ! -d $pipedir ]; then
+    mkdir $pipedir
+fi
 
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
@@ -24,6 +28,7 @@ cat <<EOF > $cfg
   <CLICON_YANG_MAIN_FILE>$fyang</CLICON_YANG_MAIN_FILE>
   <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
   <CLICON_CLISPEC_DIR>$clidir</CLICON_CLISPEC_DIR>
+  <CLICON_CLI_PIPE_DIR>$pipedir</CLICON_CLI_PIPE_DIR>
   <CLICON_CLI_DIR>/usr/local/lib/$APPNAME/cli</CLICON_CLI_DIR>
   <CLICON_CLI_MODE>$APPNAME</CLICON_CLI_MODE>
   <CLICON_SOCK>/usr/local/var/run/$APPNAME.sock</CLICON_SOCK>
@@ -87,6 +92,7 @@ show("Show a particular state of the system"){
 }
 EOF
 
+
 # First file of mode does not have CLICON_PIPETREE, next have
 cat <<EOF > $clidir/before.cli
 CLICON_MODE="default";
@@ -131,10 +137,24 @@ CLICON_MODE="|mypipe"; # Must start with |
    show {
      json, pipe_showas_fn("json");
      text, pipe_showas_fn("text");
+
    }
+   generic("Generic callbacks") <callback:string expand_dir("$pipedir", "\.sh$")>("Callback"), pipe_generic("callback");
 }
 EOF
 
+cat <<EOF > $pipedir/first.sh
+#!/usr/bin/env bash
+head -1
+EOF
+chmod 755  $pipedir/first.sh
+
+cat <<EOF > $pipedir/last.sh
+#!/usr/bin/env bash
+tail -1
+EOF
+chmod 755  $pipedir/last.sh
+
 new "test params: -f $cfg"
 if [ $BE -ne 0 ]; then
     new "kill old backend"
@@ -204,7 +224,6 @@ expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table \| grep par 2>&
 
 new "Pipes with default rule"
 mode=default
-
 new "$mode show implicit"
 expectpart "$($clixon_cli -1 -m $mode -f $cfg show implicit config)" 0 "<parameter>" "</parameter>" "table" "value"
 
@@ -229,6 +248,13 @@ expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table \| grep par 2>&
 new "$mode show autocli table parameter x value | grep value"
 expectpart "$($clixon_cli -1 -m $mode -f $cfg show autocli table parameter x value \| grep value)" 0 "<value>a</value>" --not-- "table" "parameter"
 
+# Generic callbacks
+new "generic pipe: first"
+expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config \| generic first.sh)" 0 "^<table xmlns=\"urn:example:clixon\">$" --not-- "parameter"
+
+new "generic pipe: last"
+expectpart "$($clixon_cli -1 -m $mode -f $cfg show explicit config \| generic last.sh)" 0 "^</table>$" --not-- parameter
+
 if [ $BE -ne 0 ]; then
     new "Kill backend"
     # Check if premature kill
diff --git a/yang/clixon/clixon-config@2024-11-01.yang b/yang/clixon/clixon-config@2024-11-01.yang
index 427221a0..88c3584c 100644
--- a/yang/clixon/clixon-config@2024-11-01.yang
+++ b/yang/clixon/clixon-config@2024-11-01.yang
@@ -53,6 +53,7 @@ module clixon-config {
         description
             "Added options:
                 CLICON_XMLDB_SYSTEM_ONLY_CONFIG
+                CLICON_CLI_PIPE_DIR
              Changed: CLICON_NETCONF_DUPLICATE_ALLOW to not only check but remove duplicates
              Released in Clixon 7.3";
     }
@@ -1063,6 +1064,17 @@ module clixon-config {
             description
                 "Default CLI output format.";
         }
+        leaf CLICON_CLI_PIPE_DIR {
+            type string;
+            description
+                "Directory containing generic pipe functions.
+                 The pipe function pipe_generic() uses this dir
+                 May be used for formatting and should use stdin and stdout.
+                 If scripts should be shebanged
+                 Recommend to jail this dir
+                 ";
+        }
+
         /* Internal socket */
         leaf CLICON_SOCK_FAMILY {
             type socket_address_family;