Optimization of yang schema mount: share yang-specs if equal

.github/workflows/ci.yml

@@ -5,7 +5,6 @@ on:
     branches:
        - master
        - test-actions
-       - restconf-native-stream
   pull_request:
     branches: [ master ]
 

CHANGELOG.md

@@ -15,6 +15,7 @@ Expected: June 2024
 
 ### Features
 
+* Optimization of yang schema mount: share yang-specs if equal
 * Changed datastore modstate to be last in file, as prior to 7.0
 * New: Event priority. Backend socket has higher prio
 * New: Split configure datastore multiple sub-files on mount-point boundaries
@@ -25,6 +26,7 @@ Expected: June 2024
 * CLI support for multiple inline commands separated by semi-colon
 * New `clixon-config@2024-04-01.yang` revision
   * Added options:
+    - `CLICON_YANG_SCHEMA_MOUNT_SHARE`: Share same YANGs of several moint-points
     - `CLICON_SOCK_PRIO`: Enable socket event priority
     - `CLICON_XMLDB_MULTI`: Split datastore into multiple sub files
     - `CLICON_NETCONF_DUPLICATE_ALLOW`: Disable duplicate check in NETCONF messages

include/clixon_custom.h

@@ -216,3 +216,6 @@
  */
 #define RESTCONF_NATIVE_STREAM
 
+/*! Temporary comparison of xyanglibs, should be removed asap
+ */
+#define YANG_SCHEMA_CMP_KLUDGE

lib/src/clixon_yang_schema_mount.c

@@ -76,6 +76,7 @@
 #include "clixon_err.h"
 #include "clixon_log.h"
 #include "clixon_debug.h"
+#include "clixon_options.h"
 #include "clixon_netconf_lib.h"
 #include "clixon_xml_io.h"
 #include "clixon_xml_map.h"
@@ -669,36 +670,203 @@ yang_schema_mount_statistics(clixon_handle h,
     return retval;
 }
 
-/*! Get yanglib from user plugin callback, parse it and mount it
+#ifdef YANG_SCHEMA_CMP_KLUDGE
+/*! XXX: This comparison is kludgy since xyanglib is not proper RFC8525
+ * XXX: it should be enough to compare xyanglib with xmodst
+ * @param[in] h        CLixon handle
+ * @param[in] xyanglib Only modules
+ * @param[in] xmodst   Has submodules
+ * @retval    1        Equal
+ * @retval    0        Found
+ * @retval   -1        Error
+ */
+static int
+yang_schema_cmp_kludge(clixon_handle h,
+                       cxobj        *xyanglib,
+                       cxobj        *xmodst)
+{
+    int    retval = -1;
+    cxobj *xy1;
+    cxobj *xy2;
+    cxobj *x1;
+    cxobj *x2;
+    cxobj *x2m;
+    char  *name;
+    char  *revision1;
+    char  *revision2;
+    int    nr1;
+    int    nr2;
+
+    if ((xy1 = xml_find_type(xyanglib, NULL, "module-set", CX_ELMNT)) == NULL ||
+        (xy2 = xml_find_type(xmodst, NULL, "module-set", CX_ELMNT)) == NULL)
+        goto noteq;
+    nr1 = 0;
+    x1 = NULL;
+    while ((x1 = xml_child_each(xy1, x1, CX_ELMNT)) != NULL) {
+        if (strcmp(xml_name(x1), "module"))
+            continue;
+        if ((name = xml_find_body(x1, "name")) == NULL){
+            clixon_debug(CLIXON_DBG_YANG, "no name");
+            goto noteq;
+        }
+        revision1 = xml_find_body(x1, "revision");
+        if ((x2 = xpath_first(xy2, NULL, "module[name='%s']", name)) == NULL){
+            if ((x2 = xpath_first(xy2, NULL, "module/submodule[name='%s']", name)) == NULL){
+                clixon_debug(CLIXON_DBG_YANG, "name mismatch %s\n", name);
+                goto noteq;
+            }
+        }
+        revision2 = xml_find_body(x2, "revision");
+        if (clicon_strcmp(revision1, revision2) != 0){
+            clixon_debug(CLIXON_DBG_YANG, "revision mismatch %s %s\n", revision1, revision2);
+            goto noteq;
+        }
+        nr1++;
+    }
+    nr2 = 0;
+    x2 = NULL;
+    while ((x2 = xml_child_each(xy2, x2, CX_ELMNT)) != NULL) {
+        if (strcmp(xml_name(x2), "module"))
+            continue;
+        nr2++;
+        x2m = NULL;
+        while ((x2m = xml_child_each(x2, x2m, CX_ELMNT)) != NULL) {
+            if (strcmp(xml_name(x2m), "submodule"))
+                continue;
+            nr2++;
+        }
+    }
+    if (nr1 != nr2){
+        clixon_debug(CLIXON_DBG_YANG, "nr mismatch %d %d", nr1, nr2);
+        goto noteq;
+    }
+    retval = 1;
+ done:
+    return retval;
+ noteq:
+    retval = 0;
+    goto done;
+}
+#endif // KLUDGE
+
+/*! Given xml mount-point node, find existing mountpoint
  *
+ * 1. Get modstate (xyanglib) of node: xyanglib, by querying backend state (via callback)
+ *    XXX this xyanglib is not proper RFC8525, submodules appear as modules WHY?
+ * 2. Loop through other existing mountpoints of same YANG (dont consider other YANG mtpoints):
+ * 3. Get yspec of mountpoint.
+ * 4: Get RFC 8525 module-state of yspec (proper RFC 8525)
+ * 5: Compare to given modstate
+ *    XXX: This comparison is kludgy since xyanglib is not proper RFC8525
+ * 6. If equal, then use existing yspec
+ * 7. Otherwise create new and parse yspec
+ * 8. set as new
  * @param[in]     h     Clixon handle
- * @param[in]     xt       
+ * @param[in]     xt    XML tree node
  * @retval        1     OK
  * @retval        0     No yanglib or problem when parsing yanglib
  * @retval       -1     Error
  */
+static int
+yang_schema_find_share(clixon_handle h,
+                       cxobj        *xt,
+                       cxobj        *xyanglib,
+                       yang_stmt   **yspecp)
+{
+    int        retval = -1;
+    yang_stmt *yt;
+    cvec          *cvv;
+    cg_var        *cv;
+    yang_stmt     *yspec1 = NULL;
+    cbuf          *cb = NULL;
+    cxobj         *xmodst = NULL;
+
+    yt = xml_spec(xt);
+    if ((cvv = yang_cvec_get(yt)) != NULL){
+        if ((cb = cbuf_new()) == NULL){
+            clixon_err(OE_UNIX, 0, "clicon buffer");
+            goto done;
+        }
+        /* 2. Loop through other existing mountpoints of same YANG */
+        cv = NULL;
+        while ((cv = cvec_each(cvv, cv)) != NULL) {
+            /* 3. Get yspec of mountpoint */
+            if ((yspec1 = cv_void_get(cv)) == NULL)
+                continue;
+            /* 4: Get RFC 8525 module-state of yspec */
+            cbuf_reset(cb);
+            /* Build a cb string: <modules-state>... */
+            if (yang_modules_state_build(h, yspec1, "4242", 1, cb) < 0)
+                goto done;
+            /* Parse cb, x is on the form: <top><modules-state>...
+             * Note, list is not sorted since it is state (should not be)
+             */
+            if (clixon_xml_parse_string(cbuf_get(cb), YB_MODULE, yspec1, &xmodst, NULL) < 0)
+                goto done;
+            /* 5: Compare to given modstate
+             */
+            if (xml_rootchild(xmodst, 0, &xmodst) < 0)
+                goto done;
+            if (yang_schema_cmp_kludge(h, xyanglib, xmodst) < 0)
+                goto done;
+            if (xmodst){
+                xml_free(xmodst);
+                xmodst = NULL;
+            }
+        } /* while */
+        if (cv != NULL){
+            yang_ref_inc(yspec1); /* share */
+            *yspecp = yspec1;
+        }
+    }
+    retval = 0;
+ done:
+    if (xmodst)
+        xml_free(xmodst);
+    if (cb)
+        cbuf_free(cb);
+    return retval;
+}
+
+/*! Get yanglib from user plugin callback, parse it and mount it
+ *
+ * Optionally check for shared yspec
+ * @param[in]  h   Clixon handle
+ * @param[in]  xt  XML tree node
+ * @retval     1   OK
+ * @retval     0   No yanglib or problem when parsing yanglib
+ * @retval    -1   Error
+ */
 int
 yang_schema_yanglib_parse_mount(clixon_handle h,
                                 cxobj        *xt)
 {
     int            retval = -1;
-    cxobj         *yanglib = NULL;
+    cxobj         *xyanglib = NULL;
     yang_stmt     *yspec = NULL;
     int            ret;
-    int            config = 1;
-    validate_level vl = VL_FULL;
 
-    if (clixon_plugin_yang_mount_all(h, xt, &config, &vl, &yanglib) < 0)
+    /* 1. Get modstate (xyanglib) of node: xyanglib, by querying backend state (via callback)
+     *    XXX this xyanglib is not proper RFC8525, submodules appear as modules WHY?
+     */
+    if (clixon_plugin_yang_mount_all(h, xt, NULL, NULL, &xyanglib) < 0)
         goto done;
-    if (yanglib == NULL)
+    if (xyanglib == NULL)
-        goto anydata;
-    /* Parse it and set mount-point */
-    if ((yspec = yspec_new()) == NULL)
-        goto done;
-    if ((ret = yang_lib2yspec(h, yanglib, yspec)) < 0)
-        goto done;
-    if (ret == 0)
         goto anydata;
+    /* Optimization: find equal yspec from other mount-point */
+    if (clicon_option_bool(h, "CLICON_YANG_SCHEMA_MOUNT_SHARE")) {
+        if (yang_schema_find_share(h, xt, xyanglib, &yspec) < 0)
+            goto done;
+    }
+    if (yspec == NULL){
+        /* Parse it and set mount-point */
+        if ((yspec = yspec_new()) == NULL)
+            goto done;
+        if ((ret = yang_lib2yspec(h, xyanglib, yspec)) < 0)
+            goto done;
+        if (ret == 0)
+            goto anydata;
+    }
     if (xml_yang_mount_set(h, xt, yspec) < 0)
         goto done;
     yspec = NULL;
@@ -706,8 +874,8 @@ yang_schema_yanglib_parse_mount(clixon_handle h,
  done:
     if (yspec)
         ys_free(yspec);
-    if (yanglib)
+    if (xyanglib)
-        xml_free(yanglib);
+        xml_free(xyanglib);
     return retval;
  anydata:   // Treat as anydata
     retval = 0;

yang/clixon/clixon-config@2024-04-01.yang

@@ -52,11 +52,12 @@ module clixon-config {
     revision 2024-04-01 {
         description
             "Added options:
-                    CLICON_SOCK_PRIO: Enable socket event priority
+                CLICON_YANG_SCHEMA_MOUNT_SHARE: Share same YANGs of equal moint-points.
-                    CLICON_XMLDB_MULTI: Split datastore into multiple sub files
+                CLICON_SOCK_PRIO: Enable socket event priority
-                    CLICON_NETCONF_DUPLICATE_ALLOW: Disable duplicate check in NETCONF messages.
+                CLICON_XMLDB_MULTI: Split datastore into multiple sub files
-                    CLICON_CLI_OUTPUT_FORMAT: Default CLI output format
+                CLICON_NETCONF_DUPLICATE_ALLOW: Disable duplicate check in NETCONF messages.
-                    CLICON_AUTOLOCK: Implicit locks
+                CLICON_CLI_OUTPUT_FORMAT: Default CLI output format
+                CLICON_AUTOLOCK: Implicit locks
              Released in Clixon 7.1";
     }
     revision 2024-01-01 {
@@ -556,6 +557,17 @@ module clixon-config {
                  Further, autocli syntax is added by definining a tree resolve wrapper";
             default false;
         }
+        leaf CLICON_YANG_SCHEMA_MOUNT_SHARE {
+            type boolean;
+            description
+                "For optimization purposes, share same YANGs of equal moint-points.
+                 The mount-points need to be 'equal' in the sense that it has the same YANG
+                 (yangmnt:mount-point is on same node).
+                 A comparison is made between yang modules and revision and must match exactly.
+                 If so, a new yang-spec is not created, instead the other is used.
+                 Only if CLICON_YANG_SCHEMA_MOUNT is enabled";
+            default false;
+        }
         leaf CLICON_BACKEND_REGEXP {
             type string;
             description