diff --git a/CHANGELOG.md b/CHANGELOG.md index a0abaae8..8eccb09d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Clixon Changelog +## 3.4.0 (Upcoming) + +* Datastore cache introduced: cache XML tree in memory for faster get access + +* Moved XML_CHILD_HASH to variable instead of constant. + +* Added yang to XML API + + * xml_new(char *name, cxobj *xn_parent) --> xml_new(char *name, cxobj *xn_parent, void *spec) + * xml_new_spec(char *name, cxobj *xn_parent, void *spec) --> xml_new(char *name, cxobj *xn_parent, void *spec) + * clicon_xml_parse_string --> clicon_xml_parse_str + * clicon_xml_parse_file(int fd, cxobj **xml_top, char *endtag) --> clicon_xml_parse_file(int fd, char *endtag, yang_spec *yspec, cxobj **xt) + * clicon_xml_parse_str(char *str, cxobj **xml_top) --> clicon_xml_parse_str(char *str, yang_spec *yspec, cxobj **xml_top) + * clicon_xml_parse(cxobj **cxtop, char *format, ...) --> clicon_xml_parse(cxobj **cxtop, yang_spec *yspec, char *format, ...) + * xml_parse(char *str, cxobj *xt) --> xml_parse(char *str, yang_spec *yspec, cxobj *xt); + ## 3.3.3 (25 November 2017) Thanks to Matthew Smith, Joe Loeliger at Netgate; Fredrik Pettai at diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index f65bf6c5..cb0cc3b5 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -443,7 +443,7 @@ load_extraxml(clicon_handle h, clicon_err(OE_UNIX, errno, "open(%s)", filename); goto done; } - if (clicon_xml_parse_file(fd, &xt, "") < 0) + if (clicon_xml_parse_file(fd, "", NULL, &xt) < 0) goto done; /* Replace parent w first child */ if (xml_rootchild(xt, 0, &xt) < 0) diff --git a/apps/backend/backend_plugin.c b/apps/backend/backend_plugin.c index 9c04a5b1..75029f4e 100644 --- a/apps/backend/backend_plugin.c +++ b/apps/backend/backend_plugin.c @@ -752,7 +752,7 @@ backend_statedata_call(clicon_handle h, for (i = 0; i < _nplugins; i++) { p = &_plugins[i]; if (p->p_statedata) { - if ((x = xml_new("config", NULL)) == NULL) + if ((x = xml_new("config", NULL, NULL)) == NULL) goto done; if ((p->p_statedata)(h, xpath, x) < 0){ retval = 1; diff --git a/apps/cli/cli_common.c b/apps/cli/cli_common.c index ff3d046e..564ce423 100644 --- a/apps/cli/cli_common.c +++ b/apps/cli/cli_common.c @@ -233,12 +233,12 @@ cli_dbxml(clicon_handle h, if (api_path_fmt2api_path(api_path_fmt, cvv, &api_path) < 0) goto done; /* Create config top-of-tree */ - if ((xtop = xml_new("config", NULL)) == NULL) + if ((xtop = xml_new("config", NULL, NULL)) == NULL) goto done; xbot = xtop; if (api_path && api_path2xml(api_path, yspec, xtop, 0, &xbot, &y) < 0) goto done; - if ((xa = xml_new("operation", xbot)) == NULL) + if ((xa = xml_new("operation", xbot, NULL)) == NULL) goto done; xml_type_set(xa, CX_ATTR); if (xml_value_set(xa, xml_operation2str(op)) < 0) @@ -251,7 +251,7 @@ cli_dbxml(clicon_handle h, clicon_err(OE_UNIX, errno, "cv2str_dup"); goto done; } - if ((xb = xml_new("body", xbot)) == NULL) + if ((xb = xml_new("body", xbot, NULL)) == NULL) goto done; xml_type_set(xb, CX_BODY); if (xml_value_set(xb, str) < 0) @@ -746,7 +746,7 @@ load_config_file(clicon_handle h, clicon_err(OE_UNIX, errno, "%s: open(%s)", __FUNCTION__, filename); goto done; } - if (clicon_xml_parse_file(fd, &xt, "") < 0) + if (clicon_xml_parse_file(fd, "", NULL, &xt) < 0) goto done; if (xt == NULL) goto done; @@ -1189,7 +1189,7 @@ cli_copy_config(clicon_handle h, } toname = cv_string_get(tocv); /* Create copy xml tree x2 */ - if ((x2 = xml_new("new", NULL)) == NULL) + if ((x2 = xml_new("new", NULL, NULL)) == NULL) goto done; if (xml_copy(x1, x2) < 0) goto done; diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index 174952a3..a30fb1d1 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -166,7 +166,7 @@ expand_dbvar(void *h, xcur = xt; /* default top-of-tree */ xpathcur = xpath; /* Create config top-of-tree */ - if ((xtop = xml_new("config", NULL)) == NULL) + if ((xtop = xml_new("config", NULL, NULL)) == NULL) goto done; xbot = xtop; /* This is primarily to get "y", diff --git a/apps/netconf/netconf_lib.c b/apps/netconf/netconf_lib.c index e8854708..a6d8e87e 100644 --- a/apps/netconf/netconf_lib.c +++ b/apps/netconf/netconf_lib.c @@ -190,7 +190,7 @@ netconf_output(int s, clicon_debug(1, "SEND %s", msg); if (debug > 1){ /* XXX: below only works to stderr, clicon_debug may log to syslog */ cxobj *xt = NULL; - if (clicon_xml_parse_string(&buf, &xt) == 0){ + if (clicon_xml_parse_str(buf, NULL, &xt) == 0){ clicon_xml2file(stderr, xml_child_i(xt, 0), 0, 0); fprintf(stderr, "\n"); xml_free(xt); diff --git a/apps/netconf/netconf_main.c b/apps/netconf/netconf_main.c index ed2f1373..b7bbbeb9 100644 --- a/apps/netconf/netconf_main.c +++ b/apps/netconf/netconf_main.c @@ -98,7 +98,7 @@ process_incoming_packet(clicon_handle h, } str = str0; /* Parse incoming XML message */ - if (clicon_xml_parse_string(&str, &xreq) < 0){ + if (clicon_xml_parse_str(str, NULL, &xreq) < 0){ if ((cbret = cbuf_new()) == NULL){ cprintf(cbret, "" "operation-failed" diff --git a/apps/netconf/netconf_rpc.c b/apps/netconf/netconf_rpc.c index 6733f416..b002e0fd 100644 --- a/apps/netconf/netconf_rpc.c +++ b/apps/netconf/netconf_rpc.c @@ -135,7 +135,7 @@ netconf_get_config(clicon_handle h, cxobj *xconf; if ((source = netconf_get_target(xn, "source")) == NULL){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -163,7 +163,7 @@ netconf_get_config(clicon_handle h, /* xml_filter removes parts of xml tree not matching */ if ((strcmp(xml_name(xfilterconf), xml_name(xconf))!=0) || xml_filter(xfilterconf, xconf) < 0){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-failed" "applicatio" "error" @@ -173,7 +173,7 @@ netconf_get_config(clicon_handle h, } } else{ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-failed" "applicatio" "error" @@ -241,7 +241,7 @@ get_edit_opts(cxobj *xn, retval = 1; /* hunky dory */ return retval; parerr: /* parameter error, xret set */ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "invalid-value" "protocol" "error" @@ -317,7 +317,7 @@ netconf_edit_config(clicon_handle h, /* must have target, and it should be candidate */ if ((target = netconf_get_target(xn, "target")) == NULL || strcmp(target, "candidate")){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -329,7 +329,7 @@ netconf_edit_config(clicon_handle h, if ((xfilter = xpath_first(xn, "filter")) != NULL) { if ((ftype = xml_find_value(xfilter, "type")) != NULL) if (strcmp(ftype,"restconf")){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "invalid-value" "protocol" "error" @@ -339,7 +339,7 @@ netconf_edit_config(clicon_handle h, } if ((x = xpath_first(xn, "default-operation")) != NULL){ if (xml_operation(xml_body(x), &operation) < 0){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "invalid-value" "protocol" "error" @@ -353,7 +353,7 @@ netconf_edit_config(clicon_handle h, goto ok; /* not supported opts */ if (testopt!=TEST_THEN_SET || erropt!=STOP_ON_ERROR){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-not-supported" "protocol" "error" @@ -403,7 +403,7 @@ netconf_copy_config(clicon_handle h, char *target; /* filenames */ if ((source = netconf_get_target(xn, "source")) == NULL){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -412,7 +412,7 @@ netconf_copy_config(clicon_handle h, goto ok; } if ((target = netconf_get_target(xn, "target")) == NULL){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -450,7 +450,7 @@ netconf_delete_config(clicon_handle h, if ((target = netconf_get_target(xn, "target")) == NULL || strcmp(target, "running")==0){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -486,7 +486,7 @@ netconf_lock(clicon_handle h, char *target; if ((target = netconf_get_target(xn, "target")) == NULL){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -566,7 +566,7 @@ netconf_get(clicon_handle h, /* xml_filter removes parts of xml tree not matching */ if ((strcmp(xml_name(xfilterconf), xml_name(xconf))!=0) || xml_filter(xfilterconf, xconf) < 0){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-failed" "applicatio" "error" @@ -576,7 +576,7 @@ netconf_get(clicon_handle h, } } else{ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-failed" "applicatio" "error" @@ -627,7 +627,7 @@ netconf_kill_session(clicon_handle h, cxobj *xs; if ((xs = xpath_first(xn, "//session-id")) == NULL){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -658,7 +658,7 @@ netconf_validate(clicon_handle h, char *target; if ((target = netconf_get_target(xn, "source")) == NULL){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "missing-element" "protocol" "error" @@ -826,7 +826,7 @@ netconf_create_subscription(clicon_handle h, if ((xfilter = xpath_first(xn, "//filter")) != NULL){ if ((ftype = xml_find_value(xfilter, "type")) != NULL){ if (strcmp(ftype, "xpath") != 0){ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-failed" "application" "error" @@ -901,7 +901,7 @@ netconf_rpc_dispatch(clicon_handle h, if ((ret = netconf_plugin_callbacks(h, xe, xret)) < 0) return -1; if (ret == 0){ /* not handled by callback */ - clicon_xml_parse(xret, "" + clicon_xml_parse(xret, NULL, "" "operation-failed" "rpc" "error" diff --git a/apps/restconf/restconf_methods.c b/apps/restconf/restconf_methods.c index 857fbc30..bd078216 100644 --- a/apps/restconf/restconf_methods.c +++ b/apps/restconf/restconf_methods.c @@ -369,14 +369,14 @@ api_data_post(clicon_handle h, for (i=0; i */ /* Create config top-of-tree */ - if ((xtop = xml_new("rpc", NULL)) == NULL) + if ((xtop = xml_new("rpc", NULL, NULL)) == NULL) goto done; xbot = xtop; if (api_path2xml(oppath, yspec, xtop, 1, &xbot, &y) < 0) @@ -729,7 +729,7 @@ api_operation_post(clicon_handle h, if (data && strlen(data)){ /* Parse input data as json or xml into xml */ if (parse_xml){ - if (clicon_xml_parse_str(data, &xdata) < 0){ + if (clicon_xml_parse_str(data, NULL, &xdata) < 0){ clicon_debug(1, "%s json parse fail: %s", __FUNCTION__, data); goto done; } @@ -768,7 +768,7 @@ api_operation_post(clicon_handle h, if ((cookie = FCGX_GetParam("HTTP_COOKIE", r->envp)) != NULL && get_user_cookie(cookie, "c-user", &cookieval) ==0){ - if ((xa = xml_new("id", xtop)) == NULL) + if ((xa = xml_new("id", xtop, NULL)) == NULL) goto done; xml_type_set(xa, CX_ATTR); if (xml_value_set(xa, cookieval) < 0) diff --git a/configure b/configure index bc5b4d79..7651d102 100755 --- a/configure +++ b/configure @@ -2153,9 +2153,9 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu : ${CFLAGS="-O2"} CLIXON_VERSION_MAJOR="3" -CLIXON_VERSION_MINOR="3" -CLIXON_VERSION_PATCH="3" -CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\"" +CLIXON_VERSION_MINOR="4" +CLIXON_VERSION_PATCH="0" +CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\"" # Fix to specific version (eg 3.5) or head (3) CLIGEN_VERSION="3" diff --git a/configure.ac b/configure.ac index debb57c2..95136512 100644 --- a/configure.ac +++ b/configure.ac @@ -42,9 +42,9 @@ AC_INIT(lib/clixon/clixon.h.in) : ${CFLAGS="-O2"} CLIXON_VERSION_MAJOR="3" -CLIXON_VERSION_MINOR="3" -CLIXON_VERSION_PATCH="3" -CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}\"" +CLIXON_VERSION_MINOR="4" +CLIXON_VERSION_PATCH="0" +CLIXON_VERSION="\"${CLIXON_VERSION_MAJOR}.${CLIXON_VERSION_MINOR}.${CLIXON_VERSION_PATCH}.PRE\"" # Fix to specific version (eg 3.5) or head (3) CLIGEN_VERSION="3" diff --git a/datastore/datastore_client.c b/datastore/datastore_client.c index 82d49e69..3c9bcaeb 100644 --- a/datastore/datastore_client.c +++ b/datastore/datastore_client.c @@ -88,7 +88,8 @@ usage(char *argv0) "\t-y \tYang directory (where modules are stored). Mandatory\n" "\t-m \tYang module. Mandatory\n" "and command is either:\n" - "\tget \n" + "\tget []\n" + "\tmget []\n" "\tput (merge|replace|create|delete|remove) \n" "\tcopy \n" "\tlock \n" @@ -121,6 +122,8 @@ main(int argc, char **argv) int pid; enum operation_type op; cxobj *xt = NULL; + int i; + char *xpath; /* In the startup, logs to stderr & debug flag set later */ clicon_log_init(__PROGRAM__, LOG_INFO, CLICON_LOG_STDERR); @@ -213,12 +216,38 @@ main(int argc, char **argv) if (strcmp(cmd, "get")==0){ if (argc != 1 && argc != 2) usage(argv0); - if (xmldb_get(h, db, argc==2?argv[1]:"/", 0, &xt) < 0) + if (argc==2) + xpath = argv[1]; + else + xpath = "/"; + if (xmldb_get(h, db, xpath, 0, &xt) < 0) goto done; clicon_xml2file(stdout, xt, 0, 0); fprintf(stdout, "\n"); } + else if (strcmp(cmd, "mget")==0){ + int nr; + if (argc != 2 && argc != 3) + usage(argv0); + nr = atoi(argv[1]); + if (argc==3) + xpath = argv[2]; + else + xpath = "/"; + for (i=0;i /dev/null + xml_copy_marked 87% 200x + yang_key_match 81% 600K + yang_arg2cvec 52% 400K + cvecfree 23% 400K + +10000 entries +valgrind --tool=callgrind datastore_client -d candidate -b /tmp/text -p ../datastore/text/text.so -y /tmp -m ietf-ip mget 10 /x/y[a=574][b=574] > /dev/null + */ #ifdef HAVE_CONFIG_H @@ -71,9 +81,22 @@ struct text_handle { int th_magic; /* magic */ char *th_dbdir; /* Directory of database files */ yang_spec *th_yangspec; /* Yang spec if this datastore */ - clicon_hash_t *th_dbs; /* Hash of databases */ + clicon_hash_t *th_dbs; /* Hash of db_elements. key is dbname */ }; +/* Struct per database in hash */ +struct db_element{ + int de_pid; + cxobj *de_xml; +}; + +/* Keep datastore text in memory so that get operation need only read memory. + * Write to file on modification or file change. + * Assumes single backend + * Experimental + */ +static int datastore_cache = 1; + /*! Check struct magic number for sanity checks * return 0 if OK, -1 if fail. */ @@ -161,12 +184,28 @@ text_disconnect(xmldb_handle xh) { int retval = -1; struct text_handle *th = handle(xh); - + struct db_element *de; + char **keys = NULL; + size_t klen; + int i; + if (th){ if (th->th_dbdir) free(th->th_dbdir); - if (th->th_dbs) + if (th->th_dbs){ + if (datastore_cache){ + if ((keys = hash_keys(th->th_dbs, &klen)) == NULL) + return 0; + for(i = 0; i < klen; i++) + if ((de = hash_value(th->th_dbs, keys[i], NULL)) != NULL){ + if (de->de_xml) + xml_free(de->de_xml); + } + if (keys) + free(keys); + } hash_free(th->th_dbs); + } free(th); } retval = 0; @@ -202,7 +241,7 @@ text_getopt(xmldb_handle xh, return retval; } -/*! Set value of generic plugin option. Type of value is givenby context +/*! Set value of generic plugin option. Type of value is given by context * @param[in] xh XMLDB handle * @param[in] optname Option name * @param[in] value Value of option @@ -273,11 +312,87 @@ singleconfigroot(cxobj *xt, return retval; } +static int +xml_copy_marked(cxobj *x0, + cxobj *x1, + int flag, + int test, + int *upmark) +{ + int retval = -1; + int submark; + int mark; + cxobj *x; + cxobj *xcopy; + int iskey; + yang_node *yt; + char *name; + + mark = 0; + yt = xml_spec(x0); /* xan be null */ + x = NULL; + while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) { + name = xml_name(x); + if (xml_flag(x, flag) == test?flag:0){ + /* Pass test */ + mark++; + if (x1){ + if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL) + goto done; + if (xml_copy(x, xcopy) < 0) + goto done; + } + continue; /* mark and stop here */ + } + /* If it is key dont remove it yet (see second round) */ + if (yt){ + if ((iskey = yang_key_match(yt, name)) < 0) + goto done; + if (iskey) + continue; + } + if (xml_copy_marked(x, NULL, flag, test, &submark) < 0) + goto done; + /* if x0 is list and submark anywhere, then key subs are also marked + */ + if (submark){ + mark++; /* copy */ + if (x1){ + if ((xcopy = xml_new(name, x1, xml_spec(x))) == NULL) + goto done; + if (xml_copy_marked(x, xcopy, flag, test, &submark) < 0) + goto done; + } + + } + } + retval = 0; + done: + if (upmark) + *upmark = mark; + return retval; +} + /*! Get content of database using xpath. return a set of matching sub-trees * The function returns a minimal tree that includes all sub-trees that match * xpath. * This is a clixon datastore plugin of the the xmldb api * @see xmldb_get +#ifdef DATASTORE_CACHE +text_get 90% + clixon_xml_parse_file 74% (100x) + xml_parse 70% (100x) + clicon_xml_parseparse 70% (300x) + xml_value_append 13% (800K) + xml_new + xml_purge + clicon_xml_parselex 12% (1M) + clixon_tree_prune_flagged_sub 12% (100x) + xml_purge 12% (58000) + yang_key_match 7% (77000) + yang_arg2cvec +#endif + * */ int text_get(xmldb_handle xh, @@ -295,57 +410,81 @@ text_get(xmldb_handle xh, size_t xlen; int i; struct text_handle *th = handle(xh); + struct db_element *de = NULL; - if (text_db2file(th, db, &dbfile) < 0) - goto done; - if (dbfile==NULL){ - clicon_err(OE_XML, 0, "dbfile NULL"); - goto done; - } if ((yspec = th->th_yangspec) == NULL){ clicon_err(OE_YANG, ENOENT, "No yang spec"); goto done; } - if ((fd = open(dbfile, O_RDONLY)) < 0){ - clicon_err(OE_UNIX, errno, "open(%s)", dbfile); - goto done; - } - /* Parse file into XML tree */ - if ((clicon_xml_parse_file(fd, &xt, "")) < 0) - goto done; - /* Always assert a top-level called "config". - To ensure that, deal with two cases: - 1. File is empty -> rename top-level to "config" */ - if (xml_child_nr(xt) == 0){ - if (xml_name_set(xt, "config") < 0) - goto done; + if (datastore_cache){ + if ((de = hash_value(th->th_dbs, db, NULL)) != NULL) + xt = de->de_xml; } - /* 2. File is not empty ... -> replace root */ - else{ - /* There should only be one element and called config */ - if (singleconfigroot(xt, &xt) < 0) + if (xt == NULL){ + if (text_db2file(th, db, &dbfile) < 0) goto done; - } + if (dbfile==NULL){ + clicon_err(OE_XML, 0, "dbfile NULL"); + goto done; + } + if ((fd = open(dbfile, O_RDONLY)) < 0){ + clicon_err(OE_UNIX, errno, "open(%s)", dbfile); + goto done; + } + /* Parse file into XML tree */ + if ((clicon_xml_parse_file(fd, "", yspec, &xt)) < 0) + goto done; + /* Always assert a top-level called "config". + To ensure that, deal with two cases: + 1. File is empty -> rename top-level to "config" */ + if (xml_child_nr(xt) == 0){ + if (xml_name_set(xt, "config") < 0) + goto done; + } + /* 2. File is not empty ... -> replace root */ + else{ + /* There should only be one element and called config */ + if (singleconfigroot(xt, &xt) < 0) + goto done; + } + } /* xt == NULL */ /* Here xt looks like: ... */ - /* Add yang specification backpointer to all XML nodes */ - if (xml_apply(xt, CX_ELMNT, xml_spec_populate, yspec) < 0) - goto done; if (xpath_vec(xt, xpath?xpath:"/", &xvec, &xlen) < 0) goto done; - /* If vectors are specified then mark the nodes found and - * then filter out everything else, + /* If vectors are specified then mark the nodes found with all ancestors + * and filter out everything else, * otherwise return complete tree. */ - if (xvec != NULL){ + if (xvec != NULL) for (i=0; ith_dbs, db, &de0, sizeof(de0)); + } + xt = x1; + } + else{ + /* Remove everything that is not marked */ + if (!xml_flag(xt, XML_FLAG_MARK)) + if (xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL) < 0) + goto done; + } /* reset flag */ if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)XML_FLAG_MARK) < 0) goto done; @@ -362,8 +501,8 @@ text_get(xmldb_handle xh, /* Order XML children according to YANG */ if (xml_apply(xt, CX_ELMNT, xml_order, NULL) < 0) goto done; - -#if (XML_CHILD_HASH==1) +#ifdef XXX + /// (XML_CHILD_HASH==1) /* Add hash */ if (xml_apply0(xt, CX_ELMNT, xml_hash_op, (void*)1) < 0) goto done; @@ -433,7 +572,7 @@ text_modify(cxobj *x0, case OP_REPLACE: if (x0==NULL){ // int iamkey=0; - if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, y0)) == NULL) goto done; #if 0 /* If it is key I dont want to mark it */ @@ -445,24 +584,23 @@ text_modify(cxobj *x0, #endif xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */ if (x1bstr){ /* empty type does not have body */ - if ((x0b = xml_new("body", x0)) == NULL) + if ((x0b = xml_new("body", x0, NULL)) == NULL) goto done; xml_type_set(x0b, CX_BODY); } } if (x1bstr){ if ((x0b = xml_body_get(x0)) == NULL){ - if ((x0b = xml_new("body", x0)) == NULL) + if ((x0b = xml_new("body", x0, NULL)) == NULL) goto done; xml_type_set(x0b, CX_BODY); } if (xml_value_set(x0b, x1bstr) < 0) goto done; } -#if (XML_CHILD_HASH==1) - if (xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) + if (xml_child_hash && + xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) goto done; -#endif break; case OP_DELETE: if (x0==NULL){ @@ -500,22 +638,21 @@ text_modify(cxobj *x0, if (x0){ xml_purge(x0); } - if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, y0)) == NULL) goto done; if (xml_copy(x1, x0) < 0) goto done; break; } if (x0==NULL){ - if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, y0)) == NULL) goto done; if (op==OP_NONE) xml_flag_set(x0, XML_FLAG_NONE); /* Mark for potential deletion */ } -#if (XML_CHILD_HASH==1) - if (xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) + if (xml_child_hash && + xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) goto done; -#endif /* First pass: mark existing children in base */ /* Loop through children of the modification tree */ if ((x0vec = calloc(xml_child_nr(x1), sizeof(x1))) == NULL){ @@ -636,10 +773,6 @@ text_modify_top(cxobj *x0, /*! For containers without presence and no children, remove * @param[in] x XML tree node - * @note This should really be unnecessary since yspec should be set on creation - * @code - * xml_apply(xc, CX_ELMNT, xml_spec_populate, yspec) - * @endcode * See section 7.5.1 in rfc6020bis-02.txt: * No presence: * those that exist only for organizing the hierarchy of data nodes: @@ -692,11 +825,10 @@ text_put(xmldb_handle xh, cbuf *cb = NULL; yang_spec *yspec; cxobj *x0 = NULL; - - if (text_db2file(th, db, &dbfile) < 0) - goto done; - if (dbfile==NULL){ - clicon_err(OE_XML, 0, "dbfile NULL"); + struct db_element *de = NULL; + + if ((yspec = th->th_yangspec) == NULL){ + clicon_err(OE_YANG, ENOENT, "No yang spec"); goto done; } if (x1 && strcmp(xml_name(x1),"config")!=0){ @@ -704,29 +836,37 @@ text_put(xmldb_handle xh, xml_name(x1)); goto done; } - if ((yspec = th->th_yangspec) == NULL){ - clicon_err(OE_YANG, ENOENT, "No yang spec"); - goto done; + if (datastore_cache){ + if ((de = hash_value(th->th_dbs, db, NULL)) != NULL) + x0 = de->de_xml; } - if ((fd = open(dbfile, O_RDONLY)) < 0) { - clicon_err(OE_UNIX, errno, "open(%s)", dbfile); - goto done; - } - /* Parse file into XML tree */ - if ((clicon_xml_parse_file(fd, &x0, "")) < 0) - goto done; - /* Always assert a top-level called "config". - To ensure that, deal with two cases: - 1. File is empty -> rename top-level to "config" */ - if (xml_child_nr(x0) == 0){ - if (xml_name_set(x0, "config") < 0) - goto done; - } - /* 2. File is not empty ... -> replace root */ - else{ - /* There should only be one element and called config */ - if (singleconfigroot(x0, &x0) < 0) + if (x0 == NULL){ + if (text_db2file(th, db, &dbfile) < 0) goto done; + if (dbfile==NULL){ + clicon_err(OE_XML, 0, "dbfile NULL"); + goto done; + } + if ((fd = open(dbfile, O_RDONLY)) < 0) { + clicon_err(OE_UNIX, errno, "open(%s)", dbfile); + goto done; + } + /* Parse file into XML tree */ + if ((clicon_xml_parse_file(fd, "", yspec, &x0)) < 0) + goto done; + /* Always assert a top-level called "config". + To ensure that, deal with two cases: + 1. File is empty -> rename top-level to "config" */ + if (xml_child_nr(x0) == 0){ + if (xml_name_set(x0, "config") < 0) + goto done; + } + /* 2. File is not empty ... -> replace root */ + else{ + /* There should only be one element and called config */ + if (singleconfigroot(x0, &x0) < 0) + goto done; + } } /* Here x0 looks like: ... */ if (strcmp(xml_name(x0),"config")!=0){ @@ -736,18 +876,13 @@ text_put(xmldb_handle xh, } /* Add yang specification backpointer to all XML nodes */ - if (xml_apply(x0, CX_ELMNT, xml_spec_populate, yspec) < 0) - goto done; - - /* Add yang specification backpointer to all XML nodes */ + /* XXX: where is thiscreated? Add yspec */ if (xml_apply(x1, CX_ELMNT, xml_spec_populate, yspec) < 0) goto done; - -#if (XML_CHILD_HASH==1) /* Add hash */ - if (xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) + if (xml_child_hash && + xml_apply0(x0, CX_ELMNT, xml_hash_op, (void*)1) < 0) goto done; -#endif /* * Modify base tree x with modification x1 @@ -767,6 +902,18 @@ text_put(xmldb_handle xh, /* Remove (prune) nodes that are marked (non-presence containers w/o children) */ if (xml_tree_prune_flagged(x0, XML_FLAG_MARK, 1) < 0) goto done; + + /* Write back to datastore cache if first time */ + if (datastore_cache){ + struct db_element de0 = {0,}; + if (de != NULL) + de0 = *de; + if (de0.de_xml == NULL){ + de0.de_xml = x0; + hash_add(th->th_dbs, db, &de0, sizeof(de0)); + } + } + // output: /* Print out top-level xml tree after modification to file */ if ((cb = cbuf_new()) == NULL){ @@ -776,7 +923,16 @@ text_put(xmldb_handle xh, if (clicon_xml2cbuf(cb, x0, 0, 1) < 0) goto done; /* Reopen file in write mode */ - close(fd); + if (fd != -1) + close(fd); + if (dbfile == NULL){ + if (text_db2file(th, db, &dbfile) < 0) + goto done; + if (dbfile==NULL){ + clicon_err(OE_XML, 0, "dbfile NULL"); + goto done; + } + } if ((fd = open(dbfile, O_WRONLY | O_TRUNC, S_IRWXU)) < 0) { clicon_err(OE_UNIX, errno, "open(%s)", dbfile); goto done; @@ -793,7 +949,7 @@ text_put(xmldb_handle xh, close(fd); if (cb) cbuf_free(cb); - if (x0) + if (!datastore_cache && x0) xml_free(x0); return retval; } @@ -814,8 +970,18 @@ text_copy(xmldb_handle xh, struct text_handle *th = handle(xh); char *fromfile = NULL; char *tofile = NULL; + struct db_element *de = NULL; /* XXX lock */ + if (datastore_cache){ + /* Just invalidate xml if exists in TO */ + if ((de = hash_value(th->th_dbs, to, NULL)) != NULL){ + if (de->de_xml != NULL){ + xml_free(de->de_xml); + de->de_xml = NULL; + } + } + } if (text_db2file(th, from, &fromfile) < 0) goto done; if (text_db2file(th, to, &tofile) < 0) @@ -844,8 +1010,13 @@ text_lock(xmldb_handle xh, int pid) { struct text_handle *th = handle(xh); + struct db_element *de = NULL; + struct db_element de0 = {0,}; - hash_add(th->th_dbs, db, &pid, sizeof(pid)); + if ((de = hash_value(th->th_dbs, db, NULL)) != NULL) + de0 = *de; + de0.de_pid = pid; + hash_add(th->th_dbs, db, &de0, sizeof(de0)); clicon_debug(1, "%s: locked by %u", db, pid); return 0; } @@ -863,10 +1034,12 @@ text_unlock(xmldb_handle xh, const char *db) { struct text_handle *th = handle(xh); - int zero = 0; + struct db_element *de = NULL; - hash_add(th->th_dbs, db, &zero, sizeof(zero)); - // hash_del(th->th_dbs, db); + if ((de = hash_value(th->th_dbs, db, NULL)) != NULL){ + de->de_pid = 0; + hash_add(th->th_dbs, db, de, sizeof(*de)); + } return 0; } @@ -884,15 +1057,16 @@ text_unlock_all(xmldb_handle xh, char **keys = NULL; size_t klen; int i; - int *val; - size_t vlen; + struct db_element *de; if ((keys = hash_keys(th->th_dbs, &klen)) == NULL) return 0; for(i = 0; i < klen; i++) - if ((val = hash_value(th->th_dbs, keys[i], &vlen)) != NULL && - *val == pid) - hash_del(th->th_dbs, keys[i]); + if ((de = hash_value(th->th_dbs, keys[i], NULL)) != NULL && + de->de_pid == pid){ + de->de_pid = 0; + hash_add(th->th_dbs, keys[i], de, sizeof(*de)); + } if (keys) free(keys); return 0; @@ -910,13 +1084,11 @@ text_islocked(xmldb_handle xh, const char *db) { struct text_handle *th = handle(xh); - size_t vlen; - int *val; + struct db_element *de; - if ((val = hash_value(th->th_dbs, db, &vlen)) == NULL) + if ((de = hash_value(th->th_dbs, db, NULL)) == NULL) return 0; - return *val; - return 0; + return de->de_pid; } /*! Check if db exists @@ -1111,7 +1283,7 @@ main(int argc, char **argv) if (strcmp(cmd, "put")==0){ if (argc != 6) usage(argv[0]); - if (clicon_xml_parse_file(0, &xt, "") < 0) + if (clicon_xml_parse_file(0, "", NULL, &xt) < 0) goto done; if (xml_rootchild(xt, 0, &xn) < 0) goto done; diff --git a/datastore/text/clixon_xmldb_text.h b/datastore/text/clixon_xmldb_text.h index 36d33633..968af69a 100644 --- a/datastore/text/clixon_xmldb_text.h +++ b/datastore/text/clixon_xmldb_text.h @@ -49,6 +49,5 @@ int text_unlock_all(xmldb_handle h, int pid); int text_islocked(xmldb_handle h, const char *db); int text_exists(xmldb_handle h, const char *db); int text_delete(xmldb_handle h, const char *db); -int text_init(xmldb_handle h, const char *db); #endif /* _CLIXON_XMLDB_TEXT_H */ diff --git a/develop.md b/develop.md index a5b07413..c23f5314 100644 --- a/develop.md +++ b/develop.md @@ -3,6 +3,7 @@ 1. How to document the code 2. How to work in git (branching) 3. How the meta-configure stuff works +4. How to debug ## How to document the code @@ -47,3 +48,41 @@ configure.ac --. +--> config.status* -+ +--> make* Makefile.in ---' `-> Makefile ---' ``` + +## How to debug + +### Make your own simplified yang and configuration file. +``` + +cat < /tmp/my.yang +module mymodule{ + container x { + list y { + key "a"; + leaf a { + type string; + } + } + } +} +EOF +cat < /tmp/myconf.xml + + /tmp/myconf.xml + /usr/local/share/routing/yang + example + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + sudo clixon_backend -F -s init -f /tmp/myconf.xml -y /tmp/my.yang + ``` + +### Run valgrind and callgrind + ``` + valgrind --leak-check=full --show-leak-kinds=all clixon_netconf -qf /tmp/myconf.xml -y /tmp/my.yang + valgrind --tool=callgrind clixon_netconf -qf /tmp/myconf.xml -y /tmp/my.yang + sudo kcachegrind + ``` \ No newline at end of file diff --git a/example/routing_backend.c b/example/routing_backend.c index a0f23b0a..da3d3e53 100644 --- a/example/routing_backend.c +++ b/example/routing_backend.c @@ -169,7 +169,7 @@ plugin_statedata(clicon_handle h, "eth0" "eth" "42" - "", xstate) < 0) + "", NULL, xstate) < 0) goto done; retval = 0; done: @@ -227,7 +227,7 @@ plugin_reset(clicon_handle h, if (clicon_xml_parse_str("" "lolocal" - "", &xt) < 0) + "", NULL, &xt) < 0) goto done; /* Replace parent w fiorst child */ if (xml_rootchild(xt, 0, &xt) < 0) diff --git a/example/routing_cli.c b/example/routing_cli.c index 38bf2125..6a6c26e4 100644 --- a/example/routing_cli.c +++ b/example/routing_cli.c @@ -107,7 +107,7 @@ fib_route_rpc(clicon_handle h, /* User supplied variable in CLI command */ instance = cvec_find(cvv, "instance"); /* get a cligen variable from vector */ /* Create XML for fib-route netconf RPC */ - if (clicon_xml_parse(&xtop, "%s", instance) < 0) + if (clicon_xml_parse(&xtop, NULL, "%s", instance) < 0) goto done; /* Skip top-level */ xrpc = xml_child_i(xtop, 0); diff --git a/include/clixon_custom.h b/include/clixon_custom.h index 1c00ab83..c251d9a7 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -43,8 +43,6 @@ int strverscmp (__const char *__s1, __const char *__s2); #endif -/* Hash for XML trees list entries - * Experimental - */ -#define XML_CHILD_HASH 1 + + diff --git a/lib/clixon/clixon_xml.h b/lib/clixon/clixon_xml.h index 0ff7903f..541f6d8d 100644 --- a/lib/clixon/clixon_xml.h +++ b/lib/clixon/clixon_xml.h @@ -32,6 +32,8 @@ ***** END LICENSE BLOCK ***** * XML support functions. + * @see https://www.w3.org/TR/2008/REC-xml-20081126 + * https://www.w3.org/TR/2009/REC-xml-names-20091208/ */ #ifndef _CLIXON_XML_H #define _CLIXON_XML_H @@ -74,6 +76,11 @@ typedef int (xml_applyfn_t)(cxobj *yn, void *arg); #define XML_FLAG_CHANGE 0x08 /* Node is changed (commits) or child changed rec */ #define XML_FLAG_NONE 0x10 /* Node is added as NONE */ +/* Hash for XML trees list entries + * Experimental + */ +extern int xml_child_hash; + /* * Prototypes */ @@ -106,8 +113,7 @@ cxobj *xml_child_each(cxobj *xparent, cxobj *xprev, enum cxobj_type type); cxobj **xml_childvec_get(cxobj *x); int xml_childvec_set(cxobj *x, int len); -cxobj *xml_new(char *name, cxobj *xn_parent); -cxobj *xml_new_spec(char *name, cxobj *xn_parent, void *spec); +cxobj *xml_new(char *name, cxobj *xn_parent, void *spec); void *xml_spec(cxobj *x); void *xml_spec_set(cxobj *x, void *spec); cxobj *xml_find(cxobj *xn_parent, char *name); @@ -130,14 +136,15 @@ int xml_free(cxobj *xn); int xml_print(FILE *f, cxobj *xn); int clicon_xml2file(FILE *f, cxobj *xn, int level, int prettyprint); int clicon_xml2cbuf(cbuf *xf, cxobj *xn, int level, int prettyprint); -int clicon_xml_parse_file(int fd, cxobj **xml_top, char *endtag); +int clicon_xml_parse_file(int fd, char *endtag, yang_spec *yspec, cxobj **xt); +int clicon_xml_parse_str(char *str, yang_spec *yspec, cxobj **xml_top); /* XXX obsolete */ -#define clicon_xml_parse_string(str, x) clicon_xml_parse_str((*str), x) -int clicon_xml_parse_str(char *str, cxobj **xml_top); -int clicon_xml_parse(cxobj **cxtop, char *format, ...); -int xml_parse(char *str, cxobj *x_up); +#define clicon_xml_parse_string(str, x) clicon_xml_parse_str((*str), NULL, x) +int clicon_xml_parse(cxobj **cxtop, yang_spec *yspec, char *format, ...); +int xml_parse(char *str, yang_spec *yspec, cxobj *xt); int xmltree2cbuf(cbuf *cb, cxobj *x, int level); +int xml_copy_one(cxobj *xn0, cxobj *xn1); int xml_copy(cxobj *x0, cxobj *x1); cxobj *xml_dup(cxobj *x0); @@ -152,12 +159,13 @@ int xml_body_int32(cxobj *xb, int32_t *val); int xml_body_uint32(cxobj *xb, uint32_t *val); int xml_operation(char *opstr, enum operation_type *op); char *xml_operation2str(enum operation_type op); -#if (XML_CHILD_HASH==1) +int xml_child_spec(char *name, cxobj *xp, yang_spec *yspec, yang_stmt **yp); + clicon_hash_t *xml_hash(cxobj *x); -int xml_hash_init(cxobj *x); -int xml_hash_rm(cxobj *x); int xml_hash_key(cxobj *x, yang_stmt *y, cbuf *key); int xml_hash_op(cxobj *x, void *arg); -#endif +int xml_hash_add(cxobj *x); +int xml_hash_rm_only(cxobj *x); +int xml_hash_rm_entry(cxobj *x); #endif /* _CLIXON_XML_H */ diff --git a/lib/src/clixon_hash.c b/lib/src/clixon_hash.c index d7bb94de..2e50ca2d 100644 --- a/lib/src/clixon_hash.c +++ b/lib/src/clixon_hash.c @@ -235,8 +235,8 @@ hash_add(clicon_hash_t *hash, h = new; } - /* Make copy of lvalue */ - newval = malloc(align4(vlen+3)); /* XXX: qdbm needs aligned mallocs? */ + /* Make copy of value. aligned */ + newval = malloc(align4(vlen+3)); if (newval == NULL){ clicon_err(OE_UNIX, errno, "malloc: %s", strerror(errno)); goto catch; diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c index 265c80f8..a19070ee 100644 --- a/lib/src/clixon_json.c +++ b/lib/src/clixon_json.c @@ -54,6 +54,10 @@ /* clixon */ #include "clixon_err.h" #include "clixon_log.h" +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_json.h" #include "clixon_json_parse.h" @@ -465,7 +469,7 @@ xml2json_cbuf_vec(cbuf *cb, cxobj *xp = NULL; cxobj *xc; - if ((xp = xml_new("", NULL)) == NULL) + if ((xp = xml_new("", NULL, NULL)) == NULL) goto done; for (i=0; ijy_current)) == NULL) + if ((xn = xml_new(name, jy->jy_current, NULL)) == NULL) goto done; jy->jy_current = xn; retval = 0; @@ -207,7 +207,7 @@ json_current_body(struct clicon_json_yacc_arg *jy, cxobj *xn; clicon_debug(2, "%s", __FUNCTION__); - if ((xn = xml_new("body", jy->jy_current)) == NULL) + if ((xn = xml_new("body", jy->jy_current, NULL)) == NULL) goto done; xml_type_set(xn, CX_BODY); if (value && xml_value_append(xn, value)==NULL) diff --git a/lib/src/clixon_options.c b/lib/src/clixon_options.c index 0f78d541..14146f92 100644 --- a/lib/src/clixon_options.c +++ b/lib/src/clixon_options.c @@ -150,7 +150,7 @@ clicon_option_readfile_xml(clicon_hash_t *copt, } clicon_debug(2, "Reading config file %s", __FUNCTION__, filename); fd = fileno(f); - if (clicon_xml_parse_file(fd, &xt, "") < 0) + if (clicon_xml_parse_file(fd, "", yspec, &xt) < 0) goto done; if (xml_child_nr(xt)==1 && xml_child_nr_type(xt, CX_BODY)==1){ clicon_err(OE_CFG, 0, "Config file %s: Expected XML but is probably old sh style", filename); @@ -160,8 +160,6 @@ clicon_option_readfile_xml(clicon_hash_t *copt, clicon_err(OE_CFG, 0, "Config file %s: Lacks top-level \"config\" element", filename); goto done; } - if (xml_apply0(xc, CX_ELMNT, xml_spec_populate, yspec) < 0) - goto done; if (xml_apply0(xc, CX_ELMNT, xml_default, yspec) < 0) goto done; if (xml_apply0(xc, CX_ELMNT, xml_yang_validate_add, NULL) < 0) diff --git a/lib/src/clixon_proto.c b/lib/src/clixon_proto.c index fbd73cb9..2963c323 100644 --- a/lib/src/clixon_proto.c +++ b/lib/src/clixon_proto.c @@ -173,7 +173,7 @@ clicon_msg_decode(struct clicon_msg *msg, /* body */ xmlstr = msg->op_body; clicon_debug(1, "%s %s", __FUNCTION__, xmlstr); - if (clicon_xml_parse_str(xmlstr, xml) < 0) + if (clicon_xml_parse_str(xmlstr, NULL, xml) < 0) goto done; retval = 0; done: diff --git a/lib/src/clixon_proto_client.c b/lib/src/clixon_proto_client.c index 8ca60598..fb3b1d58 100644 --- a/lib/src/clixon_proto_client.c +++ b/lib/src/clixon_proto_client.c @@ -120,7 +120,7 @@ clicon_rpc_msg(clicon_handle h, } clicon_debug(1, "%s retdata:%s", __FUNCTION__, retdata); if (retdata && - clicon_xml_parse_str(retdata, &xret) < 0) + clicon_xml_parse_str(retdata, NULL, &xret) < 0) goto done; if (xret0){ *xret0 = xret; @@ -274,7 +274,7 @@ clicon_rpc_get_config(clicon_handle h, if ((xd = xpath_first(xret, "/rpc-reply/rpc-error")) != NULL) xd = xml_parent(xd); /* point to rpc-reply */ else if ((xd = xpath_first(xret, "/rpc-reply/data")) == NULL) - if ((xd = xml_new("data", NULL)) == NULL) + if ((xd = xml_new("data", NULL, NULL)) == NULL) goto done; if (xt){ if (xml_rm(xd) < 0) @@ -522,7 +522,7 @@ clicon_rpc_get(clicon_handle h, if ((xd = xpath_first(xret, "/rpc-reply/rpc-error")) != NULL) xd = xml_parent(xd); /* point to rpc-reply */ else if ((xd = xpath_first(xret, "/rpc-reply/data")) == NULL) - if ((xd = xml_new("data", NULL)) == NULL) + if ((xd = xml_new("data", NULL, NULL)) == NULL) goto done; if (xt){ if (xml_rm(xd) < 0) diff --git a/lib/src/clixon_xml.c b/lib/src/clixon_xml.c index 2149c409..baa809fe 100644 --- a/lib/src/clixon_xml.c +++ b/lib/src/clixon_xml.c @@ -32,6 +32,8 @@ ***** END LICENSE BLOCK ***** * XML support functions. + * @see https://www.w3.org/TR/2008/REC-xml-20081126 + * https://www.w3.org/TR/2009/REC-xml-names-20091208 */ #ifdef HAVE_CONFIG_H @@ -86,12 +88,10 @@ struct xml{ char *x_value; /* attribute and body nodes have values */ int _x_vector_i; /* internal use: xml_child_each */ int x_flags; /* Flags according to XML_FLAG_* above */ - void *x_spec; /* Pointer to specification, eg yang, by + yang_stmt *x_spec; /* Pointer to specification, eg yang, by reference, dont free */ cg_var *x_cv; /* If body this contains the typed value */ -#if (XML_CHILD_HASH==1) clicon_hash_t *x_hash; /* Hash of children */ -#endif }; /* Mapping between xml type <--> string */ @@ -103,6 +103,11 @@ static const map_str2int xsmap[] = { {NULL, -1} }; +/* Hash for XML trees list entries + * Experimental + */ +int xml_child_hash = 0; + /*! Translate from xml type in enum form to string keyword * @param[in] type Xml type * @retval str String keyword @@ -490,57 +495,44 @@ xml_childvec_get(cxobj *x) return x->x_childvec; } -/*! Create new xml node given a name and parent. Free it with xml_free(). +/*! Create new xml node given a name and parent. Free with xml_free(). * * @param[in] name Name of new * @param[in] xp The parent where the new xml node should be inserted - * @retval xml Created xml object if successful + * @param[in] spec Yang stmt or NULL. + * @retval xml Created xml object if successful. Free with xml_free() * @retval NULL Error and clicon_err() called * @code * cxobj *x; - * if ((x = xml_new(name, xparent)) == NULL) + * if ((x = xml_new(name, xparent, NULL)) == NULL) * err; + * ... + * xml_free(x); * @endcode - * @see xml_new_spec Also sets yang spec. + * @note yspec may be NULL either because it is not known or it is irrelevant, + * eg for body or attribute */ cxobj * xml_new(char *name, - cxobj *xp) + cxobj *xp, + void *spec) { - cxobj *xn; + cxobj *x; - if ((xn=malloc(sizeof(cxobj))) == NULL){ + if ((x = malloc(sizeof(cxobj))) == NULL){ clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__); return NULL; } - memset(xn, 0, sizeof(cxobj)); - if ((xml_name_set(xn, name)) < 0) + memset(x, 0, sizeof(cxobj)); + if ((xml_name_set(x, name)) < 0) return NULL; - xml_parent_set(xn, xp); - if (xp) - if (xml_child_append(xp, xn) < 0) - return NULL; - return xn; -} - -/*! Create new xml node given a name, parent and spec. - * @param[in] name Name of new xml node - * @param[in] xp XML parent - * @param[in] spec Yang spec - * @retval NULL Error - * @retval x XML tree. Free with xml_free(). - */ -cxobj * -xml_new_spec(char *name, - cxobj *xp, - void *spec) -{ - cxobj *x; - - if ((x = xml_new(name, xp)) == NULL) + xml_parent_set(x, xp); + if (xp && xml_child_append(xp, x) < 0) + return NULL; + x->x_spec = spec; /* Can be NULL */ + if (xml_child_hash && xml_hash_add(x) < 0) return NULL; - x->x_spec = spec; return x; } @@ -630,7 +622,7 @@ xml_insert(cxobj *xp, { cxobj *xc; /* new child */ - if ((xc = xml_new(tag, NULL)) == NULL) + if ((xc = xml_new(tag, NULL, NULL)) == NULL) goto catch; while (xp->x_childvec_len) if (xml_addsub(xc, xml_child_i(xp, 0)) < 0) @@ -658,9 +650,8 @@ xml_purge(cxobj *xc) int i; cxobj *xp; -#if (XML_CHILD_HASH==1) - xml_hash_op(xc, 0); -#endif + if (xml_child_hash) + xml_hash_rm_entry(xc); if ((xp = xml_parent(xc)) != NULL){ /* Find child order i in parent*/ for (i=0; ix_childvec[i] = NULL; } } -#if (XML_CHILD_HASH==1) if (x->x_hash) hash_free(x->x_hash); -#endif if (x->x_childvec) free(x->x_childvec); free(x); @@ -1061,22 +1050,29 @@ clicon_xml2cbuf(cbuf *cb, /*! Basic xml parsing function. * @param[in] str Pointer to string containing XML definition. + * @param[in] yspec Yang specification or NULL * @param[out] xtop Top of XML parse tree. Assume created. * @see clicon_xml_parse_file clicon_xml_parse_string */ int -xml_parse(char *str, - cxobj *xt) +xml_parse(char *str, + yang_spec *yspec, + cxobj *xt) { int retval = -1; struct xml_parse_yacc_arg ya = {0,}; + if (xt == NULL){ + clicon_err(OE_XML, errno, "Unexpected NULL XML"); + return -1; + } if ((ya.ya_parse_string = strdup(str)) == NULL){ - clicon_err(OE_XML, errno, "%s: strdup", __FUNCTION__); + clicon_err(OE_XML, errno, "strdup"); return -1; } ya.ya_xparent = xt; ya.ya_skipspace = 1; /* remove all non-terminal bodies (strip pretty-print) */ + ya.ya_yspec = yspec; if (clixon_xml_parsel_init(&ya) < 0) goto done; if (clixon_xml_parseparse(&ya) != 0) /* yacc returns 1 on error */ @@ -1128,8 +1124,7 @@ xmltree2cbuf(cbuf *cb, return 0; } -/* - * FSM to detect a substring +/*! FSM to detect a substring */ static inline int FSM(char *tag, @@ -1147,7 +1142,8 @@ FSM(char *tag, * @param[in] fd A file descriptor containing the XML file (as ASCII characters) * @param[out] xt Pointer to an (on entry empty) pointer to an XML parse tree * _created_ by this function. - * @param endtag Read until you encounter "endtag" in the stream + * @param[in] endtag Read until you encounter "endtag" in the stream, or NULL + * @param[in] yspec Yang specification, or NULL * @retval 0 OK * @retval -1 Error with clicon_err called * @@ -1165,9 +1161,10 @@ FSM(char *tag, * May block */ int -clicon_xml_parse_file(int fd, - cxobj **cx, - char *endtag) +clicon_xml_parse_file(int fd, + char *endtag, + yang_spec *yspec, + cxobj **xt) { int retval = -1; int ret; @@ -1176,15 +1173,13 @@ clicon_xml_parse_file(int fd, char *xmlbuf = NULL; char *ptr; int maxbuf = BUFLEN; - int endtaglen = strlen(endtag); + int endtaglen = 0; int state = 0; int oldmaxbuf; - if (endtag == NULL){ - clicon_err(OE_XML, 0, "%s: endtag required\n", __FUNCTION__); - goto done; - } - *cx = NULL; + if (endtag != NULL) + endtaglen = strlen(endtag); + *xt = NULL; if ((xmlbuf = malloc(maxbuf)) == NULL){ clicon_err(OE_XML, errno, "%s: malloc", __FUNCTION__); goto done; @@ -1199,16 +1194,17 @@ clicon_xml_parse_file(int fd, break; } if (ret != 0){ - state = FSM(endtag, ch, state); + if (endtag) + state = FSM(endtag, ch, state); xmlbuf[len++] = ch; } - if (ret == 0 || state == endtaglen){ + if (ret == 0 || + (endtag && (state == endtaglen))){ state = 0; - if ((*cx = xml_new("top", NULL)) == NULL) - break; - if (xml_parse(ptr, *cx) < 0){ + if ((*xt = xml_new("top", NULL, NULL)) == NULL) + break; + if (xml_parse(ptr, yspec, *xt) < 0) goto done; - } break; } if (len>=maxbuf-1){ /* Space: one for the null character */ @@ -1224,14 +1220,13 @@ clicon_xml_parse_file(int fd, } /* while */ retval = 0; done: - if (retval < 0 && *cx){ - free(*cx); - *cx = NULL; + if (retval < 0 && *xt){ + free(*xt); + *xt = NULL; } if (xmlbuf) free(xmlbuf); return retval; - // return (*cx)?0:-1; } /*! Read an XML definition from string and parse it into a parse-tree. @@ -1252,12 +1247,13 @@ clicon_xml_parse_file(int fd, * @note you need to free the xml parse tree after use, using xml_free() */ int -clicon_xml_parse_str(char *str, - cxobj **cxtop) +clicon_xml_parse_str(char *str, + yang_spec *yspec, + cxobj **cxtop) { - if ((*cxtop = xml_new("top", NULL)) == NULL) - return -1; - return xml_parse(str, *cxtop); + if ((*cxtop = xml_new("top", NULL, NULL)) == NULL) + return -1; + return xml_parse(str, yspec, *cxtop); } @@ -1266,6 +1262,7 @@ clicon_xml_parse_str(char *str, * Utility function using stdarg instead of static string. * @param[out] xml_top Top of XML parse tree. Will add extra top element called 'top'. * you must free it after use, using xml_free() + * @param[in] yspec Yang specification, or NULL * @param[in] format Pointer to string containing XML definition. * @retval 0 OK @@ -1281,8 +1278,9 @@ clicon_xml_parse_str(char *str, * @note you need to free the xml parse tree after use, using xml_free() */ int -clicon_xml_parse(cxobj **cxtop, - char *format, ...) +clicon_xml_parse(cxobj **cxtop, + yang_spec *yspec, + char *format, ...) { int retval = -1; va_list args; @@ -1300,9 +1298,9 @@ clicon_xml_parse(cxobj **cxtop, va_start(args, format); len = vsnprintf(str, len, format, args) + 1; va_end(args); - if ((*cxtop = xml_new("top", NULL)) == NULL) + if ((*cxtop = xml_new("top", NULL, NULL)) == NULL) return -1; - if (xml_parse(str, *cxtop) < 0) + if (xml_parse(str, yspec, *cxtop) < 0) goto done; retval = 0; done: @@ -1313,28 +1311,28 @@ clicon_xml_parse(cxobj **cxtop, /*! Copy single xml node without copying children */ -static int -copy_one(cxobj *xn0, - cxobj *xn1) +int +xml_copy_one(cxobj *x0, + cxobj *x1) { cg_var *cv1; - xml_type_set(xn1, xml_type(xn0)); - if (xml_value(xn0)){ /* malloced string */ - if ((xn1->x_value = strdup(xn0->x_value)) == NULL){ + xml_type_set(x1, xml_type(x0)); + if (xml_value(x0)){ /* malloced string */ + if ((x1->x_value = strdup(x0->x_value)) == NULL){ clicon_err(OE_XML, errno, "%s: strdup", __FUNCTION__); return -1; } } - if (xml_name(xn0)) /* malloced string */ - if ((xml_name_set(xn1, xml_name(xn0))) < 0) + if (xml_name(x0)) /* malloced string */ + if ((xml_name_set(x1, xml_name(x0))) < 0) return -1; - if (xml_cv_get(xn0)){ - if ((cv1 = cv_dup(xml_cv_get(xn0))) == NULL){ + if (xml_cv_get(x0)){ + if ((cv1 = cv_dup(xml_cv_get(x0))) == NULL){ clicon_err(OE_XML, errno, "%s: cv_dup", __FUNCTION__); return -1; } - if ((xml_cv_set(xn1, cv1)) < 0) + if ((xml_cv_set(x1, cv1)) < 0) return -1; } return 0; @@ -1345,8 +1343,9 @@ copy_one(cxobj *xn0, * x1 should be a created placeholder. If x1 is non-empty, * the copied tree is appended to the existing tree. * @code - * x1 = xml_new("new", xparent); - * xml_copy(x0, x1); + * x1 = xml_new("new", xparent, NULL); + * if (xml_copy(x0, x1) < 0) + * err; * @endcode */ int @@ -1357,11 +1356,11 @@ xml_copy(cxobj *x0, cxobj *x; cxobj *xcopy; - if (copy_one(x0, x1) <0) + if (xml_copy_one(x0, x1) <0) goto done; x = NULL; while ((x = xml_child_each(x0, x, -1)) != NULL) { - if ((xcopy = xml_new(xml_name(x), x1)) == NULL) + if ((xcopy = xml_new(xml_name(x), x1, xml_spec(x))) == NULL) goto done; if (xml_copy(x, xcopy) < 0) /* recursion */ goto done; @@ -1384,7 +1383,7 @@ xml_dup(cxobj *x0) { cxobj *x1; - if ((x1 = xml_new("new", NULL)) == NULL) + if ((x1 = xml_new("new", NULL, xml_spec(x0))) == NULL) return NULL; if (xml_copy(x0, x1) < 0) return NULL; @@ -1720,7 +1719,32 @@ xml_operation2str(enum operation_type op) } } -#if (XML_CHILD_HASH==1) +/*! Given an XML object and a child name, return yang stmt of child + * If no xml parent, find root yang stmt matching name + * @param[in] name Name of child + * @param[in] xp XML parent, can be NULL. + * @param[in] yspec Yang specification (top level) + * @param[out] yresult Pointer to yang stmt of result, or NULL, if not found + */ +int +xml_child_spec(char *name, + cxobj *xp, + yang_spec *yspec, + yang_stmt **yresult) +{ + yang_stmt *y; /* result yang node */ + yang_stmt *yparent; /* parent yang */ + + if (xp && (yparent = xml_spec(xp)) != NULL) + y = yang_find_datanode((yang_node*)yparent, name); + else if (yspec) + y = yang_find_topnode(yspec, name, 0); /* still NULL for config */ + else + y = NULL; + *yresult = y; + return 0; +} + /*! Return yang hash * Not necessarily set. Either has not been set yet (by xml_spec_set( or anyxml. */ @@ -1730,7 +1754,7 @@ xml_hash(cxobj *x) return x->x_hash; } -int +static int xml_hash_init(cxobj *x) { if ((x->x_hash = hash_init()) < 0) @@ -1738,8 +1762,12 @@ xml_hash_init(cxobj *x) return 0; } +/*! XML remove hash only. Not in parent. + * @param[in] x XML object + * eg same as xml_hash_op(x, -1) + */ int -xml_hash_rm(cxobj *x) +xml_hash_rm_only(cxobj *x) { if (x && x->x_hash){ hash_free(x->x_hash); @@ -1819,7 +1847,7 @@ xml_hash_key(cxobj *x, /*! XML hash add. Create hash and add key/value to parent * - * @param[in] arg -1: rm only hash 0: rm entry, 1: add + * @param[in] arg 0: rm entry, 1: add * Typically called for a whole tree. */ int @@ -1837,10 +1865,8 @@ xml_hash_op(cxobj *x, if (op==1) xml_hash_init(x); } - else if (op==-1|| op==0) - xml_hash_rm(x); - if (op==-1) - goto ok; + else if (op==0) + xml_hash_rm_only(x); if ((xp = xml_parent(x)) == NULL) goto ok; if ((ph = xml_hash(xp))==NULL) @@ -1871,8 +1897,85 @@ xml_hash_op(cxobj *x, return retval; } -#endif +/*! XML hash add. Create hash and add key/value to parent + * + * @param[in] x XML object + * eg same as xml_hash_op(x, 1) + */ +int +xml_hash_add(cxobj *x) +{ + int retval = -1; + cxobj *xp; + clicon_hash_t *ph; + yang_stmt *y; + cbuf *key = NULL; /* cligen buffer hash key */ + if ((ph = xml_hash(x))==NULL){ + xml_hash_init(x); + ph = xml_hash(x); + } + if ((xp = xml_parent(x)) == NULL) + goto ok; + if ((y = xml_spec(x)) == NULL) + goto ok; + if ((key = cbuf_new()) == NULL){ + clicon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + if (xml_hash_key(x, y, key) < 0) + goto done; + if (cbuf_len(key)){ + if (hash_add(ph, cbuf_get(key), &x, sizeof(x)) == NULL) + goto done; + } + ok: + retval = 0; + done: + if (key) + cbuf_free(key); + return retval; +} + +/*! XML hash rm. Create hash and add key/value to parent + * + * @param[in] x XML object + * eg same as xml_hash_op(x, 0) + */ +int +xml_hash_rm_entry(cxobj *x) +{ + int retval = -1; + cxobj *xp; + clicon_hash_t *ph; + yang_stmt *y; + cbuf *key = NULL; /* cligen buffer hash key */ + + if (xml_hash(x)!=NULL) + xml_hash_rm_only(x); + if ((xp = xml_parent(x)) == NULL) + goto ok; + if ((ph = xml_hash(xp))==NULL) + goto ok; + if ((y = xml_spec(x)) == NULL) + goto ok; + if ((key = cbuf_new()) == NULL){ + clicon_err(OE_XML, errno, "cbuf_new"); + goto done; + } + if (xml_hash_key(x, y, key) < 0) + goto done; + if (cbuf_len(key)){ + if (hash_del(ph, cbuf_get(key)) < 0) + goto done; + } + ok: + retval = 0; + done: + if (key) + cbuf_free(key); + return retval; +} /* * Turn this on to get a xml parse and pretty print test program diff --git a/lib/src/clixon_xml_db.c b/lib/src/clixon_xml_db.c index 7e85f44c..6ff4c90a 100644 --- a/lib/src/clixon_xml_db.c +++ b/lib/src/clixon_xml_db.c @@ -56,8 +56,8 @@ #include "clixon_queue.h" #include "clixon_hash.h" #include "clixon_handle.h" -#include "clixon_xml.h" #include "clixon_yang.h" +#include "clixon_xml.h" #include "clixon_plugin.h" #include "clixon_options.h" #include "clixon_xml_db.h" diff --git a/lib/src/clixon_xml_map.c b/lib/src/clixon_xml_map.c index bce2968f..58ad338f 100644 --- a/lib/src/clixon_xml_map.c +++ b/lib/src/clixon_xml_map.c @@ -561,7 +561,7 @@ cvec2xml_1(cvec *cvv, cv = NULL; while ((cv = cvec_each(cvv, cv)) != NULL) len++; - if ((xt = xml_new(toptag, xp)) == NULL) + if ((xt = xml_new(toptag, xp, NULL)) == NULL) goto err; if (xml_childvec_set(xt, len) < 0) goto err; @@ -570,11 +570,11 @@ cvec2xml_1(cvec *cvv, while ((cv = cvec_each(cvv, cv)) != NULL) { if (cv_type_get(cv)==CGV_ERR || cv_name_get(cv) == NULL) continue; - if ((xn = xml_new(cv_name_get(cv), NULL)) == NULL) /* this leaks */ + if ((xn = xml_new(cv_name_get(cv), NULL, NULL)) == NULL) /* this leaks */ goto err; xml_parent_set(xn, xt); xml_child_i_set(xt, i++, xn); - if ((xb = xml_new("body", xn)) == NULL) /* this leaks */ + if ((xb = xml_new("body", xn, NULL)) == NULL) /* this leaks */ goto err; xml_type_set(xb, CX_BODY); val = cv2str_dup(cv); @@ -620,35 +620,35 @@ match_base_child(cxobj *x0, int equal; char **b1vec = NULL; int i; -#if (XML_CHILD_HASH==1) cxobj **p; cbuf *key = NULL; /* cligen buffer hash key */ size_t vlen; - *x0cp = NULL; /* return value */ - if (xml_hash(x0) == NULL) - goto nohash; - if ((key = cbuf_new()) == NULL){ - clicon_err(OE_XML, errno, "cbuf_new"); - goto done1; - } - if (xml_hash_key(x1c, yc, key) < 0) - goto done; - x0c = NULL; - if (cbuf_len(key)) - if ((p = hash_value(xml_hash(x0), cbuf_get(key), &vlen)) != NULL){ - assert(vlen == sizeof(x0c)); - x0c = *p; + if (xml_child_hash){ + *x0cp = NULL; /* return value */ + if (xml_hash(x0) == NULL) + goto nohash; + if ((key = cbuf_new()) == NULL){ + clicon_err(OE_XML, errno, "cbuf_new"); + goto done1; } - // fprintf(stderr, "%s get %s = 0x%x\n", __FUNCTION__, cbuf_get(key), (unsigned int)x0c); - *x0cp = x0c; - retval = 0; - done1: - if (key) - cbuf_free(key); - return retval; + if (xml_hash_key(x1c, yc, key) < 0) + goto done; + x0c = NULL; + if (cbuf_len(key)) + if ((p = hash_value(xml_hash(x0), cbuf_get(key), &vlen)) != NULL){ + assert(vlen == sizeof(x0c)); + x0c = *p; + } + // fprintf(stderr, "%s get %s = 0x%x\n", __FUNCTION__, cbuf_get(key), (unsigned int)x0c); + *x0cp = x0c; + retval = 0; + done1: + if (key) + cbuf_free(key); + return retval; + } nohash: -#endif /* XML_CHILD_HASH */ *x0cp = NULL; /* return value */ x1cname = xml_name(x1c); switch (yc->ys_keyword){ @@ -1328,9 +1328,9 @@ xml_default(cxobj *xt, assert(y->ys_cv); if (!cv_flag(y->ys_cv, V_UNSET)){ /* Default value exists */ if (!xml_find(xt, y->ys_argument)){ - if ((xc = xml_new_spec(y->ys_argument, xt, y)) == NULL) + if ((xc = xml_new(y->ys_argument, xt, y)) == NULL) goto done; - if ((xb = xml_new("body", xc)) == NULL) + if ((xb = xml_new("body", xc, NULL)) == NULL) goto done; xml_type_set(xb, CX_BODY); if ((str = cv2str_dup(y->ys_cv)) == NULL){ @@ -1458,6 +1458,7 @@ xml_non_config_data(cxobj *xt, return retval; } + /*! Add yang specification backpoint to XML node * @param[in] xt XML tree node * @param[in] arg Yang spec @@ -1474,21 +1475,21 @@ xml_spec_populate(cxobj *x, { int retval = -1; yang_spec *yspec = (yang_spec*)arg; - char *name; yang_stmt *y=NULL; /* yang node */ - cxobj *xp; /* xml parent */ - yang_stmt *yp; /* parent yang */ - name = xml_name(x); + if (xml_child_spec(xml_name(x), xml_parent(x), yspec, &y) < 0) + goto done; +#if 0 if ((xp = xml_parent(x)) != NULL && (yp = xml_spec(xp)) != NULL) y = yang_find_datanode((yang_node*)yp, xml_name(x)); else y = yang_find_topnode(yspec, name, 0); /* still NULL for config */ +#endif if (y) xml_spec_set(x, y); retval = 0; - // done: + done: return retval; } @@ -1700,10 +1701,10 @@ api_path2xml_vec(char **vec, clicon_err(OE_XML, 0, "malformed key, expected '='"); goto done; } - if ((x = xml_new_spec(y->ys_argument, x0, y)) == NULL) + if ((x = xml_new(y->ys_argument, x0, y)) == NULL) goto done; xml_type_set(x, CX_ELMNT); - if ((xb = xml_new("body", x)) == NULL) + if ((xb = xml_new("body", x, NULL)) == NULL) goto done; xml_type_set(xb, CX_BODY); if (restval && xml_value_set(xb, restval) < 0) @@ -1738,18 +1739,17 @@ api_path2xml_vec(char **vec, } cvi = NULL; /* create list object */ - if ((x = xml_new_spec(name, x0, y)) == NULL) + if ((x = xml_new(name, x0, y)) == NULL) goto done; xml_type_set(x, CX_ELMNT); j = 0; /* Create keys */ while ((cvi = cvec_each(cvk, cvi)) != NULL) { keyname = cv_string_get(cvi); - - if ((xn = xml_new(keyname, x)) == NULL) + if ((xn = xml_new(keyname, x, NULL)) == NULL) goto done; xml_type_set(xn, CX_ELMNT); - if ((xb = xml_new("body", xn)) == NULL) + if ((xb = xml_new("body", xn, NULL)) == NULL) goto done; xml_type_set(xb, CX_BODY); val2 = valvec?valvec[j++]:NULL; @@ -1762,7 +1762,7 @@ api_path2xml_vec(char **vec, } break; default: /* eg Y_CONTAINER, Y_LEAF */ - if ((x = xml_new_spec(name, x0, y)) == NULL) + if ((x = xml_new(name, x0, y)) == NULL) goto done; xml_type_set(x, CX_ELMNT); break; @@ -1858,17 +1858,17 @@ xml_merge1(cxobj *x0, if (y0->yn_keyword == Y_LEAF_LIST || y0->yn_keyword == Y_LEAF){ x1bstr = xml_body(x1); if (x0==NULL){ - if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, y0)) == NULL) goto done; if (x1bstr){ /* empty type does not have body */ - if ((x0b = xml_new("body", x0)) == NULL) + if ((x0b = xml_new("body", x0, NULL)) == NULL) goto done; xml_type_set(x0b, CX_BODY); } } if (x1bstr){ if ((x0b = xml_body_get(x0)) == NULL){ - if ((x0b = xml_new("body", x0)) == NULL) + if ((x0b = xml_new("body", x0, NULL)) == NULL) goto done; xml_type_set(x0b, CX_BODY); } @@ -1879,7 +1879,7 @@ xml_merge1(cxobj *x0, } /* if LEAF|LEAF_LIST */ else { /* eg Y_CONTAINER, Y_LIST */ if (x0==NULL){ - if ((x0 = xml_new_spec(x1name, x0p, y0)) == NULL) + if ((x0 = xml_new(x1name, x0p, y0)) == NULL) goto done; } /* Loop through children of the modification tree */ @@ -1992,6 +1992,7 @@ done: return retval; } + /* * Turn this on for uni-test programs * Usage: clixon_string join diff --git a/lib/src/clixon_xml_parse.h b/lib/src/clixon_xml_parse.h index 848bf55e..11ae12ea 100644 --- a/lib/src/clixon_xml_parse.h +++ b/lib/src/clixon_xml_parse.h @@ -32,6 +32,8 @@ ***** END LICENSE BLOCK ***** * XML parser + * @see https://www.w3.org/TR/2008/REC-xml-20081126 + * https://www.w3.org/TR/2009/REC-xml-names-20091208 */ #ifndef _CLIXON_XML_PARSE_H_ #define _CLIXON_XML_PARSE_H_ @@ -39,6 +41,7 @@ /* * Types */ +/*! XML parser yacc handler struct */ struct xml_parse_yacc_arg{ char *ya_parse_string; /* original (copy of) parse string */ int ya_linenum; /* Number of \n in parsed buffer */ @@ -46,7 +49,8 @@ struct xml_parse_yacc_arg{ cxobj *ya_xelement; /* xml active element */ cxobj *ya_xparent; /* xml parent element*/ - int ya_skipspace; /* If set, remove all non-terminal bodies (strip prettyr-print) */ + int ya_skipspace; /* If set, remove all non-terminal bodies (strip pretty-print) */ + yang_spec *ya_yspec; /* If set, top-level yang-spec */ }; extern char *clixon_xml_parsetext; diff --git a/lib/src/clixon_xml_parse.l b/lib/src/clixon_xml_parse.l index b52520e0..340a4225 100644 --- a/lib/src/clixon_xml_parse.l +++ b/lib/src/clixon_xml_parse.l @@ -32,6 +32,8 @@ ***** END LICENSE BLOCK ***** * XML parser + * @see https://www.w3.org/TR/2008/REC-xml-20081126 + * https://www.w3.org/TR/2009/REC-xml-names-20091208 */ %{ diff --git a/lib/src/clixon_xml_parse.y b/lib/src/clixon_xml_parse.y index a1051e8d..c97d594e 100644 --- a/lib/src/clixon_xml_parse.y +++ b/lib/src/clixon_xml_parse.y @@ -32,6 +32,8 @@ ***** END LICENSE BLOCK ***** * XML parser + * @see https://www.w3.org/TR/2008/REC-xml-20081126 + * https://www.w3.org/TR/2009/REC-xml-names-20091208 */ %union { char *string; @@ -46,7 +48,7 @@ %token BCOMMENT ECOMMENT -%type val aid +%type attvalue attqname %lex-param {void *_ya} /* Add this argument to parse() and lex() function */ %parse-param {void *_ya} @@ -68,6 +70,10 @@ /* clicon */ #include "clixon_err.h" #include "clixon_log.h" +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_xml_parse.h" @@ -79,8 +85,9 @@ clixon_xml_parseerror(void *_ya, char *s) return; } -/* note that we dont handle escaped characters correctly - there may also be some leakage here on NULL return +/* + * Note that we dont handle escaped characters correctly + * there may also be some leakage here on NULL return */ static int xml_parse_content(struct xml_parse_yacc_arg *ya, @@ -92,7 +99,7 @@ xml_parse_content(struct xml_parse_yacc_arg *ya, ya->ya_xelement = NULL; /* init */ if (xn == NULL){ - if ((xn = xml_new("body", xp)) == NULL) + if ((xn = xml_new("body", xp, NULL)) == NULL) goto done; xml_type_set(xn, CX_BODY); } @@ -105,7 +112,8 @@ xml_parse_content(struct xml_parse_yacc_arg *ya, } static int -xml_parse_version(struct xml_parse_yacc_arg *ya, char *ver) +xml_parse_version(struct xml_parse_yacc_arg *ya, + char *ver) { if(strcmp(ver, "1.0")){ clicon_err(OE_XML, errno, "Wrong XML version %s expected 1.0\n", ver); @@ -116,20 +124,35 @@ xml_parse_version(struct xml_parse_yacc_arg *ya, char *ver) return 0; } +/*! Parse Qualified name + * @param[in] ya XML parser yacc handler struct + * @param[in] prefix Prefix, namespace, or NULL + * @param[in] localpart Name + */ static int -xml_parse_id(struct xml_parse_yacc_arg *ya, char *name, char *namespace) +xml_parse_qname(struct xml_parse_yacc_arg *ya, + char *prefix, + char *name) { - if ((ya->ya_xelement=xml_new(name, ya->ya_xparent)) == NULL) { - if (namespace) - free(namespace); - free(name); - return -1; - } - xml_namespace_set(ya->ya_xelement, namespace); - if (namespace) - free(namespace); + int retval = -1; + cxobj *x; + yang_stmt *y=NULL; /* yang node */ + cxobj *xp; /* xml parent */ + + xp = ya->ya_xparent; + if (xml_child_spec(name, xp, ya->ya_yspec, &y) < 0) + goto done; + if ((x = xml_new(name, xp, y)) == NULL) + goto done; + if (xml_namespace_set(x, prefix) < 0) + goto done; + ya->ya_xelement = x; + retval = 0; + done: + if (prefix) + free(prefix); free(name); - return 0; + return retval; } static int @@ -250,39 +273,46 @@ xml_parse_bslash2(struct xml_parse_yacc_arg *ya, return retval; } +static int +xml_parse_attr(struct xml_parse_yacc_arg *ya, + char *qname, + char *attval) +{ + int retval = -1; + cxobj *xa; + if ((xa = xml_new(qname, ya->ya_xelement, NULL)) == NULL) + goto done; + xml_type_set(xa, CX_ATTR); + if (xml_value_set(xa, attval) < 0) + goto done; + retval = 0; + done: + free(qname); + free(attval); + return retval; +} + +/*! Parse Attribue Qualified name, Just transform prefix:name into a new string + * + */ static char* -xml_parse_ida(struct xml_parse_yacc_arg *ya, char *namespace, char *name) +xml_merge_attqname(struct xml_parse_yacc_arg *ya, + char *prefix, + char *name) { char *str; - int len = strlen(namespace)+strlen(name)+2; + int len = strlen(prefix)+strlen(name)+2; if ((str=malloc(len)) == NULL) return NULL; - snprintf(str, len, "%s:%s", namespace, name); - free(namespace); + snprintf(str, len, "%s:%s", prefix, name); + free(prefix); free(name); return str; } -static int -xml_parse_attr(struct xml_parse_yacc_arg *ya, char *id, char *val) -{ - int retval = -1; - cxobj *xa; - - if ((xa = xml_new(id, ya->ya_xelement)) == NULL) - goto done; - xml_type_set(xa, CX_ATTR); - if (xml_value_set(xa, val) < 0) - goto done; - retval = 0; - done: - free(id); - free(val); - return retval; -} - + %} %% @@ -309,22 +339,22 @@ encode : ENC '=' '\"' CHAR '\"' {free($4);} | ENC '=' '\'' CHAR '\'' {free($4);} ; -emnt : '<' id attrs emnt1 - { clicon_debug(3, "emnt -> < id attrs emnt1"); } +element : '<' qname attrs element1 + { clicon_debug(3, "element -> < qname attrs element1"); } ; -id : NAME { if (xml_parse_id(_YA, $1, NULL) < 0) YYABORT; - clicon_debug(3, "id -> NAME %s", $1);} - | NAME ':' NAME { if (xml_parse_id(_YA, $3, $1) < 0) YYABORT; - clicon_debug(3, "id -> NAME : NAME");} +qname : NAME { if (xml_parse_qname(_YA, NULL, $1) < 0) YYABORT; + clicon_debug(3, "qname -> NAME %s", $1);} + | NAME ':' NAME { if (xml_parse_qname(_YA, $1, $3) < 0) YYABORT; + clicon_debug(3, "qname -> NAME : NAME");} ; -emnt1 : ESLASH {_YA->ya_xelement = NULL; - clicon_debug(3, "emnt1 -> />");} - | '>' { xml_parse_endslash_pre(_YA); } - list { xml_parse_endslash_mid(_YA); } +element1 : ESLASH {_YA->ya_xelement = NULL; + clicon_debug(3, "element1 -> />");} + | '>' { xml_parse_endslash_pre(_YA); } + list { xml_parse_endslash_mid(_YA); } etg { xml_parse_endslash_post(_YA); - clicon_debug(3, "emnt1 -> > list etg");} + clicon_debug(3, "element1 -> > list etg");} ; etg : BSLASH NAME '>' @@ -339,7 +369,7 @@ list : list content { clicon_debug(3, "list -> list content"); } | content { clicon_debug(3, "list -> content"); } ; -content : emnt { clicon_debug(3, "content -> emnt"); } +content : element { clicon_debug(3, "content -> element"); } | comment { clicon_debug(3, "content -> comment"); } | CHAR { if (xml_parse_content(_YA, $1) < 0) YYABORT; clicon_debug(3, "content -> CHAR %s", $1); } @@ -350,20 +380,20 @@ comment : BCOMMENT ECOMMENT ; -attrs : attrs att +attrs : attrs attr | ; +attr : attqname '=' attvalue { if (xml_parse_attr(_YA, $1, $3) < 0) YYABORT; } + ; -aid : NAME {$$ = $1;} +attqname : NAME {$$ = $1;} | NAME ':' NAME - { if (($$ = xml_parse_ida(_YA, $1, $3)) == NULL) YYABORT; } + { if (($$ = xml_merge_attqname(_YA, $1, $3)) == NULL) YYABORT; } ; -att : aid '=' val { if (xml_parse_attr(_YA, $1, $3) < 0) YYABORT; } - ; -val : '\"' CHAR '\"' { $$=$2; /* $2 must be consumed */} +attvalue : '\"' CHAR '\"' { $$=$2; /* $2 must be consumed */} | '\"' '\"' { $$=strdup(""); /* $2 must be consumed */} ; diff --git a/lib/src/clixon_xsl.c b/lib/src/clixon_xsl.c index d7731de2..2a67de00 100644 --- a/lib/src/clixon_xsl.c +++ b/lib/src/clixon_xsl.c @@ -102,6 +102,10 @@ in #include "clixon_err.h" #include "clixon_log.h" #include "clixon_string.h" +#include "clixon_queue.h" +#include "clixon_hash.h" +#include "clixon_handle.h" +#include "clixon_yang.h" #include "clixon_xml.h" #include "clixon_xsl.h" diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index c6ee54ef..5e609d13 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -2077,7 +2077,7 @@ yang_spec_main(clicon_handle h, * ...cv_string_get(cv); * cvec_free(cvv); * @endcode - * Note: must free return value after use w cvec_free + * @note must free return value after use w cvec_free */ cvec * yang_arg2cvec(yang_stmt *ys, diff --git a/test/test_datastore_scaling.sh b/test/test_datastore_scaling.sh new file mode 100755 index 00000000..ca1f739c --- /dev/null +++ b/test/test_datastore_scaling.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Scaling test + +if [ $# = 0 ]; then + number=1000 + req=10 +elif [ $# = 1 ]; then + number=$1 + req=10 +elif [ $# = 2 ]; then + number=$1 + req=$2 +else + echo "Usage: $0 [ []]" + exit 1 +fi +rnd=$(( ( RANDOM % $number ) )) + +fyang=/tmp/scaling.yang +db=/tmp/text/candidate_db +name=text +dir=/tmp/text +conf="-d candidate -b $dir -p ../datastore/$name/$name.so -y /tmp -m ietf-ip" + +# include err() and new() functions +. ./lib.sh +clixon_cf=/tmp/scaling-conf.xml + + +# For memcheck +# clixon_netconf="valgrind --leak-check=full --show-leak-kinds=all clixon_netconf" +# clixon_netconf="valgrind --tool=callgrind clixon_netconf +clixon_netconf=clixon_netconf + + +cat < $fyang +module example{ + container x { + list y { + key "a"; + leaf a { + type string; + } + leaf b { + type string; + } + } + leaf-list c { + type string; + } + } +} +EOF + +cat < $clixon_cf + + /tmp/test_yang.xml + /usr/local/share/routing/yang + example + /usr/local/var/routing/routing.sock + /usr/local/var/routing/routing.pidfile + /usr/local/var/routing + /usr/local/lib/xmldb/text.so + +EOF + +if [ ! -d $dir ]; then + mkdir $dir +fi + +echo "datastore_client $conf mget $req /x/y[a=$rnd][b=$rnd]" + +new "generate large list config" +echo -n "" > $db +for (( i=0; i<$number; i++ )); do + echo -n "$i$i" >> $db +done +echo "" >> $db + +new "datastore_client $name init" +expectfn "datastore_client $conf init" "" + +new "datastore $name mget" +expectfn "datastore_client $conf mget 1 /x/y[a=$rnd][b=$rnd]" "^$rnd$rnd$" + +new "make $req gets" +time datastore_client $conf mget $req "/x/y[a=$rnd][b=$rnd]" > /dev/null +