experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* New "general-purpose" datastore upgrade callback added which i called once on startup, intended for lo

Browse source 
w-level general upgrades and as a complement to module-specific upgrade.
  * Called on startup after initial XML parsing, but before module-specific upgrades
  * Enabled by definign the `.ca_datastore_upgrade`
  * [General-purpose upgrade documentation](https://clixon-docs.readthedocs.io/en/latest/backend.html#ge
neral-purpose)
* JSON parse error messages change from ` on line x: syntax error,..` to `json_parse: line x: syntax err
or`
* Unknown-element error message is more descriptive, eg from `namespace is: urn:example:clixon` to: `Fai
led to find YANG spec of XML node: x with parent: xp in namespace urn:example:clixon`.
* C-API parse and validation API more capable
  * `xml_spec_populate` family of functions extended with three-value return values
    * -1: error, 0: parse OK, 1: parse and YANG binding OK.
  * `xml_parse` and `json_parse` API changes
    * Three value returns: -1: error, 0: parse OK, 1: parse and YANG binding OK.
    * Extended `xml_parse_file2` and `xml_parse_string2` extended API functions with all options available.
      * New concept called `yang_bind` that defines how XML symbols are bound to YANG after parsing
    * Existing API same except `xml_parse_file` `endtag` argument moved to `xml_parse_file2`
* C-API: Added instrumentation: `xml_size` and `xml_stats_get`.
* Fixed: Enabling modstate (CLICON_XMLDB_MODSTATE), changing a revision on a yang, and restarting made the backend daemon exit at start (thanks Matt)
  * Also: ensure to load `ietf-yang-library.yang ` if CLICON_XMLDB_MODSTATE is set
This commit is contained in:
Olof hagsand 2020-02-20 14:00:01 +01:00
parent d665992f7c
commit 9fa5e216c4
80 changed files with 2204 additions and 755 deletions
Download patch file Download diff file Expand all files Collapse all files

2
apps/restconf/README.md
View file

@ -217,7 +217,7 @@ You can also run restconf in a debugger.
sudo gdb /www-data/clixon_restconf
(gdb) run -D 1 -f /usr/local/etc/example.xml
```
but you need to ensure /www-data/fastcgi_restconf.sock has the following access:
but you need to ensure /www-data/fastcgi_restconf.sock has the following access (may need to be done after restconf has started)
```
rwxr-xr-x 1 www-data www-data 0 sep 22 11:46 /www-data/fastcgi_restconf.sock
```

19
apps/restconf/restconf_main.c
View file

@ -3,6 +3,7 @@
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
This file is part of CLIXON.
@ -233,7 +234,7 @@ api_root(clicon_handle h,
if (xml_parse_string("<restconf xmlns=\"urn:ietf:params:xml:ns:yang:ietf-restconf\"><data/><operations/><yang-library-version>2016-06-21</yang-library-version></restconf>", NULL, &xt) < 0)
goto done;
if (xml_apply(xt, CX_ELMNT, xml_spec_populate, yspec) < 0)
if (xml_spec_populate(xt, yspec, NULL) < 0)
goto done;
if ((cb = cbuf_new()) == NULL){
clicon_err(OE_XML, errno, "cbuf_new");
@ -548,15 +549,15 @@ usage(clicon_handle h,
{
fprintf(stderr, "usage:%s [options]\n"
"where options are\n"
"\t-h \t\tHelp\n"
"\t-D <level>\tDebug level\n"
"\t-f <file>\tConfiguration file (mandatory)\n"
"\t-l <s|f<file>> \tLog on (s)yslog, (f)ile (syslog is default)\n"
"\t-p <dir>\tYang directory path (see CLICON_YANG_DIR)\n"
"\t-d <dir>\tSpecify restconf plugin directory dir (default: %s)\n"
"\t-y <file>\tLoad yang spec file (override yang main module)\n"
"\t-h \t\t Help\n"
"\t-D <level>\t Debug level\n"
"\t-f <file>\t Configuration file (mandatory)\n"
"\t-l <s|f<file>> \t Log on (s)yslog, (f)ile (syslog is default)\n"
"\t-p <dir>\t Yang directory path (see CLICON_YANG_DIR)\n"
"\t-d <dir>\t Specify restconf plugin directory dir (default: %s)\n"
"\t-y <file>\t Load yang spec file (override yang main module)\n"
"\t-a UNIX|IPv4|IPv6 Internal backend socket family\n"
"\t-u <path|addr>\tInternal socket domain path or IP addr (see -a)\n"
"\t-u <path|addr>\t Internal socket domain path or IP addr (see -a)\n"
"\t-o \"<option>=<value>\" Give configuration option overriding config file (see clixon-config.yang)\n",
argv0,
clicon_restconf_dir(h)

21
apps/restconf/restconf_methods.c
View file

@ -3,6 +3,7 @@
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
This file is part of CLIXON.
@ -365,7 +366,12 @@ api_data_write(clicon_handle h,
goto done;
goto ok;
}
/* Create a dummy data tree parent to hook in the parsed data.
*/
if ((xdata0 = xml_new("data0", NULL, NULL)) == NULL)
goto done;
if (xml_copy_one(api_path?xml_parent(xbot):xbot, xdata0) < 0)
goto done;
/* Parse input data as json or xml into xml */
switch (media_in){
case YANG_DATA_XML:
@ -418,7 +424,7 @@ api_data_write(clicon_handle h,
/* The message-body MUST contain exactly one instance of the
* expected data resource.
*/
if (xml_child_nr(xdata0) != 1){
if (xml_child_nr_type(xdata0, CX_ELMNT) != 1){
if (netconf_malformed_message_xml(&xerr, "The message-body MUST contain exactly one instance of the expected data resource") < 0)
goto done;
if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@ -429,7 +435,7 @@ api_data_write(clicon_handle h,
goto done;
goto ok;
}
xdata = xml_child_i(xdata0,0);
xdata = xml_child_i_type(xdata0, 0, CX_ELMNT);
#if 0
if (debug){
cbuf *ccc=cbuf_new();
@ -481,6 +487,13 @@ api_data_write(clicon_handle h,
xml_free(xtop);
xtop = xdata;
xml_name_set(xtop, "config");
/* remove default namespace */
if ((xa = xml_find_type(xtop, NULL, "xmlns", CX_ATTR)) != NULL){
if (xml_rm(xa) < 0)
goto done;
if (xml_free(xa) < 0)
goto done;
}
}
else {
/* There is an api-path that defines an element in the datastore tree.
@ -557,7 +570,7 @@ api_data_write(clicon_handle h,
goto done;
nullspec = (xml_spec(xdata) == NULL);
/* xbot is already populated, resolve yang for added xdata too */
if (xml_apply0(xdata, CX_ELMNT, xml_spec_populate, yspec) < 0)
if (xml_spec_populate0(xdata, yspec, NULL) < 0)
goto done;
if (media_in == YANG_DATA_JSON && nullspec){
/* json2xml decode could not be done above in json_parse,

3
apps/restconf/restconf_methods_get.c
View file

@ -3,6 +3,7 @@
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
This file is part of CLIXON.
@ -203,7 +204,7 @@ api_data_get2(clicon_handle h,
goto done;
goto ok;
}
if (xml_apply(xret, CX_ELMNT, xml_spec_populate, yspec) < 0)
if (xml_spec_populate(xret, yspec, NULL) < 0)
goto done;
/* We get return via netconf which is complete tree from root
* We need to cut that tree to only the object.

129
apps/restconf/restconf_methods_post.c
View file

@ -1,8 +1,9 @@
/*
*
***** BEGIN LICENSE BLOCK *****
Copyright (C) 2009-2019 Olof Hagsand
Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC
This file is part of CLIXON.
@ -106,28 +107,28 @@ api_data_post(clicon_handle h,
int pretty,
restconf_media media_out)
{
int retval = -1;
int retval = -1;
enum operation_type op = OP_CREATE;
cxobj *xdata0 = NULL; /* Original -d data struct (including top symbol) */
cxobj *xdata; /* -d data (without top symbol)*/
int i;
cbuf *cbx = NULL;
cxobj *xtop = NULL; /* top of api-path */
cxobj *xbot = NULL; /* bottom of api-path */
yang_stmt *ybot = NULL; /* yang of xbot */
yang_stmt *ymoddata = NULL; /* yang module of data (-d) */
yang_stmt *yspec;
yang_stmt *ydata;
cxobj *xa;
cxobj *xret = NULL;
cxobj *xretcom = NULL; /* return from commit */
cxobj *xretdis = NULL; /* return from discard-changes */
cxobj *xerr = NULL; /* malloced must be freed */
cxobj *xe; /* dont free */
char *username;
int nullspec = 0;
int ret;
cxobj *xdata = NULL; /* The actual data object to modify */
int i;
cbuf *cbx = NULL;
cxobj *xtop = NULL; /* top of api-path */
cxobj *xbot = NULL; /* bottom of api-path */
yang_stmt *ybot = NULL; /* yang of xbot */
yang_stmt *ymoddata = NULL; /* yang module of data (-d) */
yang_stmt *yspec;
yang_stmt *ydata;
cxobj *xa;
cxobj *xret = NULL;
cxobj *xretcom = NULL; /* return from commit */
cxobj *xretdis = NULL; /* return from discard-changes */
cxobj *xerr = NULL; /* malloced must be freed */
cxobj *xe; /* dont free */
cxobj *x;
char *username;
int ret;
restconf_media media_in;
int nrchildren0 = 0;
clicon_debug(1, "%s api_path:\"%s\"", __FUNCTION__, api_path);
clicon_debug(1, "%s data:\"%s\"", __FUNCTION__, data);
@ -155,15 +156,6 @@ api_data_post(clicon_handle h,
goto ok;
}
}
#if 1
if (debug){
cbuf *ccc=cbuf_new();
if (clicon_xml2cbuf(ccc, xtop, 0, 0, -1) < 0)
goto done;
clicon_debug(1, "%s XURI:%s", __FUNCTION__, cbuf_get(ccc));
cbuf_free(ccc);
}
#endif
/* 4.4.1: The message-body MUST contain exactly one instance of the
* expected data resource. (tested again below)
*/
@ -178,11 +170,19 @@ api_data_post(clicon_handle h,
goto done;
goto ok;
}
/* Record how many children before parse (after check nr should be +1) */
nrchildren0 = 0;
x = NULL;
while ((x = xml_child_each(xbot, x, CX_ELMNT)) != NULL){
nrchildren0++;
xml_flag_set(x, XML_FLAG_MARK);
}
/* Parse input data as json or xml into xml */
media_in = restconf_content_type(r);
switch (media_in){
case YANG_DATA_XML:
if (xml_parse_string(data, NULL, &xdata0) < 0){
if (xml_parse_string(data, yspec, &xbot) < 0){
if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
goto done;
if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@ -195,16 +195,10 @@ api_data_post(clicon_handle h,
}
break;
case YANG_DATA_JSON:
/* Data here cannot cannot (always) be Yang populated since it is
* loosely hanging without top symbols.
* And if it is not yang populated, it cant be translated properly
* from JSON to XML.
* Therefore, yang population is done later after addsub below
* Further complication is that if data is root resource, then it will
* work, so I need to check below that it didnt.
* THIS could be simplified.
/* If xbot is top-level (api_path=null) it does not have a spec therefore look for
* top-level (yspec) otherwise assume parent (xbot) is populated.
*/
if ((ret = json_parse_str(data, yspec, &xdata0, &xerr)) < 0){
if ((ret = json_parse_str(data, yspec, &xbot, &xerr)) < 0){
if (netconf_malformed_message_xml(&xerr, clicon_err_reason) < 0)
goto done;
if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@ -230,10 +224,12 @@ api_data_post(clicon_handle h,
goto ok;
break;
} /* switch media_in */
/* 4.4.1: The message-body MUST contain exactly one instance of the
/* RFC 8040 4.4.1: The message-body MUST contain exactly one instance of the
* expected data resource.
*/
if (xml_child_nr(xdata0) != 1){
clicon_debug(1, "%s nrchildren0: %d", __FUNCTION__, nrchildren0);
if (xml_child_nr_type(xbot, CX_ELMNT) - nrchildren0 != 1){
if (netconf_malformed_message_xml(&xerr, "The message-body MUST contain exactly one instance of the expected data resource") < 0)
goto done;
if ((xe = xpath_first(xerr, NULL, "rpc-error")) == NULL){
@ -244,23 +240,32 @@ api_data_post(clicon_handle h,
goto done;
goto ok;
}
xdata = xml_child_i(xdata0, 0);
if (ys_module_by_xml(yspec, xdata, &ymoddata) < 0)
goto done;
/* Find the actual (new) object, the single unmarked one */
x = NULL;
while ((x = xml_child_each(xbot, x, CX_ELMNT)) != NULL){
if (xml_flag(x, XML_FLAG_MARK)){
xml_flag_reset(x, XML_FLAG_MARK);
continue;
}
xdata = x;
}
/* Add operation (create/replace) as attribute */
if ((xa = xml_new("operation", xdata, NULL)) == NULL)
goto done;
xml_type_set(xa, CX_ATTR);
xml_prefix_set(xa, NETCONF_BASE_PREFIX);
if (xml_value_set(xa, xml_operation2str(op)) < 0)
goto done;
/* Replace xbot with x, ie bottom of api-path with data */
if (xml_addsub(xbot, xdata) < 0)
#if 0 /* XXX postpone this, there is something wrong with NETCONF_BASE_NAMESPACE not appearing here
* but later it does due to default handling,... */
if (xml_namespace_change(xa, NETCONF_BASE_NAMESPACE, NETCONF_BASE_PREFIX) < 0)
goto done;
/* xbot is already populated, resolve yang for added xdata too
*/
nullspec = (xml_spec(xdata) == NULL);
if (xml_apply0(xdata, CX_ELMNT, xml_spec_populate, yspec) < 0)
#else
xml_prefix_set(xa, NETCONF_BASE_PREFIX); /* XXX: But this assumes proper namespace set */
#endif
if (ys_module_by_xml(yspec, xdata, &ymoddata) < 0)
goto done;
/* ybot is parent of spec(parent(data))) */
if (ymoddata && (ydata = xml_spec(xdata)) != NULL){
@ -290,20 +295,6 @@ api_data_post(clicon_handle h,
goto ok;
}
}
if (media_in == YANG_DATA_JSON && nullspec){
/* json2xml decode may not have been done above in json_parse,
need to be done here instead
UNLESS it is a root resource, then json-parse has already done it
*/
if ((ret = json2xml_decode(xdata, &xerr)) < 0)
goto done;
if (ret == 0){
if (api_return_err(h, r, xerr, pretty, media_out, 0) < 0)
goto done;
goto ok;
}
}
/* If restconf insert/point attributes are present, translate to netconf */
if (restconf_insert_attributes(xdata, qvec) < 0)
goto done;
@ -405,8 +396,6 @@ api_data_post(clicon_handle h,
xml_free(xretdis);
if (xtop)
xml_free(xtop);
if (xdata0)
xml_free(xdata0);
if (cbx)
cbuf_free(cbx);
return retval;
@ -630,7 +619,7 @@ api_operations_post_output(clicon_handle h,
if (youtput!=NULL){
xml_spec_set(xoutput, youtput); /* needed for xml_spec_populate */
#if 0
if (xml_apply(xoutput, CX_ELMNT, xml_spec_populate, yspec) < 0)
if (xml_spec_populate(xoutput, yspec, NULL) < 0)
goto done;
if ((ret = xml_yang_validate_all(xoutput, &xerr)) < 0)
goto done;
@ -861,7 +850,7 @@ api_operations_post(clicon_handle h,
}
#endif
/* 6. Validate incoming RPC and fill in defaults */
if (xml_spec_populate_rpc_input(h, xtop, yspec) < 0) /* */
if (xml_spec_populate_rpc(xtop, yspec, NULL) < 0) /* */
goto done;
if ((ret = xml_yang_validate_rpc(h, xtop, &xret)) < 0)
goto done;