experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* Optimized prefix checks at xml parse time: using many prefixes slowed down parsing considerably

Browse source 
* Cleared startup-db cache after restart
This commit is contained in:
Olof hagsand 2020-04-29 23:11:48 +02:00
parent 5b57e68e2b
commit b8ec6a4957
12 changed files with 232 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

38
lib/src/clixon_datastore.c
View file

@ -369,7 +369,31 @@ xmldb_exists(clicon_handle h,
return retval;
}
/*! Delete database. Remove file
/*! Clear database cache if any for mem/size optimization only
* @param[in] h Clicon handle
* @param[in] db Database
* @retval -1 Error
* @retval 0 OK
*/
int
xmldb_clear(clicon_handle h,
const char *db)
{
cxobj *xt = NULL;
db_elmnt *de = NULL;
if (clicon_datastore_cache(h) != DATASTORE_NOCACHE){
if ((de = clicon_db_elmnt_get(h, db)) != NULL){
if ((xt = de->de_xml) != NULL){
xml_free(xt);
de->de_xml = NULL;
}
}
}
return 0;
}
/*! Delete database, clear cache if any. Remove file
* @param[in] h Clicon handle
* @param[in] db Database
* @retval -1 Error
@ -381,18 +405,10 @@ xmldb_delete(clicon_handle h,
{
int retval = -1;
char *filename = NULL;
db_elmnt *de = NULL;
cxobj *xt = NULL;
struct stat sb;
if (clicon_datastore_cache(h) != DATASTORE_NOCACHE){
if ((de = clicon_db_elmnt_get(h, db)) != NULL){
if ((xt = de->de_xml) != NULL){
xml_free(xt);
de->de_xml = NULL;
}
}
}
if (xmldb_clear(h, db) < 0)
goto done;
if (xmldb_db2file(h, db, &filename) < 0)
goto done;
if (lstat(filename, &sb) == 0)

74
lib/src/clixon_xml.c
View file

@ -177,10 +177,10 @@ struct xml{
int x_childvec_max;/* Length of allocated vector */
cvec *x_ns_cache; /* Cached vector of namespaces */
cvec *x_ns_cache; /* Cached vector of namespaces (set by bind-yang) */
yang_stmt *x_spec; /* Pointer to specification, eg yang,
by reference, dont free */
cg_var *x_cv; /* Cached value as cligen variable (eg xml_cmp) */
cg_var *x_cv; /* Cached value as cligen variable (set by xml_cmp) */
#ifdef XML_EXPLICIT_INDEX
struct search_index *x_search_index; /* explicit search index vectors */
#endif
@ -199,8 +199,7 @@ struct xmlbody{
int _xb_vector_i; /* internal use: xml_child_each */
int _xb_i; /* internal use for sorting:
see xml_enumerate and xml_cmp */
cbuf *xb_value_cb; /* attribute and body nodes have values (XXX: this consumes
memory) cv? */
cbuf *xb_value_cb; /* attribute and body nodes have values */
};
#endif /* XML_NEW_DIFFERENTIATE */
@ -240,6 +239,7 @@ xml_stats_global(uint64_t *nr)
return 0;
}
/*! Return the alloced memory of a single XML obj
* @param[in] x XML object
* @param[out] szp Size of this XML obj
@ -280,7 +280,7 @@ xml_stats_one(cxobj *x,
#ifdef XML_NEW_DIFFERENTIATE
sz += sizeof(struct xmlbody);
#else
sz += sizeof(struct xmlbody);
sz += sizeof(struct xml);
#endif
if (x->x_value_cb)
sz += cbuf_buflen(x->x_value_cb);
@ -294,6 +294,45 @@ xml_stats_one(cxobj *x,
return 0;
}
#if 0
/*! Print memory stats of a single object
*/
static int
xml_print_stats_one(FILE *f,
cxobj *x)
{
size_t sz = 0;
xml_stats_one(x, &sz);
fprintf(f, "%s:\n", xml_name(x));
fprintf(f, " sum: \t\t%u\n", (unsigned int)sz);
if (xml_type(x) == CX_ELMNT)
fprintf(f, " base struct: \t%u\n", (unsigned int)sizeof(struct xml));
else
fprintf(f, " base struct: \t%u\n", (unsigned int)sizeof(struct xmlbody));
if (x->x_name)
fprintf(f, " name: \t%u\n", (unsigned int)strlen(x->x_name) + 1);
if (x->x_prefix)
fprintf(f, " prefix: \t%u\n", (unsigned int)strlen(x->x_prefix) + 1);
if (xml_type(x) == CX_ELMNT){
if (x->x_childvec_max)
fprintf(f, " childvec: \t%u\n", (unsigned int)(x->x_childvec_max*sizeof(struct xml*)));
if (x->x_ns_cache)
fprintf(f, " ns-cache: \t%u\n", (unsigned int)cvec_size(x->x_ns_cache));
if (x->x_cv)
fprintf(f, " value-cv: \t%u\n", (unsigned int)cv_size(x->x_cv));
if (x->x_search_index)
fprintf(f, " search-index: \t%u\n",
(unsigned int)(strlen(x->x_search_index->si_name) + 1 + clixon_xvec_len(x->x_search_index->si_xvec)*sizeof(struct cxobj*)));
}
else{
if (x->x_value_cb)
fprintf(f, " value-cb: \t%u\n", cbuf_buflen(x->x_value_cb));
}
return 0;
}
#endif
/*! Return statistics of an XML tree recursively
* @param[in] xt XML object
* @param[out] szp Size of this XML obj recursively
@ -313,6 +352,7 @@ xml_stats(cxobj *xt,
clicon_err(OE_XML, EINVAL, "xml node is NULL");
goto done;
}
// xml_print_stats_one(stderr, xt);
*nrp += 1;
xml_stats_one(xt, &sz);
if (szp)
@ -377,21 +417,21 @@ xml_prefix(cxobj *xn)
}
/*! Set prefix of xnode, prefix is copied
* @param[in] xn xml node
* @param[in] localname new prefix, null-terminated string, copied by function
* @retval -1 on error with clicon-err set
* @retval 0 OK
* @param[in] xn XML node
* @param[in] prefix New prefix, null-terminated string, copied by function
* @retval -1 Error with clicon-err set
* @retval 0 OK
*/
int
xml_prefix_set(cxobj *xn,
char *localname)
char *prefix)
{
if (xn->x_prefix){
free(xn->x_prefix);
xn->x_prefix = NULL;
}
if (localname){
if ((xn->x_prefix = strdup(localname)) == NULL){
if (prefix){
if ((xn->x_prefix = strdup(prefix)) == NULL){
clicon_err(OE_XML, errno, "strdup");
return -1;
}
@ -1136,7 +1176,9 @@ xml_spec_set(cxobj *x,
* @param[in] x XML node (body and leaf/leaf-list)
* @retval cv CLIgen variable containing value of x body
* @retval NULL
* @note only applicable if x is body and has yang-spec and is leaf or leaf-list
* Only applicable if x is body and has yang-spec and is leaf or leaf-list
* Only accessed by xml_cv_cache as part of sorting in xml_cmp
* @see xml_cv_cache
*/
cg_var *
xml_cv(cxobj *x)
@ -1146,11 +1188,13 @@ xml_cv(cxobj *x)
return x->x_cv;
}
/*! Return (cached) cligen variable value of xml node
/*! Set (cached) cligen variable value of xml node
* @param[in] x XML node (body and leaf/leaf-list)
* @param[in] cv CLIgen variable containing value of x body
* @retval 0 OK
* @note only applicable if x is body and has yang-spec and is leaf or leaf-list
* Only applicable if x is body and has yang-spec and is leaf or leaf-list
* Only accessed by xml_cv_cache as part of sorting in xml_cmp
* @see xml_cv_cache
*/
int
xml_cv_set(cxobj *x,

2
lib/src/clixon_xml_io.c
View file

@ -423,7 +423,7 @@ _xml_parse(const char *str,
for (i = 0; i < xy.xy_xlen; i++) {
x = xy.xy_xvec[i];
/* Verify namespaces after parsing */
if (xml_apply0(x, CX_ELMNT, xml_localname_check, NULL) < 0)
if (xml2ns_recurse(x) < 0)
goto done;
/* Populate, ie associate xml nodes with yang specs
*/

94
lib/src/clixon_xml_nsctx.c
View file

@ -424,7 +424,6 @@ xml_nsctx_yangspec(yang_stmt *yspec,
* if (xml2ns(xt, NULL, &namespace) < 0)
* err;
* @endcode
* @see xmlns_check
* @see xmlns_set cache is set
* @note, this function uses a cache.
*/
@ -474,6 +473,38 @@ xml2ns(cxobj *x,
return retval;
}
/*! Recursively check prefix / namespaces (and populate ns cache)
* @retval 1 OK
* @retval 0 (Some) prefix not found
* @retval -1 Error
*/
int
xml2ns_recurse(cxobj *xt)
{
int retval = -1;
cxobj *x;
char *prefix;
char *namespace;
x = NULL;
while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
if ((prefix = xml_prefix(x)) != NULL){
namespace = NULL;
if (xml2ns(x, prefix, &namespace) < 0)
goto done;
if (namespace == NULL){
clicon_err(OE_XML, ENOENT, "No namespace associated with %s:%s", prefix, xml_name(x));
goto done;
}
}
if (xml2ns_recurse(x) < 0)
goto done;
}
retval = 0;
done:
return retval;
}
/*! Add a namespace attribute to an XML node, either default or specific prefix
* @param[in] x XML tree
* @param[in] prefix prefix/ns localname. If NULL then set default xmlns
@ -573,64 +604,3 @@ xml2prefix(cxobj *xn,
goto done;
}
/*! See if xmlns:[<localname>=]<uri> exists, if so return <uri>
*
* @param[in] xn XML node
* @param[in] nsn Namespace name
* @retval URI return associated URI if found
* @retval NULL No namespace name binding found for nsn
* @see xml2ns XXX coordinate
*/
static char *
xmlns_check(cxobj *xn,
char *nsn)
{
cxobj *x = NULL;
char *xns;
while ((x = xml_child_each(xn, x, CX_ATTR)) != NULL)
if ((xns = xml_prefix(x)) && strcmp(xns, "xmlns")==0 &&
strcmp(xml_name(x), nsn) == 0)
return xml_value(x);
return NULL;
}
/*! Check namespace of xml node by searching recursively among ancestors
* @param[in] xn xml node
* @param[in] namespace check validity of namespace
* @retval 0 Found / validated or no yang spec
* @retval -1 Not found
* @note This function is grossly inefficient
*/
int
xml_localname_check(cxobj *xn,
void *arg)
{
cxobj *xp = NULL;
char *nsn;
char *n;
yang_stmt *ys = xml_spec(xn);
/* No namespace name - comply */
if ((nsn = xml_prefix(xn)) == NULL)
return 0;
/* Check if NSN defined in same node */
if (xmlns_check(xn, nsn) != NULL)
return 0;
/* Check if NSN defined in some ancestor */
while ((xp = xml_parent(xn)) != NULL) {
if (xmlns_check(xp, nsn) != NULL)
return 0;
xn = xp;
}
/* Check if my namespace */
if ((n = yang_find_myprefix(ys)) != NULL && strcmp(nsn,n)==0)
return 0;
/* Check if any imported module */
if (yang_find_module_by_prefix(ys, nsn) != NULL)
return 0;
/* Not found, error */
clicon_err(OE_XML, ENOENT, "Namespace name %s in %s:%s not found",
nsn, nsn, xml_name(xn));
return -1;
}

1
lib/src/clixon_xml_parse.y
View file

@ -198,6 +198,7 @@ xml_parse_prefixed_name(clixon_xml_yacc *xy,
xp = xy->xy_xparent;
if ((x = xml_new(name, xp, CX_ELMNT)) == NULL)
goto done;
/* Cant check namespaces here since local xmlns attributes loaded after */
if (xml_prefix_set(x, prefix) < 0)
goto done;

2
lib/src/clixon_xml_sort.c
View file

@ -80,6 +80,8 @@
* @retval -1 Error
* @note only applicable if x is body and has yang-spec and is leaf or leaf-list
* Move to clixon_xml.c?
* As a side-effect sets the cache.
* Clear cache with xml_cv_set(x, NULL)
*/
static int
xml_cv_cache(cxobj *x,