/*
 *
  ***** BEGIN LICENSE BLOCK *****
 
  Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
  Copyright (C) 2017-2019 Olof Hagsand
  Copyright (C) 2020-2021 Olof Hagsand and Rubicon Communications, LLC(Netgate)

  This file is part of CLIXON.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

  Alternatively, the contents of this file may be used under the terms of
  the GNU General Public License Version 3 or later (the "GPL"),
  in which case the provisions of the GPL are applicable instead
  of those above. If you wish to allow use of your version of this file only
  under the terms of the GPL, and not to allow others to
  use your version of this file under the terms of Apache License version 2, 
  indicate your decision by deleting the provisions above and replace them with
  the  notice and other provisions required by the GPL. If you do not delete
  the provisions above, a recipient may use your version of this file under
  the terms of any one of the Apache License version 2 or the GPL.

  ***** END LICENSE BLOCK *****

 *
 * Translation / mapping code between formats
 */
#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <ctype.h>
#include <string.h>
#include <syslog.h>
#include <fcntl.h>
#include <assert.h>
#include <arpa/inet.h>
#include <sys/param.h>
#include <netinet/in.h>

/* cligen */
#include <cligen/cligen.h>

/* clicon */

#include "clixon_string.h"
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_handle.h"
#include "clixon_string.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_options.h"
#include "clixon_data.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_xml_nsctx.h"
#include "clixon_xpath_ctx.h"
#include "clixon_xpath.h"
#include "clixon_log.h"
#include "clixon_err.h"
#include "clixon_netconf_lib.h"
#include "clixon_xml_sort.h"
#include "clixon_yang_type.h"
#include "clixon_xml_map.h"

/* Local types 
 */
/* Merge code needs a two-phase pass where objects subject to merge are first checked for,
 * the actually inserted.
 * This is to mitigate a search problem where objects inserted are among the ones checked for
 */
typedef struct  {
    cxobj     *mt_x0c;
    cxobj     *mt_x1c;
    yang_stmt *mt_yc;
} merge_twophase;

/*! Is attribute and is either of form xmlns="", or xmlns:x="" */
int
isxmlns(cxobj *x)
{
    char *prefix = NULL;

    if (xml_type(x) != CX_ATTR)
	return 0;
    if (strcmp(xml_name(x), "xmlns")==0)
	return 1;
    if ((prefix = xml_prefix(x)) != NULL
	&& strcmp(xml_prefix(x), "xmlns")==0)
	return 1;
    return 0;
}

/*! x is element and has eactly one child which in turn has none 
 * @see child_type in clixon_json.c
 */
static int
tleaf(cxobj *x)
{
    cxobj *xc;

    if (xml_type(x) != CX_ELMNT)
	return 0;
    if (xml_child_nr_notype(x, CX_ATTR) != 1)
	return 0;
    /* From here exactly one noattr child, get it */
    xc = NULL;
    while ((xc = xml_child_each(x, xc, -1)) != NULL)
	if (xml_type(xc) != CX_ATTR)
	    break;
    if (xc == NULL)
	return -1; /* n/a */
    return (xml_child_nr_notype(xc, CX_ATTR) == 0);
}

/*! Translate XML to a "pseudo-code" textual format using a callback - internal function
 * @param[in]  f      File to print to
 * @param[in]  x      XML object to print
 * @param[in]  fn     Callback to make print function
 * @param[in]  level  print 4 spaces per level in front of each line
 */
static int 
xml2txt_recurse(FILE             *f, 
		cxobj            *x, 
		clicon_output_cb *fn,
		int               level)
{
    cxobj *xc = NULL;
    int    children=0;
    int    retval = -1;

    if (f == NULL || x == NULL || fn == NULL){
	clicon_err(OE_XML, EINVAL, "f, x or fn is NULL");
	goto done;
    }
    xc = NULL;     /* count children (elements and bodies, not attributes) */
    while ((xc = xml_child_each(x, xc, -1)) != NULL)
	if (xml_type(xc) == CX_ELMNT || xml_type(xc) == CX_BODY)
	    children++;
    if (!children){ /* If no children print line */
	switch (xml_type(x)){
	case CX_BODY:
	    (*fn)(f, "%s;\n", xml_value(x));
	    break;
	case CX_ELMNT:
	    (*fn)(f, "%*s%s;\n", 4*level, "", xml_name(x));
	    break;
	default:
	    break;
	}
	goto ok;
    }
    (*fn)(f, "%*s", 4*level, "");
    (*fn)(f, "%s ", xml_name(x));
    if (!tleaf(x))
	(*fn)(f, "{\n");
    xc = NULL;
    while ((xc = xml_child_each(x, xc, -1)) != NULL){
	if (xml_type(xc) == CX_ELMNT || xml_type(xc) == CX_BODY)
	    if (xml2txt_recurse(f, xc, fn, level+1) < 0)
		break;
    }
    if (!tleaf(x))
	(*fn)(f, "%*s}\n", 4*level, "");
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Translate XML to a "pseudo-code" textual format using a callback
 * @param[in]  f      File to print to
 * @param[in]  x      XML object to print
 * @param[in]  fn     Callback to make print function
 */
int 
xml2txt_cb(FILE             *f, 
	   cxobj            *x, 
	   clicon_output_cb *fn)
{
    return xml2txt_recurse(f, x, fn, 0);
}

/*! Translate XML to a "pseudo-code" textual format using stdio file
 * @param[in]  f      File to print to
 * @param[in]  x      XML object to print
 * @param[in]  level  print 4 spaces per level in front of each line
 * @see xml2txt_cb
 */
int 
xml2txt(FILE  *f, 
	cxobj *x, 
	int    level)
{
    return xml2txt_recurse(f, x, fprintf, 0);
}

/*! Translate from XML to CLI commands
 * Howto: join strings and pass them down. 
 * Identify unique/index keywords for correct set syntax.
 * @param[in] f        Where to print cli commands
 * @param[in] x        XML Parse-tree (to translate)
 * @param[in] prepend  Print this text in front of all commands.
 * @param[in] gt       option to steer cli syntax
 * @param[in] fn       Callback to make print function
 */
int 
xml2cli_recurse(FILE              *f, 
		cxobj             *x, 
		char              *prepend, 
		enum genmodel_type gt,
		clicon_output_cb  *fn)
{
    int              retval = -1;
    cxobj           *xe = NULL;
    cbuf            *cbpre = NULL;
    yang_stmt       *ys;
    int              match;
    char            *body;

    if (xml_type(x)==CX_ATTR)
	goto ok;
    if ((ys = xml_spec(x)) == NULL)
	goto ok;
    /* If leaf/leaf-list or presence container, then print line */
    if (yang_keyword_get(ys) == Y_LEAF ||
	yang_keyword_get(ys) == Y_LEAF_LIST){
	if (prepend)
	    (*fn)(f, "%s", prepend);
	if (gt == GT_ALL || gt == GT_VARS || gt == GT_HIDE)
	    (*fn)(f, "%s ", xml_name(x));
	if ((body = xml_body(x)) != NULL){
	    if (index(body, ' '))
		(*fn)(f, "\"%s\"", body);
	    else
		(*fn)(f, "%s", body);
	}
	(*fn)(f, "\n");
	goto ok;
    }
    /* Create prepend variable string */
    if ((cbpre = cbuf_new()) == NULL){
	clicon_err(OE_PLUGIN, errno, "cbuf_new");	
	goto done;
    }
    if (prepend)
	cprintf(cbpre, "%s", prepend);

    /* If non-presence container && HIDE mode && only child is 
     * a list, then skip container keyword
     * See also yang2cli_container */
    if (yang_container_cli_hide(ys, gt) == 0)
	cprintf(cbpre, "%s ", xml_name(x));

    /* If list then first loop through keys */
    if (yang_keyword_get(ys) == Y_LIST){
	xe = NULL;
	while ((xe = xml_child_each(x, xe, -1)) != NULL){
	    if ((match = yang_key_match(ys, xml_name(xe))) < 0)
		goto done;
	    if (!match)
		continue;
	    if (gt == GT_ALL)
		cprintf(cbpre, "%s ", xml_name(xe));
	    cprintf(cbpre, "%s ", xml_body(xe));
	}
    }
    else if ((yang_keyword_get(ys) == Y_CONTAINER) &&
	     yang_find(ys, Y_PRESENCE, NULL) != NULL){
	/* If presence container, then print as leaf (but continue to children) */
	if (prepend)
	    (*fn)(f, "%s", prepend);
	if (gt == GT_ALL || gt == GT_VARS || gt == GT_HIDE)
	    (*fn)(f, "%s ", xml_name(x));
	if ((body = xml_body(x)) != NULL){
	    if (index(body, ' '))
		(*fn)(f, "\"%s\"", body);
	    else
		(*fn)(f, "%s", body);
	}
	(*fn)(f, "\n");
    }

    /* Then loop through all other (non-keys) */
    xe = NULL;
    while ((xe = xml_child_each(x, xe, -1)) != NULL){
	if (yang_keyword_get(ys) == Y_LIST){
	    if ((match = yang_key_match(ys, xml_name(xe))) < 0)
		goto done;
	    if (match){
		(*fn)(f, "%s\n", cbuf_get(cbpre));	
		continue; /* Not key itself */
	    }
	}
	if (xml2cli_recurse(f, xe, cbuf_get(cbpre), gt, fn) < 0)
	    goto done;
    }
 ok:
    retval = 0;
  done:
    if (cbpre)
	cbuf_free(cbpre);
    return retval;
}

/*! Translate from XML to CLI commands
 * Howto: join strings and pass them down. 
 * Identify unique/index keywords for correct set syntax.
 * @param[in] f        Where to print cli commands
 * @param[in] x        XML Parse-tree (to translate)
 * @param[in] prepend  Print this text in front of all commands.
 * @param[in] gt       option to steer cli syntax
 * @param[in] fn       Callback to make print function
 */
int 
xml2cli_cb(FILE              *f, 
	   cxobj             *x, 
	   char              *prepend, 
	   enum genmodel_type gt,
	   clicon_output_cb  *fn)
{
    return xml2cli_recurse(f, x, prepend, gt, fn);
}

/*! Translate from XML to CLI commands
 * Howto: join strings and pass them down. 
 * Identify unique/index keywords for correct set syntax.
 * Args:
 *  @param[in] f        Where to print cli commands
 *  @param[in] x        XML Parse-tree (to translate)
 *  @param[in] prepend  Print this text in front of all commands.
 *  @param[in] gt       option to steer cli syntax
 */
int 
xml2cli(FILE              *f, 
	cxobj             *x, 
	char              *prepend, 
	enum genmodel_type gt)
{
    return xml2cli_recurse(f, x, prepend, gt, fprintf);
}

/*! Translate a single xml node to a cligen variable vector. Note not recursive 
 * @param[in]  xt   XML tree containing one top node
 * @param[in]  ys   Yang spec containing type specification of top-node of xt
 * @param[out] cvv  CLIgen variable vector. Should be freed by cvec_free()
 * @retval     0    Everything OK, cvv allocated and set
 * @retval    -1    Something wrong, clicon_err() called to set error. No cvv returned
 * @note  cvv Should be freed by cvec_free() after use.
 * 'Not recursive' means that only one level of XML bodies is translated to cvec:s.
 * If range is wriong (eg 1000 for uint8) a warning is logged, the value is 
 * skipped, and continues.
 * yang is needed to know which type an xml element has.
 * Example: 
    <a>
      <b>23</b>
      <c>88</c>
      <d>      
        <e>99</e>
      </d>
    </a> 
         --> b:23, c:88
 * @see cvec2xml
 */
int
xml2cvec(cxobj      *xt, 
	 yang_stmt  *yt, 
	 cvec      **cvv0)
{
    int               retval = -1;
    cvec             *cvv = NULL;
    cxobj            *xc;         /* xml iteration variable */
    yang_stmt        *ys;         /* yang spec */
    cg_var           *cv;
    cg_var           *ycv;
    char             *body;
    char             *reason = NULL;
    int               ret;
    char             *name;

    xc = NULL;
    /* Tried to allocate whole cvv here, but some cg_vars may be invalid */
    if ((cvv = cvec_new(0)) == NULL){
	clicon_err(OE_UNIX, errno, "cvec_new");
	goto err;
    }
    xc = NULL;
    /* Go through all children of the xml tree */
    while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL){
	name = xml_name(xc);
	if ((ys = yang_find_datanode(yt, name)) == NULL){
	    clicon_debug(0, "%s: yang sanity problem: %s in xml but not present in yang under %s",
			 __FUNCTION__, name, yang_argument_get(yt));
	    if ((body = xml_body(xc)) != NULL){
		if ((cv = cv_new(CGV_STRING)) == NULL){
		    clicon_err(OE_PLUGIN, errno, "cv_new");
		    goto err;
		}
		cv_name_set(cv, name);
		if ((ret = cv_parse1(body, cv, &reason)) < 0){
		    clicon_err(OE_PLUGIN, errno, "cv_parse %s",name);
		    goto err;
		}
		/* If value is out-of-range, log and skip value, and continue */
		if (ret == 0){
		    clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason); 
		    if (reason)
			free(reason);
		}
		else
		    cvec_append_var(cvv, cv); /* Add to variable vector */
		cv_free(cv);
	    }
	}
	else if ((ycv = yang_cv_get(ys)) != NULL){
	    if ((body = xml_body(xc)) != NULL){
		if ((cv = cv_new(CGV_STRING)) == NULL){
		    clicon_err(OE_PLUGIN, errno, "cv_new");
		    goto err;
		}
		if (cv_cp(cv, ycv) < 0){
		    clicon_err(OE_PLUGIN, errno, "cv_cp");
		    goto err;
		}
		if ((ret = cv_parse1(body, cv, &reason)) < 0){
		    clicon_err(OE_PLUGIN, errno, "cv_parse: %s", name);
		    goto err;
		}
		if (ret == 0){
		    clicon_log(LOG_WARNING, "cv_parse %s: %s", name, reason);
		    if (reason)
			free(reason);
		}
		else
		    cvec_append_var(cvv, cv); /* Add to variable vector */
		cv_free(cv);
	    }
	}
    }
    if (clicon_debug_get() > 1){
	clicon_debug(2, "%s cvv:\n", __FUNCTION__);
	cvec_print(stderr, cvv);
    }
    *cvv0 = cvv;
    return 0;
 err:
    if (cvv)
	cvec_free(cvv);
    return retval;
}

/*! Translate a cligen variable vector to an XML tree with depth one 
 * @param[in]   cvv  CLIgen variable vector. Should be freed by cvec_free()
 * @param[in]   toptag    The XML tree in xt will have this XML tag
 * @param[in]   xt   Parent, or NULL
 * @param[out]  xt   Pointer to XML tree containing one top node. Should be freed with xml_free
 * @retval      0    Everything OK, cvv allocated and set
 * @retval     -1    Something wrong, clicon_err() called to set error. No xt returned
 * @see xml2cvec
 * @see cvec2xml   This does more but has an internal xml2cvec translation
*/
int
cvec2xml_1(cvec   *cvv, 
	   char   *toptag, 
	   cxobj  *xp,
	   cxobj **xt0)
{
    int               retval = -1;
    cxobj            *xt = NULL;
    cxobj            *xn;
    cxobj            *xb;
    cg_var           *cv;
    char             *val;
    int               len=0;
    int               i;

    cv = NULL;
    while ((cv = cvec_each(cvv, cv)) != NULL) 
	len++;
    if ((xt = xml_new(toptag, xp, CX_ELMNT)) == NULL)
	goto err;
    if (xml_childvec_set(xt, len) < 0)
	goto err;
    cv = NULL;
    i = 0;
    while ((cv = cvec_each(cvv, cv)) != NULL) {
	if (cv_type_get(cv)==CGV_ERR || cv_name_get(cv) == NULL)
	    continue;
	if ((xn = xml_new(cv_name_get(cv), NULL, CX_ELMNT)) == NULL) /* this leaks */
	    goto err;
	xml_parent_set(xn, xt);
	xml_child_i_set(xt, i++, xn);
	if ((xb = xml_new("body", xn, CX_BODY)) == NULL) /* this leaks */
	    goto err;
	val = cv2str_dup(cv);
	xml_value_set(xb, val); /* this leaks */
	if (val)
	    free(val);
    }
    *xt0 = xt;
    return 0;
 err:
    if (xt)
	xml_free(xt);
    return retval;
}

/*! Recursive help function to compute differences between two xml trees
 * @param[in]  x0         First XML tree
 * @param[in]  x1         Second XML tree
 * @param[out] x0vec      Pointervector to XML nodes existing in only first tree
 * @param[out] x0veclen   Length of first vector
 * @param[out] x1vec      Pointervector to XML nodes existing in only second tree
 * @param[out] x1veclen   Length of x1vec vector
 * @param[out] changed_x0 Pointervector to XML nodes changed orig value
 * @param[out] changed_x1 Pointervector to XML nodes changed wanted value
 * @param[out] changedlen Length of changed vector
 * Algorithm to compare two sorted lists A, B:
 *   A 0 1 2 3 5 6
 *   B 0 2 4 5 6
 * Let (a, b) be first elements of (A, B) respectively(*)
 *   a = b :  EITHER leafs: a!=b : add a in changed_x0, b in changed_x1,
 *            OR: Set (A,B) to children of (a,b) and call algorithm recursively
 *         , get next (a,b)
 *   a < b : add a in x0, get next a 
 *   a > b : add b in x1, get next b
 * (*) "comparing" a&b here is made by xml_cmp() which judges equality from a structural
 *     perspective, ie both have the same yang spec, if they are lists, they have the
 *     the same keys. NOT that the values are equal!
 * @see xml_diff  API function, this one is internal and recursive
 */
static int
xml_diff1(cxobj     *x0, 
	  cxobj     *x1,
	  cxobj   ***x0vec,
	  int       *x0veclen,
	  cxobj   ***x1vec,
	  int       *x1veclen,
	  cxobj   ***changed_x0,
	  cxobj   ***changed_x1,
	  int       *changedlen)
{
    int        retval = -1;
    cxobj     *x0c = NULL; /* x0 child */
    cxobj     *x1c = NULL; /* x1 child */
    yang_stmt *yc;
    char      *b1;
    char      *b2;
    int        eq;

    /* Traverse x0 and x1 in lock-step */
    x0c = x1c = NULL;    
    x0c = xml_child_each(x0, x0c, CX_ELMNT);
    x1c = xml_child_each(x1, x1c, CX_ELMNT);
    for (;;){
	if (x0c == NULL && x1c == NULL)
	    goto ok;
	else if (x0c == NULL){
	    if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
		goto done;
	    x1c = xml_child_each(x1, x1c, CX_ELMNT);
	    continue;
	}
	else if (x1c == NULL){
	    if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
		goto done;
	    x0c = xml_child_each(x0, x0c, CX_ELMNT);
	    continue;
	}
	/* Both x0c and x1c exists, check if they are yang-equal. */
	eq = xml_cmp(x0c, x1c, 0, 0, NULL);
	if (eq < 0){
	    if (cxvec_append(x0c, x0vec, x0veclen) < 0) 
		goto done;
	    x0c = xml_child_each(x0, x0c, CX_ELMNT);
	    continue;
	}
	else if (eq > 0){
	    if (cxvec_append(x1c, x1vec, x1veclen) < 0) 
		goto done;
	    x1c = xml_child_each(x1, x1c, CX_ELMNT);
	    continue;
	}
	else{ /* equal */
	    /* xml-spec NULL could happen with anydata children for example,
	     * if so, continute compare children but without yang
	     */
	    yc = xml_spec(x0c);
	    if (yc && yang_keyword_get(yc) == Y_LEAF){
		/* if x0c and x1c are leafs w bodies, then they may be changed */
		b1 = xml_body(x0c);
		b2 = xml_body(x1c);
		if (b1 == NULL && b2 == NULL)
		    ;
		else if (b1 == NULL || b2 == NULL || strcmp(b1, b2) != 0){
		    if (cxvec_append(x0c, changed_x0, changedlen) < 0) 
			goto done;
		    (*changedlen)--; /* append two vectors */
		    if (cxvec_append(x1c, changed_x1, changedlen) < 0) 
			goto done;
		}
	    }
	    else if (xml_diff1(x0c, x1c,   
			       x0vec, x0veclen, 
			       x1vec, x1veclen, 
			       changed_x0, changed_x1, changedlen)< 0)
		goto done;
	}
	x0c = xml_child_each(x0, x0c, CX_ELMNT);
	x1c = xml_child_each(x1, x1c, CX_ELMNT);
    }
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Compute differences between two xml trees
 * @param[in]  yspec      Yang specification
 * @param[in]  x0         First XML tree
 * @param[in]  x1         Second XML tree
 * @param[out] first      Pointervector to XML nodes existing in only first tree
 * @param[out] firstlen   Length of first vector
 * @param[out] second     Pointervector to XML nodes existing in only second tree
 * @param[out] secondlen  Length of second vector
 * @param[out] changed_x0 Pointervector to XML nodes changed orig value
 * @param[out] changed_x1 Pointervector to XML nodes changed wanted value
 * @param[out] changedlen Length of changed vector
 * All xml vectors should be freed after use.
 */
int
xml_diff(yang_stmt *yspec, 
	 cxobj     *x0, 
	 cxobj     *x1,
	 cxobj   ***first,
	 int       *firstlen,
	 cxobj   ***second,
	 int       *secondlen,
	 cxobj   ***changed_x0,
	 cxobj   ***changed_x1,
	 int       *changedlen)
{
    int retval = -1;

    *firstlen = 0;
    *secondlen = 0;    
    *changedlen = 0;
    if (x0 == NULL && x1 == NULL)
	return 0;
    if (x1 == NULL){
	if (cxvec_append(x0, first, firstlen) < 0) 
	    goto done;
	goto ok;
    }
    if (x0 == NULL){
	if (cxvec_append(x0, second, secondlen) < 0) 
	    goto done;
	goto ok;
    }
    if (xml_diff1(x0, x1,
		  first, firstlen, 
		  second, secondlen, 
		  changed_x0, changed_x1, changedlen) < 0)
	goto done;
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Prune everything that does not pass test or have at least a child* does not
 * @param[in]   xt      XML tree with some node marked
 * @param[in]   flag    Which flag to test for
 * @param[in]   test    1: test that flag is set, 0: test that flag is not set
 * @param[out]  upmark  Set if a child (recursively) has marked set.
 * The function removes all branches that does not pass the test
 * Purge all nodes that dont have MARK flag set recursively.
 * Save all nodes that is MARK:ed or have at least one (grand*)child that is MARKed
 * @code
 *    xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL);
 * @endcode
 * @note This function seems a little too complex semantics
 * @see xml_tree_prune_flagged for a simpler variant
 */
#if 1
int
xml_tree_prune_flagged_sub(cxobj *xt, 
			   int    flag,
			   int    test,
			   int   *upmark)
{
    int        retval = -1;
    int        submark;
    int        mark;
    cxobj     *x;
    cxobj     *xprev;
    int        iskey;
    int        anykey=0;
    yang_stmt *yt;

    mark = 0;
    yt = xml_spec(xt); /* xan be null */
    x = NULL;
    xprev = x = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
	if (xml_flag(x, flag) == test?flag:0){
	    /* Pass test */
	    mark++;
	    xprev = x;
	    continue; /* mark and stop here */
	}
	/* If it is key dont remove it yet (see second round) */
	if (yt){
	    if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
		goto done;
	    if (iskey){
		anykey++;
		xprev = x; /* skip if this is key */
		continue;
	    }
	}
	if (xml_tree_prune_flagged_sub(x, flag, test, &submark) < 0)
	    goto done;
	/* if xt is list and submark anywhere, then key subs are also marked
	 */
	if (submark)
	    mark++;
	else{ /* Safe with xml_child_each if last */
	    if (xml_purge(x) < 0)
		goto done;
	    x = xprev;
	}
	xprev = x;
    }
    /* Second round: if any keys were found, and no marks detected, purge now */
    if (anykey && !mark){
	x = NULL;
	xprev = x = NULL;
	while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
	    /* If it is key remove it here */
	    if (yt){
		if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
		    goto done;
		if (iskey && xml_purge(x) < 0)
		    goto done;
		x = xprev;
	    }
	    xprev = x; 
	}
    }
    retval = 0;
 done:
    if (upmark)
	*upmark = mark;
    return retval;
}
#else
/* This is optimized in the sense that xml_purge is replaced with xml_child_rm but it leaks memory,
 * in poarticualr attributes and namespace caches
 */
int
xml_tree_prune_flagged_sub(cxobj *xt, 
			   int    flag,
			   int    test,
			   int   *upmark)
{
    int        retval = -1;
    int        submark;
    int        mark;
    cxobj     *x;
    cxobj     *xprev;
    int        iskey;
    int        anykey=0;
    yang_stmt *yt;
    int        i;

    mark = 0;
    yt = xml_spec(xt); /* xan be null */
    x = NULL;
    xprev = x = NULL;
    i = 0;
    while ((x = xml_child_each(xt, x, -1)) != NULL) {
	i++;
	if (xml_type(x) != CX_ELMNT){
	    xprev = x;
	    continue;
	}
	if (xml_flag(x, flag) == test?flag:0){
	    /* Pass test */
	    mark++;
	    xprev = x;
	    continue; /* mark and stop here */
	}
	/* If it is key dont remove it yet (see second round) */
	if (yt){
	    if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
		goto done;
	    if (iskey){
		anykey++;
		xprev = x; /* skip if this is key */
		continue;
	    }
	}
	if (xml_tree_prune_flagged_sub(x, flag, test, &submark) < 0)
	    goto done;
	/* if xt is list and submark anywhere, then key subs are also marked
	 */
	if (submark)
	    mark++;
	else{ /* Safe with xml_child_each if last */
	    if (xml_child_rm(xt, i-1) < 0)
		goto done;
	    i--;
	    x = xprev;
	}
	xprev = x;
    }
    /* Second round: if any keys were found, and no marks detected, purge now */
    if (anykey && !mark){
	x = NULL;
	xprev = x = NULL;
	i = 0;
	while ((x = xml_child_each(xt, x, -1)) != NULL) {
	    i++;
	    if (xml_type(x) != CX_ELMNT){
		xprev = x;
		continue;
	    }
	    /* If it is key remove it here */
	    if (yt){
		if ((iskey = yang_key_match(yt, xml_name(x))) < 0)
		    goto done;
		if (xml_child_rm(xt, i-1) < 0)
		    goto done;
		i--;
		x = xprev;
	    }
	    xprev = x; 
	}
    }
    retval = 0;
 done:
    if (upmark)
	*upmark = mark;
    return retval;
}
#endif

/*! Prune everything that passes test
 * @param[in]   xt      XML tree with some node marked
 * @param[in]   flag    Which flag to test for
 * @param[in]   test    1: test that flag is set, 0: test that flag is not set
 * The function removes all branches that does not pass test
 * @code
 *    xml_tree_prune_flagged(xt, XML_FLAG_MARK, 1);
 * @endcode
 */
int
xml_tree_prune_flagged(cxobj *xt, 
		       int    flag,
		       int    test)
{
    int        retval = -1;
    cxobj     *x;
    cxobj     *xprev;

    x = NULL;
    xprev = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
	if (xml_flag(x, flag) == (test?flag:0)){ 	/* Pass test means purge */
	    if (xml_purge(x) < 0)
		goto done;
	    x = xprev;
	    continue; 
	}
	if (xml_tree_prune_flagged(x, flag, test) < 0)
	    goto done;
	xprev = x;
    }
    retval = 0;
 done:
    return retval;
}

/*! Add prefix:namespace pair to xml node, set cache, etc
 * @param[in]  x         XML node whose namespace should change
 * @param[in]  xp        XML node where namespace attribute should be declared (can be same)
 * @param[in]  prefix1   Use this prefix
 * @param[in]  namespace Use this namespace
 * @note x and xp must be different if x is an attribute and may be different otherwise
 */
static int
add_namespace(cxobj *x,
	      cxobj *xp,
	      char  *prefix,
	      char  *namespace)
{
    int    retval = -1;
    cxobj *xa = NULL;
    
    /* Add binding to x1p. We add to parent due to heurestics, so we dont
     * end up in adding it to large number of siblings 
     */
    if (nscache_set(x, prefix, namespace) < 0)
	goto done;
    /* Create xmlns attribute to x1p/x1 XXX same code v */
    if (prefix){
	if ((xa = xml_new(prefix, xp, CX_ATTR)) == NULL)
	    goto done;
	if (xml_prefix_set(xa, "xmlns") < 0)
	    goto done;
    }
    else{
	if ((xa = xml_new("xmlns", xp, CX_ATTR)) == NULL)
	    goto done;
    }
    if (xml_value_set(xa, namespace) < 0)
	goto done;
    xml_sort(xp); /* Ensure attr is first / XXX xml_insert? */
	
    retval = 0;
 done:
    return retval;
}

/*! Change namespace of XML node 
 *
 * @param[in]  x         XML node
 * @param[in]  ns        Change to this namespace (if ns does not exist in tree)
 * @param[in]  prefix    If change, use this prefix
 * @param      0         OK
 * @param     -1         Error
 */
int
xml_namespace_change(cxobj *x, 
                    char   *ns,
                    char   *prefix)
{
    int    retval = -1;
    char  *ns0 = NULL;     /* existing namespace */
    char  *prefix0 = NULL; /* existing prefix */
    cxobj *xp;

    ns0 = NULL;
    if (xml2ns(x, xml_prefix(x), &ns0) < 0)
       goto done;
    if (ns0 && strcmp(ns0, ns) == 0)
	goto ok; /* Already has right namespace */ 
    /* Is namespace already declared? */
    if (xml2prefix(x, ns, &prefix0) == 1){
       /* Yes it is declared and the prefix is prefix0 */
       if (xml_prefix_set(x, prefix0) < 0)
           goto done;
    }
    else{ /* Namespace does not exist, add it */
	/* Clear old prefix if any */
       if (xml_prefix_set(x, NULL) < 0)
           goto done;
       if (xml_type(x) == CX_ELMNT) /* If not element, do the namespace addition to the element */
	   xp = x;
       else
	   xp = xml_parent(x);
       if (add_namespace(x, xp, prefix, ns) < 0)
	   goto done;	   
       /* Add prefix to x, if any */
       if (prefix && xml_prefix_set(x, prefix) < 0)
	   goto done;
       
    }
 ok:
    retval = 0;
 done:
    return retval;
}

int
xml_default_create1(yang_stmt *y,
		    cxobj     *xt,
		    int        top,
		    cxobj    **xcp)
{
    int        retval = -1;
    char      *namespace;
    char      *prefix;
    int        ret;
    cxobj     *xc = NULL;

    if ((xc = xml_new(yang_argument_get(y), NULL, CX_ELMNT)) == NULL)
	goto done;
    xml_spec_set(xc, y);
    
    /* assign right prefix */
    if ((namespace = yang_find_mynamespace(y)) != NULL){
	prefix = NULL;
	if ((ret = xml2prefix(xt, namespace, &prefix)) < 0)
	    goto done;
	if (ret){ /* Namespace found, prefix returned in prefix */
	    if (xml_prefix_set(xc, prefix) < 0)
		goto done;
	}
	else{ /* Namespace does not exist in target, must add it w xmlns attr. 
		 use source prefix */
	    if (!top){
		if ((prefix = yang_find_myprefix(y)) == NULL){
		    clicon_err(OE_UNIX, errno, "strdup");
		    goto done;
		}		
	    }
	    if (add_namespace(xc, xc, prefix, namespace) < 0)
		goto done;
	    /* Add prefix to x, if any */
	    if (prefix && xml_prefix_set(xc, prefix) < 0)
		goto done;
	}
    }
    if (xml_addsub(xt, xc) < 0)
	goto done;
    *xcp = xc;
    retval = 0;
 done:
    return retval;
}

/*! Create leaf from default value
 *
 * @param[in]   yt      Yang spec
 * @param[in]   xt      XML tree
 * @param[in]   top     Use default namespace (if you create xmlns statement)
 * @retval      0       OK
 * @retval      -1      Error
 */
static int
xml_default_create(yang_stmt *y,
		   cxobj     *xt,
		   int        top)
{
    int        retval = -1;
    cxobj     *xc = NULL;
    cxobj     *xb;
    char      *str;
	
    if (xml_default_create1(y, xt, top, &xc) < 0)
	goto done;
    xml_flag_set(xc, XML_FLAG_DEFAULT);
    if ((xb = xml_new("body", xc, CX_BODY)) == NULL)
	goto done;
    if ((str = cv2str_dup(yang_cv_get(y))) == NULL){
	clicon_err(OE_UNIX, errno, "cv2str_dup");
	goto done;
    }
    if (xml_value_set(xb, str) < 0)
	goto done;
    free(str);
    retval = 0;
 done:
    return retval;
}

/*! Try to see if intermediate nodes are necessary for default values, create if so
 *
 * @param[in]   yt      Yang container (no-presence)
 * @param[out]  createp Need to create XML container
 * @retval      0       OK
 * @retval      -1      Error
 */
static int
xml_nopresence_try(yang_stmt *yt,
		   int       *createp)
{
    int        retval = -1;
    yang_stmt *y;
    
    if (yt == NULL || yang_keyword_get(yt) != Y_CONTAINER){
	clicon_err(OE_XML, EINVAL, "yt argument is not container");
	goto done;
    }
    *createp = 0;
    y = NULL;
    while ((y = yn_each(yt, y)) != NULL) {
	switch (yang_keyword_get(y)){
	case Y_LEAF:
	    if (!cv_flag(yang_cv_get(y), V_UNSET)){  /* Default value exists */
		/* Need to create container */
		*createp = 1;
		goto ok;
	    }
	    break;
	case Y_CONTAINER:
	    if (yang_find(y, Y_PRESENCE, NULL) == NULL){
		/* If this is non-presence, (and it does not exist in xt) call recursively 
		 * and create nodes if any default value exist first. Then continue and populate?
		 */
		if (xml_nopresence_try(y, createp) < 0)
		    goto done;
		if (*createp)
		    goto ok;
	    }
	    break;
	default:
	    break;
	} /* switch */
    }
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Ensure default values are set on (children of) one single xml node
 *
 * Not recursive, except in one case with one or several non-presence containers, in which case
 * XML containers may be created to host default values. That code may be a little too recursive.
 * @param[in]   yt      Yang spec
 * @param[in]   xt      XML tree (with yt as spec of xt, informally)
 * @param[in]   state   Set if global state, otherwise config
 * @retval      0       OK
 * @retval      -1      Error
 */
static int
xml_default1(yang_stmt *yt,
	     cxobj     *xt,
	     int        state)
{
    int        retval = -1;
    yang_stmt *yc;
    cxobj     *xc;
    int        top=0; /* Top symbol (set default namespace) */
    int        create = 0;

    if (xt == NULL){ /* No xml */
	clicon_err(OE_XML, EINVAL, "No XML argument");
	goto done;
    }
    switch (yang_keyword_get(yt)){
    case Y_MODULE:
    case Y_SUBMODULE:
	top++;
    case Y_CONTAINER: /* XXX maybe check for non-presence here as well */
    case Y_LIST:
    case Y_INPUT:
    case Y_OUTPUT:
	yc = NULL;
	while ((yc = yn_each(yt, yc)) != NULL) {
	    if (!state && !yang_config(yc)) 
		continue;
	    switch (yang_keyword_get(yc)){
	    case Y_LEAF:
		if (!cv_flag(yang_cv_get(yc), V_UNSET)){  /* Default value exists */
		    if (xml_find_type(xt, NULL, yang_argument_get(yc), CX_ELMNT) == NULL){
			/* No such child exist, create this leaf */
			if (xml_default_create(yc, xt, top) < 0)
			    goto done;
			xml_sort(xt);
		    }
		}
		break;
	    case Y_CONTAINER:
		if (yang_find(yc, Y_PRESENCE, NULL) == NULL){
		    /* If this is non-presence, (and it does not exist in xt) call 
		     * recursively and create nodes if any default value exist first. 
		     * Then continue and populate?
		     */
		    if (xml_find_type(xt, NULL, yang_argument_get(yc), CX_ELMNT) == NULL){
			/* No such container exist, recursively try if needed */
			if (xml_nopresence_try(yc, &create) < 0)
			    goto done;
			if (create){
			    /* Retval shows there is a default value need to create the 
			     * container */
			    if (xml_default_create1(yc, xt, top, &xc) < 0)
				goto done;
			    xml_sort(xt);
			    /* Then call it recursively */
			    if (xml_default1(yc, xc, state) < 0)
				goto done;
			}
		    }
		}
		break;
	    default:
		break;
	    }
	}
	break;
    default:
	break;
    } /* switch */
    retval = 0;
 done:
    return retval;
}

/*! Ensure default values are set on existing leaf children of this node
 * 
 * Assume yang is bound to the tree
 * @param[in]   xt      XML tree
 * @param[in]   state   If set expand defaults also for state data, otherwise only config
 * @retval      0       OK
 * @retval      -1      Error
 */
static int
xml_default(cxobj *xt,
	    int    state)
{
    int        retval = -1;
    yang_stmt *ys;
    
    if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){
	retval = 0;
	goto done;
    }
    if (xml_default1(ys, xt, state) < 0)
	goto done;
    retval = 0;
 done:
    return retval;
}

/*! Recursively fill in default values in an XML tree
 * @param[in]   xt      XML tree
 * @param[in]   state   If set expand defaults also for state data, otherwise only config
 * @retval      0       OK
 * @retval      -1      Error
 */
int
xml_default_recurse(cxobj *xn,
		    int    state)
{
    int        retval = -1;
    cxobj     *x;
    yang_stmt *y;
    
    if (xml_default(xn, state) < 0)
	goto done;
    x = NULL;
    while ((x = xml_child_each(xn, x, CX_ELMNT)) != NULL) {
	if ((y = (yang_stmt*)xml_spec(x)) != NULL){
	    if (!state && !yang_config(y))
		continue;
	}
	if (xml_default_recurse(x, state) < 0)
	    goto done;
    }
    retval = 0;
 done:
    return retval;
}

/*! Expand and set default values of global top-level on XML tree
 *
 * Not recursive, except in one case with one or several non-presence containers
 * @param[in]   xt      XML tree
 * @param[in]   yspec   Top-level YANG specification tree, all modules
 * @param[in]   state   Set if global state, otherwise config
 * @retval      0       OK
 * @retval      -1      Error
 */
static int
xml_global_defaults_create(cxobj     *xt,
			   yang_stmt *yspec,
			   int        state)

{
    int        retval = -1;
    yang_stmt *ymod = NULL;

    if (yspec == NULL || yang_keyword_get(yspec) != Y_SPEC){
	clicon_err(OE_XML, EINVAL, "yspec argument is not yang spec");
	goto done;
    }
    while ((ymod = yn_each(yspec, ymod)) != NULL) 
	if (xml_default1(ymod, xt, state) < 0)
	    goto done;
    retval = 0;
 done:
    return retval;
}

/*! Expand and set default values of global top-level on XML tree
 *
 * Not recursive, except in one case with one or several non-presence containers
 * @param[in]   h       Clixon handle
 * @param[in]   xt      XML tree, assume already filtered with xpath
 * @param[in]   xpath   Filter global defaults with this and merge with xt
 * @param[in]   yspec   Top-level YANG specification tree, all modules
 * @param[in]   state   Set if global state, otherwise config
 * @retval      0       OK
 * @retval      -1      Error
 * Uses cache?
 */
int
xml_global_defaults(clicon_handle h,
		    cxobj        *xt,
		    cvec         *nsc,
		    const char   *xpath,
		    yang_stmt    *yspec,
		    int           state)
{
    int        retval = -1;
    db_elmnt   de0 = {0,};
    db_elmnt  *de = NULL;
    cxobj     *xcache = NULL;
    cxobj     *xpart = NULL;
    cxobj    **xvec = NULL;
    size_t     xlen;
    int        i;
    cxobj     *x0;
    int        ret;
    char      *key;
    
    /* Use different keys for config and state */
    key = state ? "global-defaults-state" : "global-defaults-config";
    /* First get or compute global xml tree cache */
    if ((de = clicon_db_elmnt_get(h, key)) == NULL){
	/* Create it */
	if ((xcache = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
	    goto done;
	if (xml_global_defaults_create(xcache, yspec, state) < 0)
	    goto done;
	de0.de_xml = xcache;
	clicon_db_elmnt_set(h, key, &de0);
    }
    else
	xcache = de->de_xml;

    /* Here xcache has all global defaults. Now find the matching nodes 
     * XXX: nsc as 2nd argument
     */
    if (xpath_vec(xcache, nsc, "%s", &xvec, &xlen, xpath?xpath:"/") < 0)
	goto done;
    /* Iterate through match vector
     * For every node found in x0, mark the tree up to t1
     */
    for (i=0; i<xlen; i++){
	x0 = xvec[i];
	xml_flag_set(x0, XML_FLAG_MARK);
	xml_apply_ancestor(x0, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
    }
    /* Create a new tree and copy over the parts from the cache that matches xpath */
    if ((xpart = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
	goto done;
    if (xml_copy_marked(xcache, xpart) < 0) /* config */
	goto done;
    if (xml_apply(xcache, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
	goto done;
    if (xml_apply(xpart, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
	goto done;
    /* Merge global pruned tree with xt */
    if ((ret = xml_merge(xt, xpart, yspec, NULL)) < 1) /* XXX reason */
	goto done;
    retval = 0;
 done:
    if (xpart)
	xml_free(xpart);
    if (xvec)
	free(xvec);
    return retval;
}

/*! This node is a default set value or (recursively) a non-presence container
 * @retval 1  xt is a nopresence/default node (ie "virtual")
 * @retval 0  xt is not such a node
 */
int
xml_nopresence_default(cxobj *xt)
{
    cxobj     *xc;
    yang_stmt *yt;

    if ((yt = xml_spec(xt)) == NULL)
	return 0;
    switch (yang_keyword_get(yt)){
    case Y_CONTAINER:
	if (yang_find(yt, Y_PRESENCE, NULL))
	    return 0;
	break;
    case Y_LEAF:
	return xml_flag(xt, XML_FLAG_DEFAULT)?1:0;
	break;
    default:
	return 0;
    }
    xc = NULL;
    while ((xc = xml_child_each(xt, xc, CX_ELMNT)) != NULL) {
	if (xml_nopresence_default(xc) == 0)
	    return 0;
    }
    return 1;
}

/*! Remove xml container if it is non-presence and only contains default leafs
 * Called from xml_apply. Reason for marking is to delete it afterwords.
 * @param[in] x
 * @param[in] arg  (flag value)
 * @code
 *    if (xml_apply(xt, CX_ELMNT, xml_nopresence_default_mark, (void*)XML_FLAG_DEFAULT) < 0)
 *	err;
 *    if (xml_tree_prune_flagged(xt, XML_FLAG_DEFAULT, 1) < 0)
 *	goto done;
 * @endcode
 */
int
xml_nopresence_default_mark(cxobj *x,
			    void  *arg)
{
    if (xml_nopresence_default(x))
	xml_flag_set(x, (intptr_t)arg);
    return 0;
}

/*! Sanitize an xml tree: xml node has matching yang_stmt pointer 
 * @param[in]   xt      XML top of tree
 */
int
xml_sanity(cxobj *xt, 
	   void  *arg)
{
    int        retval = -1;
    yang_stmt *ys;
    char      *name;

    if ((ys = (yang_stmt*)xml_spec(xt)) == NULL){
	retval = 0;
	goto done;
    }
    name = xml_name(xt);
    if (strstr(yang_argument_get(ys), name)==NULL){
	clicon_err(OE_XML, 0, "xml node name '%s' does not match yang spec arg '%s'", 
		   name, yang_argument_get(ys));
	goto done;
    }
    retval = 0;
 done:
    return retval;
}

/*! Detect state data: Either mark or break on first occurence and return xerror
 * @param[in]   xt      XML tree 
 * @param[out]  xerr    If set return netconf error, abort and return if a state variable found
 * @retval      -1      Error
 * @retval      0       Status node found and return xerror
 * @retval      1       OK
 * Note that the behaviour is quite different if xerr is set or not,...
 */
int
xml_non_config_data(cxobj  *xt, 
		    cxobj **xerr) 
{
    int        retval = -1;
    cxobj     *x;
    yang_stmt *y;
    int        ret;
    cbuf      *cb = NULL;
    
    x = NULL;
    while ((x = xml_child_each(xt, x, CX_ELMNT)) != NULL) {
	if ((y = (yang_stmt*)xml_spec(x)) == NULL)
	    goto ok;
	if (!yang_config(y)){ /* config == false means state data */
	    if (xerr){        /* behaviour 1: return on error */
		if ((cb = cbuf_new()) == NULL){
		    clicon_err(OE_UNIX, errno, "cbuf_new");
		    goto done;
		}
		cprintf(cb, "module %s: state data node unexpected", yang_argument_get(ys_module(y)));
		if (netconf_bad_element_xml(xerr, "application", yang_argument_get(y), cbuf_get(cb)) < 0)
		    goto done;
		retval = 0;
		goto done;
	    }
	    xml_flag_set(x, XML_FLAG_MARK); /* behaviour 2: mark and continue */
	}
	if ((ret = xml_non_config_data(x, xerr)) < 0)
	    goto done;
	if (ret == 0){
	    retval = 0;
	    goto done;
	}
    }
 ok:
    retval = 1;
 done:
    if (cb)
	cbuf_free(cb);
    return retval;
}

/*! Given an XML node, build an xpath to root, internal function
 * @retval     0      OK
 * @retval    -1      Error. eg XML malformed
 */
static int
xml2xpath1(cxobj *x,
	   cbuf  *cb)
{
    int           retval = -1;
    cxobj        *xp;
    yang_stmt    *y = NULL;
    cvec         *cvk = NULL; /* vector of index keys */
    cg_var       *cvi;
    char         *keyname;
    cxobj        *xkey;
    cxobj        *xb;
    char         *b;
    enum rfc_6020 keyword;
    
    if ((xp = xml_parent(x)) == NULL)
	goto ok;
    xml2xpath1(xp, cb);
    /* XXX: sometimes there should be a /, sometimes not */
    cprintf(cb, "/%s", xml_name(x));
    if ((y = xml_spec(x)) != NULL){
	keyword = yang_keyword_get(y);
	switch (keyword){
	case Y_LEAF_LIST:
	    if ((b = xml_body(x)) != NULL)
		cprintf(cb, "[.=\"%s\"]", b);
	    else
		cprintf(cb, "[.=\"\"]");
	    break;
	case Y_LIST:
	    cvk = yang_cvec_get(y);
	    cvi = NULL;
	    while ((cvi = cvec_each(cvk, cvi)) != NULL) {
		keyname = cv_string_get(cvi);
		if ((xkey = xml_find(x, keyname)) == NULL)
		    goto done; /* No key in xml */
		if ((xb = xml_find(x, keyname)) == NULL)
		    goto done;
		b = xml_body(xb);
		cprintf(cb, "[%s=\"%s\"]", keyname, b?b:"");
	    }
	    break;
	default:
	    break;
	}
    }
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Given an XML node, build an xpath to root
 * Builds only unqualified xpaths, ie no predicates []
 * @param[in]  x      XML object
 * @param[out] xpath  Malloced xpath string. Need to free() after use
 * @retval     0      OK
 * @retval    -1      Error. (eg XML malformed)
 * @note x needs to be bound to YANG, see eg xml_bind_yang()
 */
int
xml2xpath(cxobj *x,
	  char **xpathp)
{
    int   retval = -1;
    cbuf *cb;
    char *xpath = NULL;

    if ((cb = cbuf_new()) == NULL){
	clicon_err(OE_XML, errno, "cbuf_new");
	goto done;
    }
    if (xml2xpath1(x, cb) < 0)
	goto done;
    /* XXX: see xpath in test statement,.. */
    xpath = cbuf_get(cb);
    if (xpathp){
	if ((*xpathp = strdup(xpath)) == NULL){
	    clicon_err(OE_UNIX, errno, "strdup");
	    goto done;
	}
	xpath = NULL;
    }
    retval = 0;
 done:
    if (cb)
	cbuf_free(cb);
    return retval;
}

/*! Check if the module tree x is in is assigned right XML namespace, assign if not
 * @param[in]  x   XML node
 *(0. You should probably find the XML root and apply this function to that.)
 * 1. Check which namespace x should have (via yang). This is correct namespace.
 * 2. Check which namespace x has via its XML tree
 * 3. If equal, OK
 * 4. Assign the correct namespace to the XML node
 * Assign default namespace to x. Create an "xmlns"
 * @code
 *    xml_yang_root(x, &xroot);
 *    xmlns_assign(xroot);
 * @endcode
 */
int
xmlns_assign(cxobj *x)
{
    int        retval = -1;
    yang_stmt *y;
    char      *ns_correct; /* correct uri */
    char      *ns_xml;     /* may be null or incorrect */

    if ((y = xml_spec(x)) == NULL){
	clicon_err(OE_YANG, ENOENT, "XML %s does not have yang spec", xml_name(x));
	goto done;
    }
    /* 1. Check which namespace x should have (via yang). This is correct namespace. */
    if ((ns_correct = yang_find_mynamespace(y)) == NULL){
	clicon_err(OE_YANG, ENOENT, "yang node %s does not have namespace", yang_argument_get(y));
	goto done;
    }
    /* 2. Check which namespace x has via its XML tree */
    if (xml2ns(x, NULL, &ns_xml) < 0)
	goto done;
    /* 3. If equal, OK, 4. Else, find root of XML tree */
    if (ns_xml && strcmp(ns_xml, ns_correct)==0)
	goto ok;
    /* 4. Assign the correct namespace */
    if (xmlns_set(x, NULL, ns_correct) < 0)
	goto done;
 ok:
    retval = 0;
 done:
    return retval;
}

/*! Given a src element node x0 and a target node x1, assign (optional) prefix and namespace
 * @see assign_namespace_element  this is a subroutine
 */
static int
assign_namespace(cxobj *x0, /* source */
		 cxobj *x1, /* target */
		 cxobj *x1p,
		 int    isroot,
		 char  *ns,
		 char  *prefix0)
{
    int        retval = -1;
    char      *prefix1 = NULL;
    char      *pexist = NULL;
    cvec      *nsc0 = NULL;
    cvec      *nsc = NULL;
    yang_stmt *y;
    
    /* 2a. Detect if namespace is declared in x1 target parent */
    if (xml2prefix(x1p, ns, &pexist) == 1){
	/* Yes, and it has prefix pexist */
	if (pexist){
	    if ((prefix1 = strdup(pexist)) == NULL){
		clicon_err(OE_UNIX, errno, "strdup");
		goto done;
	    }
	}
	else
	    prefix1 = NULL;
	/* 3. If yes, assign prefix to x1 */
	if (prefix1 && xml_prefix_set(x1, prefix1) < 0)
	    goto done;
	/* And copy namespace context from parent to child */
	if ((nsc0 = nscache_get_all(x1p)) != NULL){
	    if ((nsc = cvec_dup(nsc0)) == NULL){
		clicon_err(OE_UNIX, errno, "cvec_dup");
		goto done;
	    }
	    nscache_replace(x1, nsc);
	}
	/* Just in case */
	if (nscache_set(x1, prefix1, ns) < 0)
	    goto done;
    }
    else{ /* No, namespace does not exist in x1 _parent_ 
	   * Check if it is exists in x1 itself */
	if (xml2prefix(x1, ns, &pexist) == 1){
	    /* Yes it exists, but is it equal? */
	    if (clicon_strcmp(pexist, prefix0) == 0)
		; /* Equal, reuse */
	    else{ /* namespace exist, but not equal, use existing */
		/* Add prefix to x1, if any */
		if (pexist && xml_prefix_set(x1, pexist) < 0)
		    goto done;
	    }
	    goto ok; /* skip */
	}
	else { /* namespace does not exist in target x1, use source prefix 
		* use the prefix defined in the module
		*/
	    if (isroot){
		if (prefix0 && (prefix1 = strdup(prefix0)) == NULL){
		    clicon_err(OE_UNIX, errno, "strdup");
		    goto done;
		}
	    }
	    else{
		char *ptmp;
		if ((y = xml_spec(x0)) == NULL){
		    clicon_err(OE_YANG, ENOENT, "XML %s does not have yang spec",
			       xml_name(x0));
		    goto done;
		}
		/* Find local (imported) prefix for that module namespace */
		if (yang_find_prefix_by_namespace(y, ns, &ptmp) < 0)
		    goto done;
		if ((prefix1 = strdup(ptmp)) == NULL){
		    clicon_err(OE_UNIX, errno, "strdup");
		    goto done;
		}   

	    }
	}
	if (add_namespace(x1, x1, prefix1, ns) < 0)
	    goto done;
	if (prefix1 && xml_prefix_set(x1, prefix1) < 0)
	    goto done;	    
    }
 ok:
    /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
    retval = 0;
 done:
    if (prefix1)
	free(prefix1);
    return retval;
}

/*! Given a src element node x0 and a target node x1, assign (optional) prefix and namespace
 * @param[in]  x0       Source XML tree
 * @param[in]  x1       Target XML tree
 * @param[in]  x1p      Target XML tree parent
 * @retval     0        OK
 * @retval     -1       OK
 * 1. Find N=namespace(x0)
 * 2. Detect if N is declared in x1 parent
 * 3. If yes, assign prefix to x1
 * 4. If no, create new prefix/namespace binding and assign that to x1p (x1 if x1p is root)
 * 5. Add prefix to x1, if any
 * 6. Ensure x1 cache is updated
 * @note switch use of x0 and x1 compared to datastore text_modify
 * @see xml2ns
*/
int
assign_namespace_element(cxobj *x0, /* source */
			 cxobj *x1, /* target */
			 cxobj *x1p)
{
    int        retval = -1;
    char      *namespace = NULL;
    char      *prefix0 = NULL;;
    int        isroot;
    
    isroot = xml_parent(x1p)==NULL &&
	xml_flag(x1p, XML_FLAG_TOP) &&
	xml_prefix(x1p)==NULL;
    /* 1. Find N=namespace(x0) in x0 element */
    prefix0 = xml_prefix(x0);
    if (xml2ns(x0, prefix0, &namespace) < 0)
	goto done;
    if (namespace == NULL){
	clicon_err(OE_XML, ENOENT, "No namespace found for prefix:%s",
		   prefix0?prefix0:"NULL");
	goto done;
    }
    if (assign_namespace(x0, x1, x1p, isroot, namespace, prefix0) < 0)
	goto done;
    /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
    retval = 0;
 done:
    return retval;
}

/* If origin body has namespace definitions, copy them. The reason is that
 * some bodies rely on namespace prefixes, such as NACM path, but there is 
 * no way we can now this here.
 * However, this may lead to namespace collisions if these prefixes are not
 * canonical, and may collide with the assign_namespace_element() above (but that 
 * is for element sysmbols)
 */
int
assign_namespace_body(cxobj *x0, /* source */
		      char  *x0bstr,
		      cxobj *x1) /* target */
{
    int    retval = -1;
    char  *namespace = NULL;
    char  *name;
    char  *prefix;
    char  *prefix0 = NULL;;
    char  *pexisting = NULL;;
    cxobj *xa;
    
    xa = NULL;
    while ((xa = xml_child_each(x0, xa, CX_ATTR)) != NULL) {
	prefix = xml_prefix(xa);
	name = xml_name(xa);
	namespace = xml_value(xa);
	if ((strcmp(name, "xmlns")==0 && prefix==NULL) ||
	    (prefix != NULL && strcmp(prefix, "xmlns")==0)){
	    if (prefix == NULL)
		prefix0 = NULL;
	    else
		prefix0 = name;
	    if (strcmp(namespace, NETCONF_BASE_NAMESPACE) ==0 ||
		strcmp(namespace, YANG_XML_NAMESPACE) ==0)
		continue;
	    /* Detect if prefix:namespace is declared already? */
	    if (xml2prefix(x1, namespace, &pexisting) == 1){
		/* Yes, and it has prefix pexist */
		if (clicon_strcmp(pexisting, prefix0) ==0)
		    continue;
	    }
	    if (add_namespace(x1, x1, prefix0, namespace) < 0)
		goto done;
	}
    }
    /* 6. Ensure x1 cache is updated (I think it is done w xmlns_set above) */
    retval = 0;
 done:
    return retval;
}

/*! Merge a base tree x0 with x1 with yang spec y
 * @param[in]  x0  Base xml tree (can be NULL in add scenarios)
 * @param[in]  y0  Yang spec corresponding to xml-node x0. NULL if x0 is NULL
 * @param[in]  x0p Parent of x0
 * @param[in]  x1  xml tree which modifies base
 * @param[out] reason If retval=0 a malloced string
 * @retval     1      OK
 * @retval     0      Yang error, reason is set
 * @retval    -1      Error
 * Assume x0 and x1 are same on entry and that y is the spec
 */
static int
xml_merge1(cxobj              *x0,  /* the target */
	   yang_stmt          *y0,
	   cxobj              *x0p,
	   cxobj              *x1,  /* the source */
	   char              **reason)
{
    int        retval = -1;
    char      *x1cname; /* child name */
    cxobj     *x0c; /* base child */
    cxobj     *x0b; /* base body */
    cxobj     *x1c; /* mod child */
    char      *x1bstr; /* mod body string */
    yang_stmt *yc;  /* yang child */
    cbuf      *cbr = NULL; /* Reason buffer */
    int        ret;
    int        i;
    merge_twophase *twophase = NULL;
    int twophase_len;
    
    assert(x1 && xml_type(x1) == CX_ELMNT);
    assert(y0);

    if (x0 == NULL){
	cvec   *nsc = NULL;
	cg_var *cv;
	char   *ns;
	char   *px;
	nsc = cvec_dup(nscache_get_all(x1));
	if (xml_rm(x1) < 0)
	    goto done;
	/* This is to make the anydata case a little more robust, more could be done */
        if (xml_spec(x1) == NULL){
            if (xml_addsub(x0p, x1) < 0)
                goto done;
        }
        else
	    if (xml_insert(x0p, x1, INS_LAST, NULL, NULL) < 0)
		goto done;
	cv = NULL;
	while ((cv = cvec_each(nsc, cv)) != NULL){
	    px = cv_name_get(cv);
	    ns = cv_string_get(cv);
	    /* Check if it exists */
	    if (xml2prefix(x1, ns, NULL) == 0)
		if (xmlns_set(x1, px, ns) < 0)
		    goto done;
	}
	if (nsc)
	    cvec_free(nsc);
	goto ok;
    }
    if (yang_keyword_get(y0) == Y_LEAF_LIST || yang_keyword_get(y0) == Y_LEAF){
	x1bstr = xml_body(x1);
	if (x1bstr){
	    if ((x0b = xml_body_get(x0)) == NULL){
		if ((x0b = xml_new("body", x0, CX_BODY)) == NULL)
		    goto done; 
	    }
	    if (xml_value_set(x0b, x1bstr) < 0)
		goto done;
	}
	if (xml_parent(x0) == NULL &&
	    xml_insert(x0p, x0, INS_LAST, NULL, NULL) < 0) 
	    goto done;
	if (assign_namespace_element(x1, x0, x0p) < 0)
	    goto done;
    } /* if LEAF|LEAF_LIST */
    else { /* eg Y_CONTAINER, Y_LIST  */
	if (assign_namespace_element(x1, x0, x0p) < 0)
	    goto done;
	twophase_len = xml_child_nr(x1);
	if ((twophase = calloc(twophase_len, sizeof(*twophase))) == NULL){
	    clicon_err(OE_UNIX, errno, "calloc");
	    goto done;
	}
	i = 0;
	/* Loop through children of the modification tree */
	x1c = NULL;
	while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
	    x1cname = xml_name(x1c);
	    /* Get yang spec of the child */
	    if ((yc = yang_find_datanode(y0, x1cname)) == NULL){
		if (reason){
		    if ((cbr = cbuf_new()) == NULL){
			clicon_err(OE_XML, errno, "cbuf_new");
			goto done;
		    }
		    cprintf(cbr, "XML node %s/%s has no corresponding yang specification (Invalid XML or wrong Yang spec?", xml_name(x1), x1cname);
		    if ((*reason = strdup(cbuf_get(cbr))) == NULL){
			clicon_err(OE_UNIX, errno, "strdup");
			goto done;
		    }
		}
		goto fail;
	    }
	    /* See if there is a corresponding node in the base tree */
	    x0c = NULL;
	    if (yc && match_base_child(x0, x1c, yc, &x0c) < 0)
		goto done;
	    /* If x0 already has a value, do not replace it with a default value in x1 */
	    if (x0c && xml_flag(x1c, XML_FLAG_DEFAULT))
		continue;
	    /* Save x0c, x1c, yc and merge in second wave, so that x1c entries dont "interfer"
	     * with itself, ie that later searches are among earlier objects already added
	     * to x0 */
	    twophase[i].mt_x0c = x0c;
	    twophase[i].mt_x1c = x1c;
	    twophase[i].mt_yc  = yc;
	    i++;
	} /* while */
	twophase_len = i; /* Inital length included non-elements */
	/* Second run where actual merging is done 
	 * Loop through children of the modification tree */
	for (i=0; i<twophase_len; i++){
	    assert(twophase[i].mt_x1c);
	    if ((ret = xml_merge1(twophase[i].mt_x0c,
			   twophase[i].mt_yc,
			   x0,
			   twophase[i].mt_x1c,
				  reason)) < 0)
		goto done;
	    if (ret == 0)
		goto fail;
	}
	if (xml_parent(x0) == NULL &&
	    xml_insert(x0p, x0, INS_LAST, NULL, NULL) < 0) 
	    goto done;
    } /* else Y_CONTAINER  */
 ok:
    retval = 1;
 done:
    if (twophase)
	free(twophase);
    if (cbr)
	cbuf_free(cbr);
    return retval;
 fail:
    retval = 0;
    goto done;
}

/*! Merge XML trees x1 into x0 according to yang spec yspec
 * @param[in]  x0     Base xml tree (can be NULL in add scenarios)
 * @param[in]  x1     xml tree which modifies base
 * @param[in]  yspec  Yang spec
 * @param[out] reason If retval=0, reason is set. Malloced. Needs to be freed by caller
 * @retval     1      OK
 * @retval     0      Yang error, reason is set
 * @retval    -1      Error
 * @note both x0 and x1 need to be top-level trees
 */
int
xml_merge(cxobj     *x0,
	  cxobj     *x1,
	  yang_stmt *yspec,
	  char     **reason)
{
    int        retval = -1;
    char      *x1cname; /* child name */
    cxobj     *x0c; /* base child */
    cxobj     *x1c; /* mod child */
    yang_stmt *yc;
    yang_stmt *ymod;
    cbuf      *cbr = NULL; /* Reason buffer */
    int        i;
    merge_twophase *twophase = NULL;
    int        twophase_len;
    int        ret;

    if (x0 == NULL || x1 == NULL){
	clicon_err(OE_UNIX, EINVAL, "parameters x0 or x1 is NULL");
	goto done;
    }
    twophase_len = xml_child_nr(x1);
    if ((twophase = calloc(twophase_len, sizeof(*twophase))) == NULL){
	clicon_err(OE_UNIX, errno, "calloc");
	goto done;
    }
    /* Loop through children of the modification tree */
    i = 0;
    x1c = NULL;
    while ((x1c = xml_child_each(x1, x1c, CX_ELMNT)) != NULL) {
	x1cname = xml_name(x1c);
	if ((ys_module_by_xml(yspec, x1c, &ymod)) < 0)
	    goto done;
	if (ymod == NULL){
	    if (reason &&
		(*reason = strdup("Namespace not found or yang spec not loaded")) == NULL){
		    clicon_err(OE_UNIX, errno, "strdup");
		    goto done;
	    }
	    goto fail;	
	}
	/* Get yang spec of the child */
	if ((yc = yang_find_datanode(ymod, x1cname)) == NULL){
	    if (reason){
		if ((cbr = cbuf_new()) == NULL){
		    clicon_err(OE_XML, errno, "cbuf_new");
		    goto done;
		}
		cprintf(cbr, "XML node %s/%s has no corresponding yang specification (Invalid XML or wrong Yang spec?)", xml_name(x1), x1cname);
		if ((*reason = strdup(cbuf_get(cbr))) == NULL){
		    clicon_err(OE_UNIX, errno, "strdup");
		    goto done;
		}
	    }
	    goto fail;
	}
	x0c = NULL;
	/* See if there is a corresponding node (x1c) in the base tree (x0) */
	if (yc && match_base_child(x0, x1c, yc, &x0c) < 0)
	    goto done;
	/* If x0 already has a value, do not replace it with a default value in x1 */
	if (x0c && xml_flag(x1c, XML_FLAG_DEFAULT))
	    continue;
	/* Save x0c, x1c, yc and merge in second wave, so that x1c entries don't "interfere"
	 * with itself, ie that later searches are among earlier objects already added
	 * to x0 */
	twophase[i].mt_x0c = x0c;
	twophase[i].mt_x1c = x1c;
	twophase[i].mt_yc  = yc;
	i++;
    }
    twophase_len = i; /* Inital length included non-elements */
    /* Second run where actual merging is done 
     * Loop through children of the modification tree */
    for (i=0; i<twophase_len; i++){
	assert(twophase[i].mt_x1c);
	if ((ret = xml_merge1(twophase[i].mt_x0c,
			      twophase[i].mt_yc,
			      x0,
			      twophase[i].mt_x1c,
			      reason)) < 0)
	    goto done;
	if (ret == 0)
	    goto fail;
    }
    retval = 1; /* OK */
 done:
    if (twophase)
	free(twophase);
    if (cbr)
	cbuf_free(cbr);
    return retval;
 fail:
    retval = 0;
    goto done;
}

/*! Get integer value from xml node from yang enumeration 
 * @param[in]  node XML node in a tree
 * @param[out] val  Integer value returned
 * @retval     0    OK, value parsed
 * @retval    -1    Error, value not obtained or parsed, no reason given
 * @note assume yang spec set
 * @note The function only returns assigned values, but according to RFC:
   If a value is not specified, then one will be automatically assigned.
   If the "enum" substatement is the first one defined, the assigned
   value is zero (0); otherwise, the assigned value is one greater than
   the current highest enum value (i.e., the highest enum value,
   * Thanks: Matthew Smith
 */
int
yang_enum_int_value(cxobj   *node, 
		    int32_t *val)
{
    int retval = -1;
    yang_stmt *yspec;
    yang_stmt *ys;
    yang_stmt *ytype;
    yang_stmt *yrestype;  /* resolved type */
    yang_stmt *yenum; 
    yang_stmt *yval; 
    char      *reason = NULL;

    if (node == NULL)
	goto done;
    if ((ys = (yang_stmt *) xml_spec(node)) == NULL)
	goto done;
    if ((yspec = ys_spec(ys)) == NULL)
	goto done;
    if ((ytype = yang_find(ys, Y_TYPE, NULL)) == NULL)
	goto done;
    if (yang_type_resolve(ys, ys, ytype, &yrestype, 
			  NULL, NULL, NULL, NULL, NULL) < 0)
	goto done;
    if (yrestype==NULL || strcmp(yang_argument_get(yrestype), "enumeration"))
	goto done;
    if ((yenum = yang_find(yrestype, Y_ENUM, xml_body(node))) == NULL)
	goto done;
    /* Should assign value if yval not found */
    if ((yval = yang_find(yenum, Y_VALUE, NULL)) == NULL)
	goto done;
    /* reason is string containing why int could not be parsed */
    if (parse_int32(yang_argument_get(yval), val, &reason) < 0)
	goto done;
    retval = 0;
done:
    return retval;
}


/*! Given XML tree x0 with marked nodes, copy marked nodes to new tree x1
 * Two marks are used: XML_FLAG_MARK and XML_FLAG_CHANGE
 *
 * The algorithm works as following:
 * (1) Copy individual nodes marked with XML_FLAG_CHANGE 
 * until nodes marked with XML_FLAG_MARK are reached, where 
 * (2) the complete subtree of that node is copied. 
 * (3) Special case: key nodes in lists are copied if any node in list is marked
 *  @note you may want to check:!yang_config(ys)
 */
int
xml_copy_marked(cxobj *x0, 
		cxobj *x1)
{
    int        retval = -1;
    int        mark;
    cxobj     *x;
    cxobj     *xcopy;
    int        iskey;
    yang_stmt *yt;
    char      *name;
    char      *prefix;

    assert(x0 && x1);
    yt = xml_spec(x0); /* can be null */
    xml_spec_set(x1, yt);
   /* Copy prefix*/
    if ((prefix = xml_prefix(x0)) != NULL)
	if (xml_prefix_set(x1, prefix) < 0)
	    goto done;
    
    /* Copy all attributes */
    x = NULL;
    while ((x = xml_child_each(x0, x, CX_ATTR)) != NULL) {
	name = xml_name(x);
	if ((xcopy = xml_new(name, x1, CX_ATTR)) == NULL)
	    goto done;
	if (xml_copy(x, xcopy) < 0) 
	    goto done;
    }

    /* Go through children to detect any marked nodes:
     * (3) Special case: key nodes in lists are copied if any 
     * node in list is marked
     */
    mark = 0;
    x = NULL;
    while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) {
	if (xml_flag(x, XML_FLAG_MARK|XML_FLAG_CHANGE)){
	    mark++;
	    break;
	}
    }
    x = NULL;
    while ((x = xml_child_each(x0, x, CX_ELMNT)) != NULL) {
	name = xml_name(x);
	if (xml_flag(x, XML_FLAG_MARK)){
	    /* (2) the complete subtree of that node is copied. */
	    if ((xcopy = xml_new(name, x1, CX_ELMNT)) == NULL)
		goto done;
	    if (xml_copy(x, xcopy) < 0) 
		goto done;
	    continue; 
	}
	if (xml_flag(x, XML_FLAG_CHANGE)){
	    /*  Copy individual nodes marked with XML_FLAG_CHANGE */
	    if ((xcopy = xml_new(name, x1, CX_ELMNT)) == NULL)
		goto done;
	    if (xml_copy_marked(x, xcopy) < 0) /*  */
		goto done;
	}
	/* (3) Special case: key nodes in lists are copied if any 
	 * node in list is marked */
	if (mark && yt && yang_keyword_get(yt) == Y_LIST){
	    /* XXX: I think yang_key_match is suboptimal here */
	    if ((iskey = yang_key_match(yt, name)) < 0)
		goto done;
	    if (iskey){
		if ((xcopy = xml_new(name, x1, CX_ELMNT)) == NULL)
		    goto done;
		if (xml_copy(x, xcopy) < 0) 
		    goto done;
	    }
	}
    }
    retval = 0;
 done:
    return retval;
}