* Fixed extension/unknown problem shown in latest openconfig where other than a single space was used between the unknown identifier and string

* Removed skiplist of openconfig/yangmodels tests to inside the test

CHANGELOG.md

@@ -117,6 +117,7 @@ Developers may need to change their code
 
 ### Corrected Bugs
 
+* Fixed extension/unknown problem shown in latest openconfig where other than a single space was used between the unknown identifier and string
 * Fixed: [Augment that reference a submodule as target node fails #178](https://github.com/clicon/clixon/issues/178)
 * Fixed a memory error that was reported in slack by Pawel Maslanka
   * The crash printout was: `realloc(): invalid next size Aborted`
@@ -494,6 +495,7 @@ Thanks to everyone at Netgate for making this possible
 
 ### C-API changes on existing features (For developers)
 
+* Removed `xsock0` parameter from `clicon_rpc_msg()`, use `clicon_rpc_msg_persistent()` instead
 * Length of xml vector in many structs changed from `size_t` to `int`since it is a vector size, not byte size.
   * Example: `transaction_data_t`
 * `xml_merge()` changed to use 3-value return: 1:OK, 0:Yang failed, -1: Error

docker/main/startsystem.sh

@@ -60,7 +60,7 @@ echo "$STORE" > /usr/local/var/example/running_db
 # - NOTE all restconf tests skipped which makes these tests very constrained
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
-SKIPLIST="test_api.sh test_c++.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh test_augment.sh test_choice.sh test_identity.sh test_nacm_datanode_read.sh test_nacm_datanode.sh test_nacm_datanode_write.sh test_nacm_default.sh test_nacm_ext.sh test_nacm_module_read.sh test_nacm_module_write.sh test_nacm_protocol.sh test_nacm.sh test_nacm_recovery.sh test_perf.sh test_perf_state_only.sh test_perf_state.sh test_restconf2.sh test_restconf_err.sh test_restconf_jukebox.sh test_restconf_listkey.sh test_restconf_patch.sh test_restconf.sh test_restconf_startup.sh test_rpc.sh test_ssl_certs.sh test_stream.sh test_submodule.sh test_upgrade_auto.sh test_upgrade_interfaces.sh test_upgrade_repair.sh test_yang_namespace.sh"
+SKIPLIST="test_api.sh test_c++.sh test_install.sh test_privileges.sh test_augment.sh test_choice.sh test_identity.sh test_nacm_datanode_read.sh test_nacm_datanode.sh test_nacm_datanode_write.sh test_nacm_default.sh test_nacm_ext.sh test_nacm_module_read.sh test_nacm_module_write.sh test_nacm_protocol.sh test_nacm.sh test_nacm_recovery.sh test_perf.sh test_perf_state_only.sh test_perf_state.sh test_restconf2.sh test_restconf_err.sh test_restconf_jukebox.sh test_restconf_listkey.sh test_restconf_patch.sh test_restconf.sh test_restconf_startup.sh test_rpc.sh test_ssl_certs.sh test_stream.sh test_submodule.sh test_upgrade_auto.sh test_upgrade_interfaces.sh test_upgrade_repair.sh test_yang_namespace.sh"
 #IETFRFC=
 EOF
 

docker/main/startsystem_evhtp.sh

@@ -64,7 +64,7 @@ echo "$STORE" > /usr/local/var/example/running_db
 # - test_install.sh since you dont have the make environment
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
-SKIPLIST="test_api.sh test_client.sh test_c++.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh"
+SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh"
 RCPROTO=https
 #IETFRFC=
 EOF

docker/main/startsystem_fcgi.sh

@@ -88,7 +88,7 @@ EOF
 # - test_order.sh XXX this is a bug need debugging
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
-SKIPLIST="test_api.sh test_client.sh test_c++.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh"
+SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh"
 #IETFRFC=
 EOF
 

fuzz/backend/README.md

@@ -41,6 +41,7 @@ Make a modification to how Clixon sends internal messages in `include/clixon_cus
 ```
   #define CLIXON_PROTO_PLAIN
 ```
+(Note this is obsolete)
 
 Build clixon statically with the afl-clang compiler:
 ```

include/clixon_custom.h

@@ -92,14 +92,3 @@
  * clixon-4.4
  */
 #define STATE_ORDERED_BY_SYSTEM
-
-/*! Make internal XML protocol use plain strings instead of binary header
- * Experimental
- * This is only for testing, a specific usecase is fuzzing as described in fuzz/backend
- * Note session-ids are not handled properly (a bunch of other things too)
- * This could be mitigated by sending the session-id as an attribute
- * But doing this one should probably revise/remove the code around clicon_msg_encode/decode
- * which currently is somewhat hardwired. I.e., it may be difficult to have both variants as ifdef:s
- * and you may consider replacing the old code altogether.
- */
-#undef CLIXON_PROTO_PLAIN

lib/src/clixon_yang.c

@@ -2077,11 +2077,11 @@ ys_populate_unknown(clicon_handle h,
     if (nodeid_split(yang_argument_get(ys), &prefix, &id) < 0)
 	goto done;
     if ((ymod = yang_find_module_by_prefix(ys, prefix)) == NULL){
-	clicon_err(OE_YANG, ENOENT, "Extension %s:%s, module not found", prefix, id);
+	clicon_err(OE_YANG, ENOENT, "Extension \"%s:%s\", module not found", prefix, id);
 	goto done;
     }
     if ((yext = yang_find(ymod, Y_EXTENSION, id)) == NULL){
-	clicon_err(OE_YANG, ENOENT, "Extension %s:%s not found", prefix, id);
+	clicon_err(OE_YANG, ENOENT, "Extension \"%s:%s\" not found", prefix, id);
 	goto done;
     }
     /* Optional argument (only if "argument") - save it in ys_cv */

lib/src/clixon_yang_parse.l

@@ -102,15 +102,16 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 %s COMMENT1
 %s COMMENT2
 %s UNKNOWN
+%s UNKNOWN2
 
 %%
  /* Common tokens */
 <KEYWORD,BOOLEAN,INTEGER,STRARG,STRING>[ \t]            
-<KEYWORD,STRING,UNKNOWN,COMMENT2><<EOF>> { return MY_EOF; }
+<KEYWORD,STRING,UNKNOWN,UNKNOWN2,COMMENT2><<EOF>> { return MY_EOF; }
-<KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,COMMENT1,UNKNOWN>\n   { _YY->yy_linenum++; }
+<KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,COMMENT1,UNKNOWN,UNKNOWN2>\n   { _YY->yy_linenum++; }
-<KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,COMMENT1,UNKNOWN>\r   
+<KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,COMMENT1,UNKNOWN,UNKNOWN2>\r   
-<KEYWORD,STRING>"/*"    { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT1); }
+<KEYWORD,STRING,UNKNOWN,UNKNOWN2>"/*"    { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT1); }
-<KEYWORD,STRING>"//"    { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT2); }
+<KEYWORD,STRING,UNKNOWN,UNKNOWN2>"//"    { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT2); }
 
 
 <KEYWORD>input            { return K_INPUT; } /* No argument */
@@ -198,10 +199,16 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 
 <UNKNOWN>:                { return *yytext; }
 <UNKNOWN>;                { BEGIN(KEYWORD); return *yytext; }
-<UNKNOWN>\"               { _YY->yy_lex_string_state =UNKNOWN; BEGIN(STRINGDQ); return *yytext; }
 <UNKNOWN>\{               { BEGIN(KEYWORD); return *yytext; }
-<UNKNOWN>[ \t]+           { return SEP; }
+<UNKNOWN>[ \t\n]+         { BEGIN(UNKNOWN2);return SEP; }
-<UNKNOWN>[^{";: \t]+      { clixon_yang_parselval.string = strdup(yytext);
+<UNKNOWN>[^{"';: \t\n]+   { clixon_yang_parselval.string = strdup(yytext);
+                            return CHARS; }
+
+<UNKNOWN2>;                { BEGIN(KEYWORD); return *yytext; }
+<UNKNOWN2>\"               { _YY->yy_lex_string_state =STRING; BEGIN(STRINGDQ); return *yytext; }
+<UNKNOWN2>\'               { _YY->yy_lex_string_state =STRING; BEGIN(STRINGSQ); return SQ; }
+<UNKNOWN2>\{               { BEGIN(KEYWORD); return *yytext; }
+<UNKNOWN2>[^{"'; \t\n]+    { clixon_yang_parselval.string = strdup(yytext);
                              return CHARS; }
 
 <BOOLEAN>true             { clixon_yang_parselval.string = strdup(yytext);
@@ -229,7 +236,7 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 <STRING>\"                { _YY->yy_lex_string_state =STRING; BEGIN(STRINGDQ); return *yytext; }
 <STRING>\'                { _YY->yy_lex_string_state =STRING; BEGIN(STRINGSQ); return SQ; }
 <STRING>\+                { return *yytext; }
-<STRING>.                 { clixon_yang_parselval.string = strdup(yytext);
+<STRING>[^\"\'\+\{\;\n \t]+    { clixon_yang_parselval.string = strdup(yytext); /* XXX [.]+ */
                             return CHARS;}
 
 <STRINGDQ>\\              { _YY->yy_lex_state = STRINGDQ; BEGIN(DQESC); }

lib/src/clixon_yang_parse.y

@@ -193,11 +193,16 @@
 #include "clixon_yang_parse_lib.h"
 #include "clixon_yang_parse.h"
 
-/* Enable for debugging, steals some cycles otherwise */
+/* Best debugging is to enable PARSE_DEBUG below and add -d to the LEX compile statement in the Makefile
+ * And then run the testcase with -D 1
+ * Disable it to stop any calls to clicon_debug. Having it on by default would mean very large debug outputs.
+ */
 #if 0
 #define _PARSE_DEBUG(s) clicon_debug(1,(s))
+#define _PARSE_DEBUG1(s, s1) clicon_debug(1,(s), (s1))
 #else
 #define _PARSE_DEBUG(s)
+#define _PARSE_DEBUG1(s, s1)
 #endif
     
 extern int clixon_yang_parseget_lineno  (void);
@@ -1567,12 +1572,12 @@ deviate_substmt : type_stmt         { _PARSE_DEBUG("deviate-substmt -> type-stmt
 unknown_stmt  : ustring ':' ustring optsep ';'
                  { char *id; if ((id=string_del_join($1, ":", $3)) == NULL) _YYERROR("unknown_stmt");
 		   if (ysp_add(_yy, Y_UNKNOWN, id, NULL) == NULL) _YYERROR("unknown_stmt"); 
-		   _PARSE_DEBUG("unknown-stmt -> ustring : ustring");
+		   _PARSE_DEBUG("unknown-stmt -> ustring : ustring ;");
 	       }
               | ustring ':' ustring SEP string optsep ';'
 	        { char *id; if ((id=string_del_join($1, ":", $3)) == NULL) _YYERROR("unknown_stmt");
-		   if (ysp_add(_yy, Y_UNKNOWN, id, $5) == NULL){ _YYERROR("unknwon_stmt"); }
+		   if (ysp_add(_yy, Y_UNKNOWN, id, $5) == NULL){ _YYERROR("unknown_stmt"); }
-		   _PARSE_DEBUG("unknown-stmt -> ustring : ustring string");
+		   _PARSE_DEBUG("unknown-stmt -> ustring : ustring SEP string ;");
 	       }
               | ustring ':' ustring optsep
                  { char *id; if ((id=string_del_join($1, ":", $3)) == NULL) _YYERROR("unknown_stmt");
@@ -1744,7 +1749,7 @@ qstrings      : qstrings '+' qstring
 			 $$ = realloc($1, len + strlen($3) + 1); 
 			 sprintf($$+len, "%s", $3);
 			 free($3); 
-			 _PARSE_DEBUG("qstrings-> qstrings + qstring"); 
+			 _PARSE_DEBUG("qstrings-> qstrings '+' qstring"); 
 		     }
               | qstring    
                      { $$=$1;
@@ -1752,13 +1757,13 @@ qstrings      : qstrings '+' qstring
               ;
 
 qstring        : '"' ustring '"'  { $$=$2;
-		   _PARSE_DEBUG("string-> \" ustring \"");}
+		   _PARSE_DEBUG("qstring-> \" ustring \"");}
                | '"' '"'  { $$=strdup("");
-		   _PARSE_DEBUG("string-> \"  \"");} 
+		   _PARSE_DEBUG("qstring-> \"  \"");} 
                | SQ ustring SQ  { $$=$2;
-		   _PARSE_DEBUG("string-> ' ustring '"); }
+		   _PARSE_DEBUG("qstring-> ' ustring '"); }
                | SQ SQ  { $$=strdup("");
-		   _PARSE_DEBUG("string-> '  '");} 
+		   _PARSE_DEBUG("qstring-> '  '");} 
                ;
 
 /* unquoted string */
@@ -1767,11 +1772,11 @@ ustring       : ustring CHARS
 			 int len = strlen($1);
 			 $$ = realloc($1, len+strlen($2) + 1);
 			 sprintf($$+len, "%s", $2); 
+ 			 _PARSE_DEBUG1("ustring-> string + CHARS(%s)", $2); 
 			 free($2);
- 			 _PARSE_DEBUG("ustring-> string + CHARS"); 
 		     }
               | CHARS 
-	             {$$=$1; } 
+	              {  _PARSE_DEBUG1("ustring-> CHARS(%s)", $1); $$=$1; } 
               ;
 
 abs_schema_nodeid : abs_schema_nodeid '/' node_identifier
@@ -1861,4 +1866,3 @@ stmtend        : ';'
                ;
 
 %%
-

test/test_openconfig.sh

@@ -14,6 +14,7 @@ fyang=$dir/test.yang
 new "openconfig"
 if [ ! -d "$OPENCONFIG" ]; then
 #    err "Hmm Openconfig dir does not seem to exist, try git clone https://github.com/openconfig/public?"
+    echo "...skipped: OPENCONFIG not set"
     if [ "$s" = $0 ]; then exit 0; else return 0; fi
 fi
 

test/test_yang_extension.sh

@@ -82,6 +82,21 @@ module $APPNAME{
    ex:e4 arg1{
       uses bar;
    }
+   
+   extension posix-pattern {
+      argument "pattern";
+   }
+   typedef dotted-quad {
+      description "Only present for complex parsing of unknown-stmt";
+      type string {
+         pattern 
+             "[a-f]" + "[0-9]";
+         ex:posix-pattern
+          // Strictly this comment is not supported if you see RFC syntax with only a SEP
+          // in unknwon-stmt: identifier [sep string]
+            '[f-w]' + '[o-q]';
+      }
+   }
 }
 EOF
 
@@ -110,7 +125,7 @@ new "Add extension bar (is implemented)"
 expecteof "$clixon_netconf -qf $cfg -D $DBG" 0 "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><bar xmlns=\"urn:example:clixon\">a string</bar></config></edit-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><ok/></rpc-reply>]]>]]>"
 
 new "netconf get config"
-expecteof "$clixon_netconf -qf $cfg -D $DBG" 0 "<rpc $DEFAULTNS><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><data><bar xmlns=\"urn:example:clixon\">a string</bar></data></rpc-reply>]]>]]>$"
+expecteof "$clixon_netconf -qf $cfg -D $DBG" 0 "<rpc $DEFAULTNS><get-config><source><candidate/></source></get-config></rpc>]]>]]>" "^<rpc-reply $DEFAULTNS><data><bar xmlns=\"urn:example:clixon\">a string</bar></data></rpc-reply>]]>]]>"
 
 if [ $BE -ne 0 ]; then
     new "Kill backend"

test/test_yang_models_ieee.sh

@@ -17,6 +17,7 @@ fyang=$dir/test.yang
 
 if [ ! -d "$YANGMODELS" ]; then
 #    err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?"
+    echo "...skipped: YANGMODELS not set"
     if [ "$s" = $0 ]; then exit 0; else return 0; fi
 fi
 

test/test_yang_models_ietf.sh

@@ -18,6 +18,7 @@ fyang=$dir/test.yang
 YANGMODELS=/home/olof/tmp/yang
 if [ ! -d "$YANGMODELS" ]; then
 #    err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?"
+    echo "...skipped: YANGMODELS not set"
     if [ "$s" = $0 ]; then exit 0; else return 0; fi
 fi