diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e50f925..f57359ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,7 @@ Developers may need to change their code ### Corrected Bugs +* Fixed extension/unknown problem shown in latest openconfig where other than a single space was used between the unknown identifier and string * Fixed: [Augment that reference a submodule as target node fails #178](https://github.com/clicon/clixon/issues/178) * Fixed a memory error that was reported in slack by Pawel Maslanka * The crash printout was: `realloc(): invalid next size Aborted` @@ -494,6 +495,7 @@ Thanks to everyone at Netgate for making this possible ### C-API changes on existing features (For developers) +* Removed `xsock0` parameter from `clicon_rpc_msg()`, use `clicon_rpc_msg_persistent()` instead * Length of xml vector in many structs changed from `size_t` to `int`since it is a vector size, not byte size. * Example: `transaction_data_t` * `xml_merge()` changed to use 3-value return: 1:OK, 0:Yang failed, -1: Error diff --git a/docker/main/startsystem.sh b/docker/main/startsystem.sh index 66f5e3aa..526c8efc 100755 --- a/docker/main/startsystem.sh +++ b/docker/main/startsystem.sh @@ -60,7 +60,7 @@ echo "$STORE" > /usr/local/var/example/running_db # - NOTE all restconf tests skipped which makes these tests very constrained cat < /usr/local/bin/test/site.sh # Add your local site specific env variables (or tests) here. -SKIPLIST="test_api.sh test_c++.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh test_augment.sh test_choice.sh test_identity.sh test_nacm_datanode_read.sh test_nacm_datanode.sh test_nacm_datanode_write.sh test_nacm_default.sh test_nacm_ext.sh test_nacm_module_read.sh test_nacm_module_write.sh test_nacm_protocol.sh test_nacm.sh test_nacm_recovery.sh test_perf.sh test_perf_state_only.sh test_perf_state.sh test_restconf2.sh test_restconf_err.sh test_restconf_jukebox.sh test_restconf_listkey.sh test_restconf_patch.sh test_restconf.sh test_restconf_startup.sh test_rpc.sh test_ssl_certs.sh test_stream.sh test_submodule.sh test_upgrade_auto.sh test_upgrade_interfaces.sh test_upgrade_repair.sh test_yang_namespace.sh" +SKIPLIST="test_api.sh test_c++.sh test_install.sh test_privileges.sh test_augment.sh test_choice.sh test_identity.sh test_nacm_datanode_read.sh test_nacm_datanode.sh test_nacm_datanode_write.sh test_nacm_default.sh test_nacm_ext.sh test_nacm_module_read.sh test_nacm_module_write.sh test_nacm_protocol.sh test_nacm.sh test_nacm_recovery.sh test_perf.sh test_perf_state_only.sh test_perf_state.sh test_restconf2.sh test_restconf_err.sh test_restconf_jukebox.sh test_restconf_listkey.sh test_restconf_patch.sh test_restconf.sh test_restconf_startup.sh test_rpc.sh test_ssl_certs.sh test_stream.sh test_submodule.sh test_upgrade_auto.sh test_upgrade_interfaces.sh test_upgrade_repair.sh test_yang_namespace.sh" #IETFRFC= EOF diff --git a/docker/main/startsystem_evhtp.sh b/docker/main/startsystem_evhtp.sh index 2729c0c2..0edec974 100755 --- a/docker/main/startsystem_evhtp.sh +++ b/docker/main/startsystem_evhtp.sh @@ -64,7 +64,7 @@ echo "$STORE" > /usr/local/var/example/running_db # - test_install.sh since you dont have the make environment cat < /usr/local/bin/test/site.sh # Add your local site specific env variables (or tests) here. -SKIPLIST="test_api.sh test_client.sh test_c++.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh" +SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh" RCPROTO=https #IETFRFC= EOF diff --git a/docker/main/startsystem_fcgi.sh b/docker/main/startsystem_fcgi.sh index 362bc926..48b6400f 100755 --- a/docker/main/startsystem_fcgi.sh +++ b/docker/main/startsystem_fcgi.sh @@ -88,7 +88,7 @@ EOF # - test_order.sh XXX this is a bug need debugging cat < /usr/local/bin/test/site.sh # Add your local site specific env variables (or tests) here. -SKIPLIST="test_api.sh test_client.sh test_c++.sh test_yangmodels.sh test_openconfig.sh test_install.sh test_privileges.sh" +SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh" #IETFRFC= EOF diff --git a/fuzz/backend/README.md b/fuzz/backend/README.md index 9631d243..e9d16aed 100644 --- a/fuzz/backend/README.md +++ b/fuzz/backend/README.md @@ -41,6 +41,7 @@ Make a modification to how Clixon sends internal messages in `include/clixon_cus ``` #define CLIXON_PROTO_PLAIN ``` +(Note this is obsolete) Build clixon statically with the afl-clang compiler: ``` diff --git a/include/clixon_custom.h b/include/clixon_custom.h index 5764ca8e..931dc163 100644 --- a/include/clixon_custom.h +++ b/include/clixon_custom.h @@ -92,14 +92,3 @@ * clixon-4.4 */ #define STATE_ORDERED_BY_SYSTEM - -/*! Make internal XML protocol use plain strings instead of binary header - * Experimental - * This is only for testing, a specific usecase is fuzzing as described in fuzz/backend - * Note session-ids are not handled properly (a bunch of other things too) - * This could be mitigated by sending the session-id as an attribute - * But doing this one should probably revise/remove the code around clicon_msg_encode/decode - * which currently is somewhat hardwired. I.e., it may be difficult to have both variants as ifdef:s - * and you may consider replacing the old code altogether. - */ -#undef CLIXON_PROTO_PLAIN diff --git a/lib/src/clixon_yang.c b/lib/src/clixon_yang.c index dac35031..51dc15a7 100644 --- a/lib/src/clixon_yang.c +++ b/lib/src/clixon_yang.c @@ -2077,11 +2077,11 @@ ys_populate_unknown(clicon_handle h, if (nodeid_split(yang_argument_get(ys), &prefix, &id) < 0) goto done; if ((ymod = yang_find_module_by_prefix(ys, prefix)) == NULL){ - clicon_err(OE_YANG, ENOENT, "Extension %s:%s, module not found", prefix, id); + clicon_err(OE_YANG, ENOENT, "Extension \"%s:%s\", module not found", prefix, id); goto done; } if ((yext = yang_find(ymod, Y_EXTENSION, id)) == NULL){ - clicon_err(OE_YANG, ENOENT, "Extension %s:%s not found", prefix, id); + clicon_err(OE_YANG, ENOENT, "Extension \"%s:%s\" not found", prefix, id); goto done; } /* Optional argument (only if "argument") - save it in ys_cv */ diff --git a/lib/src/clixon_yang_parse.l b/lib/src/clixon_yang_parse.l index 8b199a5e..d542061c 100644 --- a/lib/src/clixon_yang_parse.l +++ b/lib/src/clixon_yang_parse.l @@ -102,15 +102,16 @@ identifier [A-Za-z_][A-Za-z0-9_\-\.]* %s COMMENT1 %s COMMENT2 %s UNKNOWN +%s UNKNOWN2 %% /* Common tokens */ [ \t] -<> { return MY_EOF; } -\n { _YY->yy_linenum++; } -\r -"/*" { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT1); } -"//" { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT2); } +<> { return MY_EOF; } +\n { _YY->yy_linenum++; } +\r +"/*" { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT1); } +"//" { _YY->yy_lex_state = YYSTATE; BEGIN(COMMENT2); } input { return K_INPUT; } /* No argument */ @@ -194,16 +195,22 @@ identifier [A-Za-z_][A-Za-z0-9_\-\.]* \} { return *yytext; } ; { return *yytext; } . { clixon_yang_parselval.string = strdup(yytext); - BEGIN(UNKNOWN); return CHARS; } + BEGIN(UNKNOWN); return CHARS; } : { return *yytext; } ; { BEGIN(KEYWORD); return *yytext; } -\" { _YY->yy_lex_string_state =UNKNOWN; BEGIN(STRINGDQ); return *yytext; } \{ { BEGIN(KEYWORD); return *yytext; } -[ \t]+ { return SEP; } -[^{";: \t]+ { clixon_yang_parselval.string = strdup(yytext); +[ \t\n]+ { BEGIN(UNKNOWN2);return SEP; } +[^{"';: \t\n]+ { clixon_yang_parselval.string = strdup(yytext); return CHARS; } +; { BEGIN(KEYWORD); return *yytext; } +\" { _YY->yy_lex_string_state =STRING; BEGIN(STRINGDQ); return *yytext; } +\' { _YY->yy_lex_string_state =STRING; BEGIN(STRINGSQ); return SQ; } +\{ { BEGIN(KEYWORD); return *yytext; } +[^{"'; \t\n]+ { clixon_yang_parselval.string = strdup(yytext); + return CHARS; } + true { clixon_yang_parselval.string = strdup(yytext); return BOOL; } false { clixon_yang_parselval.string = strdup(yytext); @@ -229,7 +236,7 @@ identifier [A-Za-z_][A-Za-z0-9_\-\.]* \" { _YY->yy_lex_string_state =STRING; BEGIN(STRINGDQ); return *yytext; } \' { _YY->yy_lex_string_state =STRING; BEGIN(STRINGSQ); return SQ; } \+ { return *yytext; } -. { clixon_yang_parselval.string = strdup(yytext); +[^\"\'\+\{\;\n \t]+ { clixon_yang_parselval.string = strdup(yytext); /* XXX [.]+ */ return CHARS;} \\ { _YY->yy_lex_state = STRINGDQ; BEGIN(DQESC); } diff --git a/lib/src/clixon_yang_parse.y b/lib/src/clixon_yang_parse.y index d7416355..4cd589e0 100644 --- a/lib/src/clixon_yang_parse.y +++ b/lib/src/clixon_yang_parse.y @@ -193,11 +193,16 @@ #include "clixon_yang_parse_lib.h" #include "clixon_yang_parse.h" -/* Enable for debugging, steals some cycles otherwise */ +/* Best debugging is to enable PARSE_DEBUG below and add -d to the LEX compile statement in the Makefile + * And then run the testcase with -D 1 + * Disable it to stop any calls to clicon_debug. Having it on by default would mean very large debug outputs. + */ #if 0 #define _PARSE_DEBUG(s) clicon_debug(1,(s)) +#define _PARSE_DEBUG1(s, s1) clicon_debug(1,(s), (s1)) #else #define _PARSE_DEBUG(s) +#define _PARSE_DEBUG1(s, s1) #endif extern int clixon_yang_parseget_lineno (void); @@ -219,7 +224,7 @@ clixon_yang_parseerror(void *_yy, } int - yang_parse_init(clixon_yang_yacc *yy) +yang_parse_init(clixon_yang_yacc *yy) { return 0; } @@ -1567,12 +1572,12 @@ deviate_substmt : type_stmt { _PARSE_DEBUG("deviate-substmt -> type-stmt unknown_stmt : ustring ':' ustring optsep ';' { char *id; if ((id=string_del_join($1, ":", $3)) == NULL) _YYERROR("unknown_stmt"); if (ysp_add(_yy, Y_UNKNOWN, id, NULL) == NULL) _YYERROR("unknown_stmt"); - _PARSE_DEBUG("unknown-stmt -> ustring : ustring"); + _PARSE_DEBUG("unknown-stmt -> ustring : ustring ;"); } | ustring ':' ustring SEP string optsep ';' { char *id; if ((id=string_del_join($1, ":", $3)) == NULL) _YYERROR("unknown_stmt"); - if (ysp_add(_yy, Y_UNKNOWN, id, $5) == NULL){ _YYERROR("unknwon_stmt"); } - _PARSE_DEBUG("unknown-stmt -> ustring : ustring string"); + if (ysp_add(_yy, Y_UNKNOWN, id, $5) == NULL){ _YYERROR("unknown_stmt"); } + _PARSE_DEBUG("unknown-stmt -> ustring : ustring SEP string ;"); } | ustring ':' ustring optsep { char *id; if ((id=string_del_join($1, ":", $3)) == NULL) _YYERROR("unknown_stmt"); @@ -1744,7 +1749,7 @@ qstrings : qstrings '+' qstring $$ = realloc($1, len + strlen($3) + 1); sprintf($$+len, "%s", $3); free($3); - _PARSE_DEBUG("qstrings-> qstrings + qstring"); + _PARSE_DEBUG("qstrings-> qstrings '+' qstring"); } | qstring { $$=$1; @@ -1752,13 +1757,13 @@ qstrings : qstrings '+' qstring ; qstring : '"' ustring '"' { $$=$2; - _PARSE_DEBUG("string-> \" ustring \"");} + _PARSE_DEBUG("qstring-> \" ustring \"");} | '"' '"' { $$=strdup(""); - _PARSE_DEBUG("string-> \" \"");} + _PARSE_DEBUG("qstring-> \" \"");} | SQ ustring SQ { $$=$2; - _PARSE_DEBUG("string-> ' ustring '"); } + _PARSE_DEBUG("qstring-> ' ustring '"); } | SQ SQ { $$=strdup(""); - _PARSE_DEBUG("string-> ' '");} + _PARSE_DEBUG("qstring-> ' '");} ; /* unquoted string */ @@ -1767,11 +1772,11 @@ ustring : ustring CHARS int len = strlen($1); $$ = realloc($1, len+strlen($2) + 1); sprintf($$+len, "%s", $2); + _PARSE_DEBUG1("ustring-> string + CHARS(%s)", $2); free($2); - _PARSE_DEBUG("ustring-> string + CHARS"); } | CHARS - {$$=$1; } + { _PARSE_DEBUG1("ustring-> CHARS(%s)", $1); $$=$1; } ; abs_schema_nodeid : abs_schema_nodeid '/' node_identifier @@ -1861,4 +1866,3 @@ stmtend : ';' ; %% - diff --git a/test/test_openconfig.sh b/test/test_openconfig.sh index e966550e..77234937 100755 --- a/test/test_openconfig.sh +++ b/test/test_openconfig.sh @@ -14,7 +14,8 @@ fyang=$dir/test.yang new "openconfig" if [ ! -d "$OPENCONFIG" ]; then # err "Hmm Openconfig dir does not seem to exist, try git clone https://github.com/openconfig/public?" - if [ "$s" = $0 ]; then exit 0; else return 0; fi + echo "...skipped: OPENCONFIG not set" + if [ "$s" = $0 ]; then exit 0; else return 0; fi fi OCDIR=$OPENCONFIG/release/models diff --git a/test/test_yang_extension.sh b/test/test_yang_extension.sh index b819e812..7df8404d 100755 --- a/test/test_yang_extension.sh +++ b/test/test_yang_extension.sh @@ -82,6 +82,21 @@ module $APPNAME{ ex:e4 arg1{ uses bar; } + + extension posix-pattern { + argument "pattern"; + } + typedef dotted-quad { + description "Only present for complex parsing of unknown-stmt"; + type string { + pattern + "[a-f]" + "[0-9]"; + ex:posix-pattern + // Strictly this comment is not supported if you see RFC syntax with only a SEP + // in unknwon-stmt: identifier [sep string] + '[f-w]' + '[o-q]'; + } + } } EOF @@ -110,7 +125,7 @@ new "Add extension bar (is implemented)" expecteof "$clixon_netconf -qf $cfg -D $DBG" 0 "a string]]>]]>" "^]]>]]>" new "netconf get config" -expecteof "$clixon_netconf -qf $cfg -D $DBG" 0 "]]>]]>" "^a string]]>]]>$" +expecteof "$clixon_netconf -qf $cfg -D $DBG" 0 "]]>]]>" "^a string]]>]]>" if [ $BE -ne 0 ]; then new "Kill backend" diff --git a/test/test_yang_models_ieee.sh b/test/test_yang_models_ieee.sh index a352699d..d9ca4a33 100755 --- a/test/test_yang_models_ieee.sh +++ b/test/test_yang_models_ieee.sh @@ -17,6 +17,7 @@ fyang=$dir/test.yang if [ ! -d "$YANGMODELS" ]; then # err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?" + echo "...skipped: YANGMODELS not set" if [ "$s" = $0 ]; then exit 0; else return 0; fi fi diff --git a/test/test_yang_models_ietf.sh b/test/test_yang_models_ietf.sh index 129232f1..4b493336 100755 --- a/test/test_yang_models_ietf.sh +++ b/test/test_yang_models_ietf.sh @@ -18,6 +18,7 @@ fyang=$dir/test.yang YANGMODELS=/home/olof/tmp/yang if [ ! -d "$YANGMODELS" ]; then # err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?" + echo "...skipped: YANGMODELS not set" if [ "$s" = $0 ]; then exit 0; else return 0; fi fi