From b70e22096e354c178995766e70b5dbd30cc1bb38 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Thu, 23 Sep 2021 13:22:40 +0200 Subject: [PATCH] - Pagination: enabled LIST_PAGINATION and remobved constant - Changed logic on how to find clixon_restconf in pseudo plugin - Removed ==== in constants to avoid conflict with git merge - Remove assert - Added fuzzing for netconf --- apps/backend/backend_client.c | 11 +++-- apps/backend/backend_get.c | 7 --- apps/backend/backend_plugin_restconf.c | 39 +++++++++++------ apps/cli/cli_generate.c | 57 ++++++++++++------------- apps/cli/cli_show.c | 10 ----- apps/restconf/restconf_methods_get.c | 7 --- apps/restconf/restconf_native.c | 2 +- doc/DEVELOP.md | 2 + docker/main/cleanup.sh | 2 +- fuzz/README.md | 2 - fuzz/cli/input/1.cli | 2 +- fuzz/cli/input/3.cli | 1 + fuzz/netconf/input/1.xml | 2 +- fuzz/netconf/input/2.xml | 2 +- fuzz/netconf/input/3.xml | 2 +- fuzz/netconf/runfuzz.sh | 4 +- fuzz/netconf/xml.dict | 27 +----------- include/clixon_custom.h | 7 --- lib/clixon/clixon_yang_parse_lib.h | 1 - lib/src/clixon_json.c | 3 -- lib/src/clixon_netconf_lib.c | 15 ------- lib/src/clixon_proto_client.c | 2 - lib/src/clixon_yang_parse_lib.c | 1 - test/test_restconf_basic_auth.sh | 10 ++--- test/test_restconf_err.sh | 10 ++--- test/test_restconf_internal.sh | 11 ++--- test/test_restconf_internal_usecases.sh | 10 +++-- test/test_restconf_ssl_certs.sh | 20 ++++++++- 28 files changed, 114 insertions(+), 155 deletions(-) diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index eed8b767..bf880838 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -57,7 +57,6 @@ #include #include #include -#include #include /* cligen */ @@ -394,7 +393,10 @@ from_client_edit_config(clicon_handle h, goto ok; } } - assert(cbuf_len(cbret) == 0); + if (cbuf_len(cbret) != 0){ + clicon_err(OE_NETCONF, EINVAL, "Internal error: cbret is not empty"); + goto done; + } cprintf(cbret, "ce_s); + if (s != ce->ce_s){ + clicon_err(OE_NETCONF, EINVAL, "Internal error: s != ce->ce_s"); + goto done; + } if (clicon_msg_rcv(ce->ce_s, &msg, &eof) < 0) goto done; if (eof) diff --git a/apps/backend/backend_get.c b/apps/backend/backend_get.c index 595ceaac..17f1c779 100644 --- a/apps/backend/backend_get.c +++ b/apps/backend/backend_get.c @@ -389,8 +389,6 @@ get_nacm_and_reply(clicon_handle h, return retval; } -#ifdef LIST_PAGINATION - /*! Help function for parsing restconf query parameter and setting netconf attribute * * If not "unbounded", parse and set a numeric value @@ -652,7 +650,6 @@ get_list_pagination(clicon_handle h, xml_free(xret); return retval; } -#endif /* LIST_PAGINATION */ /*! Common get/get-config code for retrieving configuration and state information. * @@ -692,11 +689,9 @@ get_common(clicon_handle h, cbuf *cbmsg = NULL; /* For error msg */ char *xpath0; cbuf *cbreason = NULL; -#ifdef LIST_PAGINATION int list_pagination = 0; char *valstr; cxobj *x; -#endif /* LIST_PAGINATION */ clicon_debug(1, "%s", __FUNCTION__); username = clicon_username_get(h); @@ -736,7 +731,6 @@ get_common(clicon_handle h, goto ok; } } -#ifdef LIST_PAGINATION /* Check if list pagination */ if ((x = xml_find_type(xe, NULL, "list-pagination", CX_ELMNT)) != NULL && (valstr = xml_body(x)) != NULL && @@ -752,7 +746,6 @@ get_common(clicon_handle h, goto done; goto ok; } -#endif /* LIST_PAGINATION */ /* Read configuration */ switch (content){ case CONTENT_CONFIG: /* config data only */ diff --git a/apps/backend/backend_plugin_restconf.c b/apps/backend/backend_plugin_restconf.c index aef1df7b..c7db8249 100644 --- a/apps/backend/backend_plugin_restconf.c +++ b/apps/backend/backend_plugin_restconf.c @@ -246,9 +246,11 @@ restconf_pseudo_process_control(clicon_handle h) int i; int nr; cbuf *cb = NULL; - char *dir = NULL; + char *dir0 = NULL; + char *dir1 = NULL; char *pgm; struct stat fstat; + int found = 0; nr = 10; if ((argv = calloc(nr, sizeof(char *))) == NULL){ @@ -265,19 +267,32 @@ restconf_pseudo_process_control(clicon_handle h) * If not, use the Makefile * Use PATH? */ - if ((dir = clicon_option_str(h, "CLICON_RESTCONF_INSTALLDIR")) == NULL){ - if ((dir = CLIXON_CONFIG_SBINDIR) == NULL){ - clicon_err(OE_RESTCONF, EINVAL, "Both option CLICON_RESTCONF_INSTALLDIR and makefile constant CLIXON_CONFIG_SBINDIR are NULL which make sit not possible to know where clixon_restconf is installed(shouldnt happen)"); - goto done; + if ((dir0 = clicon_option_str(h, "CLICON_RESTCONF_INSTALLDIR")) != NULL){ + cprintf(cb, "%s/clixon_restconf", dir0); + pgm = cbuf_get(cb); + if (stat(pgm, &fstat) == 0){ /* Sanity check: program exists */ + clicon_debug(1, "Found %s", pgm); + found++; } + else + clicon_debug(1, "Not found: %s", pgm); } - cprintf(cb, "%s/clixon_restconf", dir); - pgm = cbuf_get(cb); - /* Sanity check: program exists */ - if (stat(pgm, &fstat) < 0) { - clicon_err(OE_FATAL, errno, "%s, you may have set CLICON_BACKEND_RESTCONF_PROCESS but clixon_restconf is not found in %s. Try overriding with CLICON_RESTCONF_INSTALLDIR", - pgm, - CLIXON_CONFIG_SBINDIR); + if (!found && + (dir1 = CLIXON_CONFIG_SBINDIR) != NULL){ + cbuf_reset(cb); + cprintf(cb, "%s/clixon_restconf", dir1); + pgm = cbuf_get(cb); + clicon_debug(1, "Looking for %s", pgm); + if (stat(pgm, &fstat) == 0){ /* Sanity check: program exists */ + clicon_debug(1, "Found %s", pgm); + found++; + } + else + clicon_debug(1, "Not found: %s", pgm); + } + if (!found){ + clicon_err(OE_RESTCONF, 0, "clixon_restconf not found in neither CLICON_RESTCONF_INSTALLDIR(%s) nor CLIXON_CONFIG_SBINDIR(%s). Try overriding with CLICON_RESTCONF_INSTALLDIR", + dir0, dir1); goto done; } argv[i++] = pgm; diff --git a/apps/cli/cli_generate.c b/apps/cli/cli_generate.c index 8e777750..a2cd6cfb 100644 --- a/apps/cli/cli_generate.c +++ b/apps/cli/cli_generate.c @@ -39,6 +39,33 @@ * | | ------------> | cli | * | list{key A;}| | syntax | * +-------------+ +-------------+ + + * YANG generate CLI + + This is an example yang module: +module m { + container x { + namespace "urn:example:m"; + prefix m; + list m1 { + key "a"; + leaf a { + type string; + } + leaf b { + type string; + } + } + } +} + +You can see which CLISPEC it generates via clixon_cli -D 2: + x,cli_set("/example:x");{ + m1 a (|),overwrite_me("/example:x/m1=%s/"); +{ + b (|),overwrite_me("/example:x/m1=%s/b"); + } + } */ #ifdef HAVE_CONFIG_H #include "clixon_config.h" /* generated by config & autoconf */ @@ -69,36 +96,6 @@ /* variable expand function */ #define GENERATE_EXPAND_XMLDB "expand_dbvar" -/*===================================================================== - * YANG generate CLI - *=====================================================================*/ -/* - This is an example yang module: -module m { - container x { - namespace "urn:example:m"; - prefix m; - list m1 { - key "a"; - leaf a { - type string; - } - leaf b { - type string; - } - } - } -} - -You can see which CLISPEC it generates via clixon_cli -D 2: - x,cli_set("/example:x");{ - m1 a (|),overwrite_me("/example:x/m1=%s/"); -{ - b (|),overwrite_me("/example:x/m1=%s/b"); - } - } -*/ - /*! Create cligen variable expand entry with xmlkey format string as argument * @param[in] h clicon handle * @param[in] ys yang_stmt of the node at hand diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index fc89414c..76c2166d 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -891,8 +891,6 @@ cli_show_options(clicon_handle h, return retval; } -#ifdef LIST_PAGINATION - /*! Show pagination * @param[in] h Clicon handle * @param[in] cvv Vector of cli string and instantiated variables @@ -1016,11 +1014,3 @@ cli_pagination(clicon_handle h, cbuf_free(cb); return retval; } -#else -int -cli_pagination(clicon_handle h, cvec *cvv, cvec *argv) -{ - fprintf(stderr, "Not yet implemented\n"); - return 0; -} -#endif /* LIST_PAGINATION */ diff --git a/apps/restconf/restconf_methods_get.c b/apps/restconf/restconf_methods_get.c index df37aca9..4378afcc 100644 --- a/apps/restconf/restconf_methods_get.c +++ b/apps/restconf/restconf_methods_get.c @@ -300,7 +300,6 @@ api_data_get2(clicon_handle h, return retval; } -#ifdef LIST_PAGINATION /*! GET Collection * According to restconf collection draft. Lists, work in progress * @param[in] h Clixon handle @@ -566,7 +565,6 @@ api_data_collection(clicon_handle h, free(xvec); return retval; } -#endif /* LIST_PAGINATION */ /*! REST HEAD method * @param[in] h Clixon handle @@ -644,13 +642,8 @@ api_data_get(clicon_handle h, break; case YANG_COLLECTION_XML: case YANG_COLLECTION_JSON: -#ifdef LIST_PAGINATION if (api_data_collection(h, req, api_path, pcvec, pi, qvec, pretty, media_out) < 0) goto done; -#else - if (restconf_notimplemented(h, req, pretty, media_out) < 0) - goto done; -#endif break; default: break; diff --git a/apps/restconf/restconf_native.c b/apps/restconf/restconf_native.c index 11c806e7..15e82bae 100644 --- a/apps/restconf/restconf_native.c +++ b/apps/restconf/restconf_native.c @@ -266,7 +266,7 @@ restconf_connection_sanity(clicon_handle h, if (rc->rc_ssl == NULL && rc->rc_proto == HTTP_2 && clicon_option_bool(h, "CLICON_RESTCONF_HTTP2_PLAIN") == 0){ - if (netconf_invalid_value_xml(&xerr, "protocol", "Plain HTTP/2 is disabled") < 0) + if (netconf_invalid_value_xml(&xerr, "protocol", "Only HTTP/2 with TLS is enabled, plain http/2 is disabled") < 0) goto done; if ((media_str = restconf_param_get(h, "HTTP_ACCEPT")) == NULL){ media_out = YANG_DATA_JSON; diff --git a/doc/DEVELOP.md b/doc/DEVELOP.md index b944a2ea..3c21c08c 100644 --- a/doc/DEVELOP.md +++ b/doc/DEVELOP.md @@ -104,6 +104,8 @@ Notes: Use `/* */`. Use `//` only for temporal comments. +Do not use "======", ">>>>>" or "<<<<<<" in comments since git merge conflict uses that. + ## How to work in git Clixon uses semantic versioning (https://semver.org). diff --git a/docker/main/cleanup.sh b/docker/main/cleanup.sh index efdc5d40..ee6d3b0f 100755 --- a/docker/main/cleanup.sh +++ b/docker/main/cleanup.sh @@ -1,7 +1,7 @@ #!/bin/sh # Kill all controller containers (optionally do `make clean`) -sudo docker kill clixon-system +sudo docker kill clixon-system 2> /dev/null # ignore errors diff --git a/fuzz/README.md b/fuzz/README.md index f2812e11..02374fdf 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -2,8 +2,6 @@ Clixon can be fuzzed with [american fuzzy lop](https://github.com/google/AFL/releases) but not without pain. -So far the backend and cli can be fuzzed. - Some issues are as follows: - Static linking. Fuzzing requires static linking. You can statically link clixon using: `LINKAGE=static ./configure` but that does not work with Clixon plugins (at least yet). Therefore fuzzing has been made with no plugins using the hello example only. - Multiple processes. Only the backend can run stand-alone, cli/netconf/restconf requires a backend. When you fuzz eg clixon_cli, the backend must be running and it will be slow due to IPC. Possibly one could link them together and run as a monolith by making a threaded image. diff --git a/fuzz/cli/input/1.cli b/fuzz/cli/input/1.cli index 5ccc80fe..f2ee1f5f 100644 --- a/fuzz/cli/input/1.cli +++ b/fuzz/cli/input/1.cli @@ -1 +1 @@ -set hello world +set table parameter a value 42 \ No newline at end of file diff --git a/fuzz/cli/input/3.cli b/fuzz/cli/input/3.cli index 49a0751d..65f190b4 100644 --- a/fuzz/cli/input/3.cli +++ b/fuzz/cli/input/3.cli @@ -1 +1,2 @@ validate +commit diff --git a/fuzz/netconf/input/1.xml b/fuzz/netconf/input/1.xml index 3137d37a..4613beb4 100644 --- a/fuzz/netconf/input/1.xml +++ b/fuzz/netconf/input/1.xml @@ -1 +1 @@ -]]>]]> +a
]]>]]> diff --git a/fuzz/netconf/input/2.xml b/fuzz/netconf/input/2.xml index 35a2f6b9..69348365 100644 --- a/fuzz/netconf/input/2.xml +++ b/fuzz/netconf/input/2.xml @@ -1 +1 @@ -]]>]]> +>]]>]]> diff --git a/fuzz/netconf/input/3.xml b/fuzz/netconf/input/3.xml index a151547f..97058bcc 100644 --- a/fuzz/netconf/input/3.xml +++ b/fuzz/netconf/input/3.xml @@ -1 +1 @@ -]]>]]> +>]]>]]> diff --git a/fuzz/netconf/runfuzz.sh b/fuzz/netconf/runfuzz.sh index f564099c..4d5d4ce4 100755 --- a/fuzz/netconf/runfuzz.sh +++ b/fuzz/netconf/runfuzz.sh @@ -48,5 +48,5 @@ test ! -d output || rm -rf output #test -d input || mkdir input test -d output || mkdir output -# Run script -afl-fuzz -i input -o output -m $MEGS -- clixon_netconf -f $cfg +# Run script +afl-fuzz -i input -o output -m $MEGS -x xml.dict -- clixon_netconf -qf $cfg -o CLICON_NETCONF_HELLO_OPTIONAL=true diff --git a/fuzz/netconf/xml.dict b/fuzz/netconf/xml.dict index db52a0aa..5f875950 100644 --- a/fuzz/netconf/xml.dict +++ b/fuzz/netconf/xml.dict @@ -12,11 +12,8 @@ attr_generic=" a=\"1\"" attr_href=" href=\"1\"" attr_standalone=" standalone=\"no\"" attr_version=" version=\"1\"" -attr_xml_base=" xml:base=\"1\"" -attr_xml_id=" xml:id=\"1\"" -attr_xml_lang=" xml:lang=\"1\"" -attr_xml_space=" xml:space=\"1\"" attr_xmlns=" xmlns=\"1\"" +attr_xmlns=" xmlns:ns=\"1\"" entity_builtin="<" entity_decimal="" @@ -26,30 +23,13 @@ entity_hex="" string_any="ANY" string_brackets="[]" string_cdata="CDATA" -string_col_fallback=":fallback" -string_col_generic=":a" -string_col_include=":include" string_dashes="--" -string_empty="EMPTY" string_empty_dblquotes="\"\"" string_empty_quotes="''" -string_entities="ENTITIES" -string_entity="ENTITY" -string_fixed="#FIXED" -string_id="ID" -string_idref="IDREF" -string_idrefs="IDREFS" -string_implied="#IMPLIED" -string_nmtoken="NMTOKEN" -string_nmtokens="NMTOKENS" -string_notation="NOTATION" string_parentheses="()" string_pcdata="#PCDATA" string_percent="%a" string_public="PUBLIC" -string_required="#REQUIRED" -string_schema=":schema" -string_system="SYSTEM" string_ucs4="UCS-4" string_utf16="UTF-16" string_utf8="UTF-8" @@ -59,11 +39,6 @@ tag_attlist="" tag_doctype="" tag_open_close="" tag_open_exclamation="&1)" 0 "HTTP/$HVER 400" "HTTP cert verification failed, unknown ca" - + new "too weak cert (sign w md5)" - expectpart "$(curl $CURLOPTS --key $certdir/mymd5.key --cert $certdir/mymd5.crt -X GET $RCPROTO://localhost/restconf/data/example:x 2>&1)" "35 58" # "md too weak" + # Either curl error or error return ret=$(curl $CURLOPTS --key $certdir/mymd5.key --cert $certdir/mymd5.crt -X GET $RCPROTO://localhost/restconf/data/example:x 2> /dev/null) + r=$? + if [ $r = 0 ]; then + # Check return value + match=$(echo "$ret" | grep --null -o "HTTP/$HVER 400") + r1=$? + if [ $r1 != 0 ]; then + err "HTTP/$HVER 400" "$match" + fi + match=$(echo "$ret" | grep --null -o "HTTP cert verification failed") + r1=$? + if [ $r1 != 0 ]; then + err "HTTP cert verification failed" "$match" + fi + elif [ $r != 35 -a $r != 58 ]; then + err "35 58" "$r" + fi new "Random cert" expectpart "$(curl $CURLOPTS --key $certdir/random.key --cert $certdir/random.crt -X GET $RCPROTO://localhost/restconf/data/example:x 2>&1)" 0 "HTTP/$HVER 400" "HTTP cert verification failed"