diff --git a/.gitignore b/.gitignore index 9aaadb67..0a54c94a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ Makefile apps/Makefile apps/*/Makefile docker/Makefile +docker/base/Makefile +docker/system/Makefile etc/Makefile example/Makefile lib/Makefile diff --git a/CHANGELOG.md b/CHANGELOG.md index 812e6241..ed28216e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 3.9.0 (Preliminary Target: February 2019) ### Major New features -1. Correct XML namespace handling +* Correct XML namespace handling * According to [XML 1.0](https://www.w3.org/TR/2009/REC-xml-names-20091208) in restconf and Netconf. * Remaining deviations from strict namespace handling: * edit-config xpath select statement does not support namespaces @@ -56,7 +56,7 @@ ``` * To keep previous non-strict namespace handling (backwards compatible), set CLICON_XML_NS_STRICT to false. * See [https://github.com/clicon/clixon/issues/49] -1. Yang upgrade (RFC7950) +* Yang upgrade (RFC7950) * YANG parser cardinality checked (https://github.com/clicon/clixon/issues/48) * See https://github.com/clicon/clixon/issues/84 * RPC method input parameters validated @@ -76,7 +76,7 @@ * Note CLIXON_DATADIR (=/usr/local/share/clixon) need to be in the list * CLICON_YANG_MAIN_FILE Provides a filename with a single module filename. * CLICON_YANG_MAIN_DIR Provides a directory where all yang modules should be loaded. -1. NACM (RFC8341) +* NACM (RFC8341) * Experimental support, no performance enhancements and need further testing * Incoming RPC Message validation is supported (3.4.4) * Data Node Access validation is supported (3.4.5), except: diff --git a/configure b/configure index 092aa607..317af820 100755 --- a/configure +++ b/configure @@ -4433,7 +4433,7 @@ _ACEOF -ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile datastore/Makefile datastore/text/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile" +ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile docker/base/Makefile datastore/Makefile datastore/text/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure @@ -5143,6 +5143,7 @@ do "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;; "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;; "docker/system/Makefile") CONFIG_FILES="$CONFIG_FILES docker/system/Makefile" ;; + "docker/base/Makefile") CONFIG_FILES="$CONFIG_FILES docker/base/Makefile" ;; "datastore/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/Makefile" ;; "datastore/text/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/text/Makefile" ;; "util/Makefile") CONFIG_FILES="$CONFIG_FILES util/Makefile" ;; diff --git a/configure.ac b/configure.ac index 0cda1b5a..3261210c 100644 --- a/configure.ac +++ b/configure.ac @@ -243,6 +243,7 @@ AC_OUTPUT(Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile + docker/base/Makefile datastore/Makefile datastore/text/Makefile util/Makefile diff --git a/docker/Makefile.in b/docker/Makefile.in index 62d290dd..ce1c8e60 100644 --- a/docker/Makefile.in +++ b/docker/Makefile.in @@ -38,30 +38,21 @@ CFLAGS = @CFLAGS@ LDFLAGS = @LDFLAGS@ LIBS = @LIBS@ -# Example docker image. PLEASE CHANGE THIS IF YOU PUSH -IMG_BASE = olofhagsand/clixon # base image - SHELL = /bin/sh -SUBDIRS = system +SUBDIRS = base +SUBDIRS += system #SUBDIRS += cluster -.PHONY: all clean depend install docker push +.PHONY: all clean distclean depend install-include install uninstall all: $(SUBDIRS) - echo "Run make docker to build docker image" clean: distclean: clean rm -f Makefile *~ .depend -docker: Dockerfile - sudo docker build -t $(IMG_BASE) . # --no-cache - -push: - sudo docker push $(IMG_BASE) - depend: install-include: diff --git a/docker/README.md b/docker/README.md index a1b8ec53..f6e404ac 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,24 +1,6 @@ # Clixon base docker image -This directory contains code for building and pushing the clixon base docker -container. By default it is pushed to olofhagsand/clixon, but you can change -the IMAGE in Makefile.in and push it to another name. - -There are also sub-directories with examples og other clixon example systems. - -The clixon docker image is a base image that can be used to build -clixon applications. It has all the whole code for a clixon release -which it downloads from git - it does not use local code (note it may even use develop branch). - -See [system/README.md] for how to build the clixon example application using the base image. - -## Build and push - -Perform the build by 'make docker'. -You may also do 'make push' if you want to push the image, but you may then consider changing the image name (in the makefile:s). - -You may run the container directly by going directly to example and -the docker runtime scripts there - -(You may have to login for push with sudo docker login -u ) +This directory contains sub-directories with examples of Clixon docker images: + * [base] Clixon base image (pulls code from github) + * [system] Example and test application (builds from local dir) diff --git a/docker/Dockerfile b/docker/base/Dockerfile similarity index 100% rename from docker/Dockerfile rename to docker/base/Dockerfile diff --git a/docker/base/Makefile.in b/docker/base/Makefile.in new file mode 100644 index 00000000..e7a051ce --- /dev/null +++ b/docker/base/Makefile.in @@ -0,0 +1,68 @@ +# +# ***** BEGIN LICENSE BLOCK ***** +# +# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren +# +# This file is part of CLIXON +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Alternatively, the contents of this file may be used under the terms of +# the GNU General Public License Version 3 or later (the "GPL"), +# in which case the provisions of the GPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of the GPL, and not to allow others to +# use your version of this file under the terms of Apache License version 2, +# indicate your decision by deleting the provisions above and replace them with +# the notice and other provisions required by the GPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the Apache License version 2 or the GPL. +# +# ***** END LICENSE BLOCK ***** +# +VPATH = @srcdir@ +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ +CC = @CC@ +CFLAGS = @CFLAGS@ +LDFLAGS = @LDFLAGS@ +LIBS = @LIBS@ + +# docker.hub image. PLEASE CHANGE THIS IF YOU PUSH YOUR OWN +IMG = olofhagsand/clixon # base image + +SHELL = /bin/sh + +.PHONY: all clean distclean docker push depend install-include install uninstall + +all: + echo "Run make docker to build docker image" + +clean: + +distclean: clean + rm -f Makefile *~ .depend + +docker: Dockerfile + sudo docker build -t $(IMG) . # --no-cache + +push: + +depend: + +install-include: + +install: + +uninstall: + diff --git a/docker/base/README.md b/docker/base/README.md new file mode 100644 index 00000000..14b40a4a --- /dev/null +++ b/docker/base/README.md @@ -0,0 +1,23 @@ +# Clixon base docker image + +This directory contains code for building and pushing the clixon base docker +container. By default it is pushed to olofhagsand/clixon, but you can change +the IMAGE in Makefile.in and push it to another name. + +The clixon docker base image can be used to build clixon +applications. It has all the whole code for a clixon release which it +downloads from git - it does not use local code (note it may even use +develop branch). + +See [../system/README.md] for how to build the clixon example application using the base image. + +## Build and push + +Perform the build by 'make docker'. +You may also do 'make push' if you want to push the image, but you may then consider changing the image name (in the makefile:s). + +You may run the container directly by going directly to example and +the docker runtime scripts there + +(You may have to login for push with sudo docker login -u ) + diff --git a/docker/system/Dockerfile b/docker/system/Dockerfile index 26e86857..1a533c4c 100644 --- a/docker/system/Dockerfile +++ b/docker/system/Dockerfile @@ -31,34 +31,57 @@ # ***** END LICENSE BLOCK ***** # -FROM olofhagsand/clixon +# This application could use the base image, but it does not at this point. +#FROM olofhagsand/clixon +FROM debian MAINTAINER Olof Hagsand +# Clixon dependenies (not needed if build from base) +RUN apt-get update && apt-get install -y \ + git make gcc flex bison \ + libfcgi-dev \ + libcurl4-openssl-dev + +# Application-specific RUN apt-get update && apt-get install -y nginx -RUN apt-get update && apt-get install -y procps curl # ps for debugging + +# Test-specific +RUN apt-get update && apt-get install -y sudo curl procps # for test scripts # The example uses "clicon" group RUN groupadd clicon RUN usermod -a -G clicon www-data # Create a directory to hold source-code, dependencies etc -RUN mkdir /example -WORKDIR /example +RUN mkdir /clixon +WORKDIR /clixon -# Clone clixon (again) since example application is there. -# Replace this with your application -RUN git clone https://github.com/clicon/clixon.git - -# Build clixon -WORKDIR /example/clixon +# Clone and build cligen (not necessary if use base image) +RUN git clone https://github.com/olofhagsand/cligen.git +WORKDIR /clixon/cligen RUN ./configure -WORKDIR /example/clixon/example +RUN make +RUN make install + +# Copy Clixon from local dir +RUN mkdir /clixon/clixon +WORKDIR /clixon/clixon +COPY clixon . + +RUN ./configure +RUN make +RUN make install +RUN make install-include + +WORKDIR /clixon/clixon/example RUN make RUN make install RUN install example.xml /usr/local/etc/clixon.xml +RUN ldconfig + # Copy startscript -WORKDIR / +WORKDIR /clixon COPY startsystem.sh startsystem.sh RUN install startsystem.sh /usr/local/bin/ diff --git a/docker/system/Makefile.in b/docker/system/Makefile.in index 56bae9a9..ddc36f46 100644 --- a/docker/system/Makefile.in +++ b/docker/system/Makefile.in @@ -39,7 +39,7 @@ LDFLAGS = @LDFLAGS@ LIBS = @LIBS@ # Example docker image. PLEASE CHANGE THIS IF YOU PUSH -IMG_SYSTEM = clixon/clixon-system +IMG = clixon/clixon-system SHELL = /bin/sh @@ -48,13 +48,18 @@ SHELL = /bin/sh all: echo "Run make docker to build docker image" +# (recursively) clone the repo from top-level - NOTE changes must be committed +clixon: + git clone file://$(realpath ${top_srcdir}) + clean: + rm -rf clixon # clone of top-srcdir distclean: clean rm -f Makefile *~ .depend -docker: Dockerfile - sudo docker build -t $(IMG_SYSTEM) . # --no-cache +docker: clixon Dockerfile + sudo docker build -t $(IMG) . # --no-cache push: diff --git a/docker/system/README.md b/docker/system/README.md index d6ae8d8c..9c5901e4 100644 --- a/docker/system/README.md +++ b/docker/system/README.md @@ -1,31 +1,50 @@ # Clixon example container This directory show how to build a "monolithic" clixon docker -container containing the example application with both restconf, -netconf, cli and backend. +container exporting port 80 and contains the example application with +both restconf, netconf, cli and backend. + +Note that the container is built from scratch instead of using the +base image. And it differs from the base image in that it builds the +code from the local git locally (regardless of status and branch) +wheras the base image pulls from remote github master branch. + +Note that it could use the base image, and may in the future, see +comments in the Dockerfile. + The directory contains the following files: cleanup.sh kill containers Dockerfile Docker build instructions - lib.sh script library functions Makefile.in "make docker" builds the container README.md This file start.sh Start containers - startsystem.sh Internal start script copied to inside the container - stat.sh Shows container status + startsystem.sh Internal start script copied to inside the container (dont run from shell) -How to build and start the container: +How to build and start the container (called clixon-system): ``` $ make docker $ ./start.sh ``` +The start.sh has a number of environment variables to alter the default behaviour: +* PORT - Nginx exposes port 80 per default. Set `PORT=8080` for example to access restconf using 8080. +* DBG - Set debug. The clixon_backend will be shown on docker logs. +* CONFIG - Set XML configuration file other than the default example. +* STORE - Set running datastore content to other than default. + +Example: +``` + $ DBG=1 PORT=8080 ./start.sh +``` + Once running you can access it as follows: -* CLI: `sudo docker exec -it ef62ccfe1782 clixon_cli` -* Netconf: `sudo docker exec -it ef62ccfe1782 clixon_netconf` +* CLI: `sudo docker exec -it clixon-system clixon_cli` +* Netconf: `sudo docker exec -it clixon-system clixon_netconf` * Restconf: `curl -G http://localhost/restconf` +* Run tests: `sudo docker exec -it clixon-system bash -c 'cd /clixon/clixon/test; exec ./all.sh'` To check status and then kill it: ``` - $ ./stat.sh + $ sudo docker ps --all $ ./cleanup.sh -``` \ No newline at end of file +``` diff --git a/docker/system/cleanup.sh b/docker/system/cleanup.sh index 973bc0f7..efdc5d40 100755 --- a/docker/system/cleanup.sh +++ b/docker/system/cleanup.sh @@ -1,13 +1,7 @@ #!/bin/sh -# include err(), stat() and other functions -. ./lib.sh - # Kill all controller containers (optionally do `make clean`) -echo "You may want run make clean as well" - -kill1 clixon/clixon-system - +sudo docker kill clixon-system diff --git a/docker/system/lib.sh b/docker/system/lib.sh deleted file mode 100755 index 820935ab..00000000 --- a/docker/system/lib.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# Lib functions like err(), stat() and others - -# Error function -# usage: err $msg -err(){ - echo "\e[31m\n[Error $1]" - echo "\e[0m" - exit 1 -} - -#json field XXX notused? -jf(){ - sed -e 's/[{}]/''/g' | awk -v k="text" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' | grep key | awk -F : '{gsub(/"/,"",$3); print $3}' -} - -# Status function -# usage: stat $name -stat(){ - name=$1 - ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}') - if [ -n "$ps" ]; then - ip=$(sudo docker inspect -f '{{.NetworkSettings.IPAddress }}' $ps) - echo "$name \t$ps $ip" - else - err "$name failed" - fi -} - -# Kill function -kill1(){ - name=$1 - ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}') - if [ -n "$ps" ]; then - echo -n "$name\t" && sudo docker kill $ps - fi -} - diff --git a/docker/system/start.sh b/docker/system/start.sh index c3625818..172c6f1e 100755 --- a/docker/system/start.sh +++ b/docker/system/start.sh @@ -6,11 +6,24 @@ >&2 echo "Running script: $0" -# include err(), stat() and other functions -. ./lib.sh +pwd + +# Error function +# usage: err $msg +err(){ + echo "\e[31m\n[Error $1]" + echo "\e[0m" + exit 1 +} # Turn on debug in containers (restconf, backend) -DBG=${DBG:-1} +DBG=${DBG:-0} + +# Expose other host port than port 80 +PORT=${PORT:-80} + +# Initial running datastore content (other than empty) +STORE=${STORE:-} CONFIG0=$(cat < @@ -38,18 +51,14 @@ EOF ) CONFIG=${CONFIG:-$CONFIG0} -STORE=${STORE:-} # Start clixon-example backend # -p 4535 to access via cli from host >&2 echo -n "Starting Backend..." -sudo docker run -p 80:80 --rm -e DBG=$DBG -e CONFIG="$CONFIG" -e STORE="$STORE" -td clixon/clixon-system || err "Error starting clixon-system" +sudo docker run -p $PORT:80 --name clixon-system --rm -e DBG=$DBG -e CONFIG="$CONFIG" -e STORE="$STORE" -td clixon/clixon-system || err "Error starting clixon-system" >&2 echo "clixon-system started" -name=clixon/clixon-system -ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}') -echo "sudo docker exec -it $ps clixon_cli # example command" diff --git a/docker/system/startsystem.sh b/docker/system/startsystem.sh index 5d63b209..3bb57cef 100755 --- a/docker/system/startsystem.sh +++ b/docker/system/startsystem.sh @@ -45,6 +45,15 @@ server { } EOF +# This is a clixon site test file. Disable all model testing. +cat < /clixon/clixon/test/site.sh +# Add your local site specific env variables (or tests) here. +MODELS=0 # Dont run yangmodels/openconfig tests +IETFRFC=/clixon/clixon/yang/standard +EOF + + + # Start nginx #/usr/sbin/nginx -g 'daemon off;' -c /etc/nginx/nginx.conf /usr/sbin/nginx -c /etc/nginx/nginx.conf @@ -58,8 +67,6 @@ su -c "/www-data/clixon_restconf -l f/www-data/restconf.log -D $DBG" -s /bin/sh >&2 echo "start clixon_backend:" /usr/local/sbin/clixon_backend -FD $DBG -s running -l e # logs on docker logs - -# Start clixon cli in foreground -#/usr/local/bin/clixon_cli -D $DBG - -#/bin/sleep 100000000 +# Alt: let backend be in foreground, but test scripts may +# want to restart backend +/bin/sleep 100000000 diff --git a/docker/system/stat.sh b/docker/system/stat.sh deleted file mode 100755 index 16c271da..00000000 --- a/docker/system/stat.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# Show stats (IP address etc) about the clixon containers -# include err(), stat() and other functions -. ./lib.sh - -stat clixon/clixon-system - -name=clixon/clixon-system -ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}') -echo "sudo docker exec -it $ps clixon_cli # example command" diff --git a/lib/src/clixon_xml_parse.l b/lib/src/clixon_xml_parse.l index 12d8fceb..35a9d5af 100644 --- a/lib/src/clixon_xml_parse.l +++ b/lib/src/clixon_xml_parse.l @@ -112,7 +112,6 @@ ncname {namestart}{namechar}* return NAME; /* rather be catch-all */ } \: return *clixon_xml_parsetext; -\n { _YA->ya_linenum++;} <> { return MY_EOF; } ""[ \t] +[ \t] <> { return MY_EOF; } \n { _YY->yy_linenum++; } \r diff --git a/test/lib.sh b/test/lib.sh index 663c517b..2135e05a 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -46,7 +46,8 @@ testname= : ${OPENCONFIG=$(pwd)/public} # Standard IETF RFC yang files. -: ${IETFRFC=$YANGMODELS/standard/ietf/RFC} +: ${IETFRFC=../yang/standard} +#: ${IETFRFC=$YANGMODELS/standard/ietf/RFC} # For memcheck #clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli" diff --git a/test/test_openconfig.sh b/test/test_openconfig.sh index 5b35ca25..2fd16ae9 100755 --- a/test/test_openconfig.sh +++ b/test/test_openconfig.sh @@ -3,8 +3,9 @@ # Notes: # - openconfig test suites are patched to counter Clixon issues as follows: # - release/models/mpls/openconfig-mpls-te.yang -# issue: https://github.com/clicon/clixon/issues/60 -# - Env variable YANGMODELS should point to checkout place. (define it in site.sh for example) +# issue: https://github.com/clicon/clixon/issues/6 +# - Env-var MODELS should be 1 +# - Env-var OPENCONFIG should point to checkout place. (define it in site.sh for example) APPNAME=example @@ -20,7 +21,8 @@ fyang=$dir/test.yang new "openconfig" if [ ! -d "$OPENCONFIG" ]; then - err "Hmm Openconfig dir does not seem to exist, try git clone https://github.com/openconfig/public?" +# err "Hmm Openconfig dir does not seem to exist, try git clone https://github.com/openconfig/public?" + exit fi OCDIR=$OPENCONFIG/release/models diff --git a/test/test_yangmodels.sh b/test/test_yangmodels.sh index b2b93df1..8277a848 100755 --- a/test/test_yangmodels.sh +++ b/test/test_yangmodels.sh @@ -1,6 +1,7 @@ #!/bin/bash # Parse yangmodels from https://github.com/YangModels/yang # Notes: +# - Env-var MODELS should be 1 # - Env variable YANGMODELS should point to checkout place. (define it in site.sh for example) # - Only cisco/nx/9.2-2 # Many other versions # - Only cisco/xe/1631 # Many other versions @@ -30,7 +31,8 @@ cfg=$dir/conf_yang.xml fyang=$dir/test.yang if [ ! -d "$YANGMODELS" ]; then - err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?" +# err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?" + exit fi # Experimental IEEE diff --git a/test/travis/before_script.sh b/test/travis/before_script.sh index 72a79e3b..d218a36e 100755 --- a/test/travis/before_script.sh +++ b/test/travis/before_script.sh @@ -9,5 +9,8 @@ sudo make install-include (cd example && make && sudo make install) sudo groupadd clicon sudo usermod -a -G clicon $(whoami) -sudo usermod -a -G clicon root sudo usermod -a -G clicon www-data +# Build and start the system docker container +(cd docker/system && make docker && start.sh) +# Run clixon testcases +(cd docker/system && sudo docker exec -it clixon-system bash -c 'cd /clixon/clixon/test; exec ./all.sh')