Added docker/base as minimal clixon container and docker/system as full clixon system.

.gitignore

@@ -8,6 +8,8 @@ Makefile
 apps/Makefile
 apps/*/Makefile
 docker/Makefile
+docker/base/Makefile
+docker/system/Makefile
 etc/Makefile
 example/Makefile
 lib/Makefile

CHANGELOG.md

@@ -3,7 +3,7 @@
 ## 3.9.0 (Preliminary Target: February 2019)
 
 ### Major New features
-1. Correct XML namespace handling
+* Correct XML namespace handling
   * According to [XML 1.0](https://www.w3.org/TR/2009/REC-xml-names-20091208) in restconf and Netconf.
   * Remaining deviations from strict namespace handling:
     * edit-config xpath select statement does not support namespaces
@@ -56,7 +56,7 @@
     ```
   * To keep previous non-strict namespace handling (backwards compatible), set CLICON_XML_NS_STRICT to false.
   * See [https://github.com/clicon/clixon/issues/49]
-1. Yang upgrade (RFC7950)
+* Yang upgrade (RFC7950)
   * YANG parser cardinality checked (https://github.com/clicon/clixon/issues/48)
     * See https://github.com/clicon/clixon/issues/84
   * RPC method input parameters validated
@@ -76,7 +76,7 @@
       * Note CLIXON_DATADIR (=/usr/local/share/clixon) need to be in the list
     * CLICON_YANG_MAIN_FILE Provides a filename with a single module filename.
     * CLICON_YANG_MAIN_DIR Provides a directory where all yang modules should be loaded.
-1. NACM (RFC8341)
+* NACM (RFC8341)
   * Experimental support, no performance enhancements and need further testing
   * Incoming RPC Message validation is supported (3.4.4)
   * Data Node Access validation is supported (3.4.5), except:

configure

@@ -4433,7 +4433,7 @@ _ACEOF
 
 
 
-ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile datastore/Makefile datastore/text/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile"
+ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile extras/rpm/Makefile docker/Makefile docker/system/Makefile docker/base/Makefile datastore/Makefile datastore/text/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/standard/Makefile doc/Makefile test/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -5143,6 +5143,7 @@ do
     "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;;
     "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;;
     "docker/system/Makefile") CONFIG_FILES="$CONFIG_FILES docker/system/Makefile" ;;
+    "docker/base/Makefile") CONFIG_FILES="$CONFIG_FILES docker/base/Makefile" ;;
     "datastore/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/Makefile" ;;
     "datastore/text/Makefile") CONFIG_FILES="$CONFIG_FILES datastore/text/Makefile" ;;
     "util/Makefile") CONFIG_FILES="$CONFIG_FILES util/Makefile" ;;

configure.ac

@@ -243,6 +243,7 @@ AC_OUTPUT(Makefile
 	  extras/rpm/Makefile
 	  docker/Makefile
   	  docker/system/Makefile
+    	  docker/base/Makefile
 	  datastore/Makefile
 	  datastore/text/Makefile
   	  util/Makefile 

docker/Makefile.in

@@ -38,30 +38,21 @@ CFLAGS  	= @CFLAGS@
 LDFLAGS 	= @LDFLAGS@
 LIBS    	= @LIBS@
 
-# Example docker image. PLEASE CHANGE THIS IF YOU PUSH
-IMG_BASE      = olofhagsand/clixon # base image
-
 SHELL	= /bin/sh
 
-SUBDIRS  = system
+SUBDIRS  = base
+SUBDIRS  += system
 #SUBDIRS += cluster
 
-.PHONY: all clean depend install docker push
+.PHONY: all clean distclean depend install-include install uninstall
 
 all:	$(SUBDIRS)
-	echo "Run make docker to build docker image"
 
 clean:
 
 distclean: clean
 	rm -f Makefile *~ .depend
 
-docker: Dockerfile
-	sudo docker build -t $(IMG_BASE) .  # --no-cache
-
-push:
-	sudo docker push $(IMG_BASE)
-
 depend:
 
 install-include:

docker/README.md

@@ -1,24 +1,6 @@
 # Clixon base docker image
 
-This directory contains code for building and pushing the clixon base docker
+This directory contains sub-directories with examples of Clixon docker images:
-container. By default it is pushed to olofhagsand/clixon, but you can change
+    * [base]    Clixon base image (pulls code from github)
-the IMAGE in Makefile.in and push it to another name.
+    * [system]  Example and test application (builds from local dir)
-
-There are also sub-directories with examples og other clixon example systems.
-
-The clixon docker image is a base image that can be used to build
-clixon applications. It has all the whole code for a clixon release
-which it downloads from git - it does not use local code (note it may even use develop branch).
-
-See [system/README.md] for how to build the clixon example application using the base image.
-
-## Build and push
-
-Perform the build by 'make docker'. 
-You may also do 'make push' if you want to push the image, but you may then consider changing the image name (in the makefile:s).
-
-You may run the container directly by going directly to example and
-the docker runtime scripts there
-
-(You may have to login for push with sudo docker login -u <username>)
 

docker/base/Makefile.in

@@ -0,0 +1,68 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# 
+# Copyright (C) 2009-2019 Olof Hagsand and Benny Holmgren
+#
+# This file is part of CLIXON
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Alternatively, the contents of this file may be used under the terms of
+# the GNU General Public License Version 3 or later (the "GPL"),
+# in which case the provisions of the GPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of the GPL, and not to allow others to
+# use your version of this file under the terms of Apache License version 2, 
+# indicate your decision by deleting the provisions above and replace them with
+# the notice and other provisions required by the GPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the Apache License version 2 or the GPL.
+#
+# ***** END LICENSE BLOCK *****
+#
+VPATH       	= @srcdir@
+srcdir  	= @srcdir@
+top_srcdir  	= @top_srcdir@
+CC		= @CC@
+CFLAGS  	= @CFLAGS@ 
+LDFLAGS 	= @LDFLAGS@
+LIBS    	= @LIBS@
+
+# docker.hub image. PLEASE CHANGE THIS IF YOU PUSH YOUR OWN
+IMG   = olofhagsand/clixon # base image
+
+SHELL	= /bin/sh
+
+.PHONY: all clean distclean docker push depend install-include install uninstall
+
+all:	
+	echo "Run make docker to build docker image"
+
+clean:
+
+distclean: clean
+	rm -f Makefile *~ .depend
+
+docker: Dockerfile
+	sudo docker build -t $(IMG) .  # --no-cache
+
+push:
+
+depend:
+
+install-include:
+
+install:
+
+uninstall:
+

docker/base/README.md

@@ -0,0 +1,23 @@
+# Clixon base docker image
+
+This directory contains code for building and pushing the clixon base docker
+container. By default it is pushed to olofhagsand/clixon, but you can change
+the IMAGE in Makefile.in and push it to another name.
+
+The clixon docker base image can be used to build clixon
+applications. It has all the whole code for a clixon release which it
+downloads from git - it does not use local code (note it may even use
+develop branch).
+
+See [../system/README.md] for how to build the clixon example application using the base image.
+
+## Build and push
+
+Perform the build by 'make docker'. 
+You may also do 'make push' if you want to push the image, but you may then consider changing the image name (in the makefile:s).
+
+You may run the container directly by going directly to example and
+the docker runtime scripts there
+
+(You may have to login for push with sudo docker login -u <username>)
+

docker/system/Dockerfile

@@ -31,34 +31,57 @@
 # ***** END LICENSE BLOCK *****
 #
 
-FROM olofhagsand/clixon
+# This application could use the base image, but it does not at this point.
+#FROM olofhagsand/clixon 
+FROM debian
 MAINTAINER Olof Hagsand <olof@hagsand.se>
 
+# Clixon dependenies (not needed if build from base)
+RUN apt-get update && apt-get install -y \
+  git make gcc flex bison \
+  libfcgi-dev \
+  libcurl4-openssl-dev
+
+# Application-specific
 RUN apt-get update && apt-get install -y nginx
-RUN apt-get update && apt-get install -y procps curl # ps for debugging
+
+# Test-specific
+RUN apt-get update && apt-get install -y sudo curl procps # for test scripts
 
 # The example uses "clicon" group
 RUN groupadd clicon
 RUN usermod -a -G clicon www-data
 
 # Create a directory to hold source-code, dependencies etc
-RUN mkdir /example
+RUN mkdir /clixon
-WORKDIR /example
+WORKDIR /clixon
 
-# Clone clixon (again) since example application is there.
+# Clone and build cligen (not necessary if use base image)
-# Replace this with your application
+RUN git clone https://github.com/olofhagsand/cligen.git
-RUN git clone https://github.com/clicon/clixon.git
+WORKDIR /clixon/cligen
-
-# Build clixon
-WORKDIR /example/clixon
 RUN ./configure
-WORKDIR /example/clixon/example
+RUN make
+RUN make install
+
+# Copy Clixon from local dir
+RUN mkdir /clixon/clixon
+WORKDIR /clixon/clixon
+COPY clixon .
+
+RUN ./configure
+RUN make
+RUN make install
+RUN make install-include
+
+WORKDIR /clixon/clixon/example
 RUN make
 RUN make install
 RUN install example.xml /usr/local/etc/clixon.xml
 
+RUN ldconfig
+
 # Copy startscript
-WORKDIR /
+WORKDIR /clixon
 COPY startsystem.sh startsystem.sh 
 RUN install startsystem.sh /usr/local/bin/
 

docker/system/Makefile.in

@@ -39,7 +39,7 @@ LDFLAGS 	= @LDFLAGS@
 LIBS    	= @LIBS@
 
 # Example docker image. PLEASE CHANGE THIS IF YOU PUSH
-IMG_SYSTEM   = clixon/clixon-system
+IMG   = clixon/clixon-system
 
 SHELL	= /bin/sh
 
@@ -48,13 +48,18 @@ SHELL	= /bin/sh
 all:	
 	echo "Run make docker to build docker image"
 
+# (recursively) clone the repo from top-level - NOTE changes must be committed
+clixon:
+	git clone file://$(realpath ${top_srcdir})
+
 clean:
+	rm -rf clixon # clone of top-srcdir
 
 distclean: clean
 	rm -f Makefile *~ .depend
 
-docker: Dockerfile
+docker: clixon Dockerfile
-	sudo docker build -t $(IMG_SYSTEM) .  # --no-cache
+	sudo docker build -t $(IMG) .  # --no-cache
 
 push:
 

docker/system/README.md

@@ -1,31 +1,50 @@
 # Clixon example container
 
 This directory show how to build a "monolithic" clixon docker
-container containing the example application with both restconf,
+container exporting port 80 and contains the example application with
-netconf, cli and backend.
+both restconf, netconf, cli and backend.
+
+Note that the container is built from scratch instead of using the
+base image. And it differs from the base image in that it builds the
+code from the local git locally (regardless of status and branch)
+wheras the base image pulls from remote github master branch.
+
+Note that it could use the base image, and may in the future, see
+comments in the Dockerfile.
+
 The directory contains the following files:
 	 cleanup.sh     kill containers
 	 Dockerfile     Docker build instructions
-	 lib.sh         script library functions
 	 Makefile.in    "make docker" builds the container
 	 README.md	This file
 	 start.sh       Start containers
-	 startsystem.sh Internal start script copied to inside the container
+	 startsystem.sh Internal start script copied to inside the container (dont run from shell)
-	 stat.sh        Shows container status
 
-How to build and start the container:
+How to build and start the container (called clixon-system):
 ```
   $ make docker
   $ ./start.sh 
 ```
 
+The start.sh has a number of environment variables to alter the default behaviour:
+* PORT - Nginx exposes port 80 per default. Set `PORT=8080` for example to access restconf using 8080.
+* DBG - Set debug. The clixon_backend will be shown on docker logs.
+* CONFIG - Set XML configuration file other than the default example.
+* STORE - Set running datastore content to other than default.
+
+Example:
+```
+  $ DBG=1 PORT=8080 ./start.sh
+```
+
 Once running you can access it as follows:
-* CLI: `sudo docker exec -it ef62ccfe1782 clixon_cli`
+* CLI: `sudo docker exec -it clixon-system clixon_cli`
-* Netconf: `sudo docker exec -it ef62ccfe1782 clixon_netconf`
+* Netconf: `sudo docker exec -it clixon-system clixon_netconf`
 * Restconf: `curl -G http://localhost/restconf`
+* Run tests: `sudo docker exec -it clixon-system bash -c 'cd /clixon/clixon/test; exec ./all.sh'`
 
 To check status and then kill it:
 ```
-  $ ./stat.sh
+  $ sudo docker ps --all
   $ ./cleanup.sh 
 ```

docker/system/cleanup.sh

@@ -1,13 +1,7 @@
 #!/bin/sh
 
-# include err(), stat() and other functions
-. ./lib.sh
-
 # Kill all controller containers (optionally do `make clean`)
-echo "You may want run make clean as well"
+sudo docker kill clixon-system
-
-kill1 clixon/clixon-system
-
 
 
 

docker/system/lib.sh

@@ -1,38 +0,0 @@
-#!/bin/sh
-# Lib functions like err(), stat() and others
-
-# Error function
-# usage: err $msg
-err(){
-    echo  "\e[31m\n[Error $1]"
-    echo  "\e[0m"
-    exit 1
-}
-
-#json field XXX notused?
-jf(){
-  sed -e 's/[{}]/''/g' | awk -v k="text" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' | grep key | awk -F : '{gsub(/"/,"",$3); print $3}'
-}
-
-# Status function
-# usage: stat $name
-stat(){
-    name=$1
-    ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}')
-    if [ -n "$ps" ]; then
-	ip=$(sudo docker inspect -f '{{.NetworkSettings.IPAddress }}' $ps)
-	echo "$name \t$ps $ip"
-    else
-	err "$name failed"
-    fi
-}
-
-# Kill function
-kill1(){
-    name=$1
-    ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}')
-    if [ -n "$ps" ]; then
-	echo -n "$name\t" && sudo docker kill $ps
-    fi
-}
-

docker/system/start.sh

@@ -6,11 +6,24 @@
 
 >&2 echo "Running script: $0"
 
-# include err(), stat() and other functions
+pwd
-. ./lib.sh
+
+# Error function
+# usage: err $msg
+err(){
+    echo  "\e[31m\n[Error $1]"
+    echo  "\e[0m"
+    exit 1
+}
 
 # Turn on debug in containers (restconf, backend)
-DBG=${DBG:-1}
+DBG=${DBG:-0}
+
+# Expose other host port than port 80
+PORT=${PORT:-80}
+
+# Initial running datastore content (other than empty)
+STORE=${STORE:-}
 
 CONFIG0=$(cat <<EOF
 <config>
@@ -38,18 +51,14 @@ EOF
 )
 
 CONFIG=${CONFIG:-$CONFIG0}
-STORE=${STORE:-}
 
 # Start clixon-example backend
 # -p 4535 to access via cli from host
 >&2 echo -n "Starting Backend..."
-sudo docker run -p 80:80 --rm -e DBG=$DBG -e CONFIG="$CONFIG" -e STORE="$STORE" -td clixon/clixon-system || err "Error starting clixon-system"
+sudo docker run -p $PORT:80 --name clixon-system --rm -e DBG=$DBG -e CONFIG="$CONFIG" -e STORE="$STORE" -td clixon/clixon-system || err "Error starting clixon-system"
 
 >&2 echo "clixon-system started"
 
-name=clixon/clixon-system
-ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}')
-echo "sudo docker exec -it $ps clixon_cli # example command"
 
 
 

docker/system/startsystem.sh

@@ -45,6 +45,15 @@ server {
 }
 EOF
 
+# This is a clixon site test file. Disable all model testing.
+cat <<EOF > /clixon/clixon/test/site.sh
+# Add your local site specific env variables (or tests) here.
+MODELS=0 # Dont run yangmodels/openconfig  tests
+IETFRFC=/clixon/clixon/yang/standard
+EOF
+
+
+
 # Start nginx
 #/usr/sbin/nginx -g 'daemon off;' -c /etc/nginx/nginx.conf
 /usr/sbin/nginx -c /etc/nginx/nginx.conf
@@ -58,8 +67,6 @@ su -c "/www-data/clixon_restconf -l f/www-data/restconf.log -D $DBG" -s /bin/sh
 >&2 echo "start clixon_backend:"
 /usr/local/sbin/clixon_backend -FD $DBG -s running -l e # logs on docker logs
 
-
+# Alt: let backend be in foreground, but test scripts may
-# Start clixon cli in foreground
+# want to restart backend
-#/usr/local/bin/clixon_cli -D $DBG
+/bin/sleep 100000000
-
-#/bin/sleep 100000000

docker/system/stat.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-# Show stats (IP address etc) about the clixon containers
-# include err(), stat() and other functions
-. ./lib.sh
-
-stat clixon/clixon-system
-
-name=clixon/clixon-system
-ps=$(sudo docker ps -f ancestor=$name|tail -n +2|grep $name|awk '{print $1}')
-echo "sudo docker exec -it $ps clixon_cli # example command"

lib/src/clixon_xml_parse.l

@@ -112,7 +112,6 @@ ncname     {namestart}{namechar}*
                             return NAME; /* rather be catch-all */
                           }
 <START>\:               return *clixon_xml_parsetext;
-<START>\n             { _YA->ya_linenum++;}
 <START><<EOF>>        { return MY_EOF; }
 <START>"<?xml"        { BEGIN(TEXTDECL); return BXMLDCL;}
 <START>"<?"           { BEGIN(PIDECL); return BQMARK;}

lib/src/clixon_yang_parse.l

@@ -103,7 +103,7 @@ identifier      [A-Za-z_][A-Za-z0-9_\-\.]*
 
 %%
  /* Common tokens */
-<KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,ARGUMENT,UNKNOWN>[ \t]            
+<KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,UNKNOWN>[ \t]            
 <KEYWORD,STRING,UNKNOWN,COMMENT2><<EOF>> { return MY_EOF; }
 <KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,COMMENT1,UNKNOWN>\n   { _YY->yy_linenum++; }
 <KEYWORD,BOOLEAN,INTEGER,STRARG,STRING,COMMENT1,UNKNOWN>\r   

test/lib.sh

@@ -46,7 +46,8 @@ testname=
 : ${OPENCONFIG=$(pwd)/public}
 
 # Standard IETF RFC yang files. 
-: ${IETFRFC=$YANGMODELS/standard/ietf/RFC}
+: ${IETFRFC=../yang/standard}
+#: ${IETFRFC=$YANGMODELS/standard/ietf/RFC}
 
 # For memcheck
 #clixon_cli="valgrind --leak-check=full --show-leak-kinds=all clixon_cli"

test/test_openconfig.sh

@@ -3,8 +3,9 @@
 # Notes:
 # - openconfig test suites are patched to counter Clixon issues as follows:
 #    - release/models/mpls/openconfig-mpls-te.yang
-#      issue: https://github.com/clicon/clixon/issues/60
+#      issue: https://github.com/clicon/clixon/issues/6
-# - Env variable YANGMODELS should point to checkout place. (define it in site.sh for example)
+# - Env-var MODELS should be 1
+# - Env-var OPENCONFIG should point to checkout place. (define it in site.sh for example)
 
 APPNAME=example
 
@@ -20,7 +21,8 @@ fyang=$dir/test.yang
 
 new "openconfig"
 if [ ! -d "$OPENCONFIG" ]; then
-    err "Hmm Openconfig dir does not seem to exist, try git clone https://github.com/openconfig/public?"
+#    err "Hmm Openconfig dir does not seem to exist, try git clone https://github.com/openconfig/public?"
+    exit
 fi
 
 OCDIR=$OPENCONFIG/release/models

test/test_yangmodels.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # Parse yangmodels from https://github.com/YangModels/yang
 # Notes:
+# - Env-var MODELS should be 1
 # - Env variable YANGMODELS should point to checkout place. (define it in site.sh for example)
 # - Only cisco/nx/9.2-2 # Many other versions
 # - Only cisco/xe/1631  # Many other versions
@@ -30,7 +31,8 @@ cfg=$dir/conf_yang.xml
 fyang=$dir/test.yang
 
 if [ ! -d "$YANGMODELS" ]; then
-    err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?"
+#    err "Hmm Yangmodels dir does not seem to exist, try git clone https://github.com/YangModels/yang?"
+    exit
 fi
 
 # Experimental IEEE

test/travis/before_script.sh

@@ -9,5 +9,8 @@ sudo make install-include
 (cd example && make && sudo make install)
 sudo groupadd clicon
 sudo usermod -a -G clicon $(whoami)
-sudo usermod -a -G clicon root
 sudo usermod -a -G clicon www-data
+# Build and start the system docker container
+(cd docker/system && make docker && start.sh)
+# Run clixon testcases
+(cd docker/system && sudo docker exec -it clixon-system bash -c 'cd /clixon/clixon/test; exec ./all.sh')