End-to-End testing using docker

Since Travis CI only supports ancient Ubuntu versions we need a lot of
hacks to make things work. At the same time we use docker for normal
builds. So we can simplify things by just doing the end-to-end tests in
docker too. This needs a bit more flexibility in the e2e/e2e.sh script.

Also so we don't run the end-to-end tests (and their test data download)
on each docker build we introduce a docker-compose.test.yml for docker
cloud. On Travis we manually run the appropriate docker commands

.travis.yml

@@ -1,39 +1,21 @@
 language: generic
-sudo: false
-dist: trusty
+sudo: required
 
+
+# We need a newer docker version since our multi-stage build
+# relies on old stages not leaking into the current stage
+# see https://github.com/moby/moby/issues/35641
 addons:
   apt:
-    sources:
-      - deadsnakes
-      - ubuntu-toolchain-r-test
     packages:
-      - gcc-7
-      - g++-7
-      - libsparsehash-dev
-      - python3.6
-      - python3-yaml
-      - cmake
-      - netcat
-
-cache:
-  directories:
-    - e2e_data/scientist-collection/
-
-env:
-  - CC=gcc-7 CXX=g++-7
+      - docker-ce
 
-before_script:
-  - $CXX --version
-  - mkdir build
-  - cd build
-  - cmake ..
+services:
+  - docker
 
 script:
-  - make -j 3
-  - make test
-  - cd ..
-  - e2e/e2e.sh
+  - docker build -t qlever .
+  - docker run -it --rm --entrypoint e2e/e2e.sh qlever
 
 notifications:
   email:

Dockerfile

@@ -12,12 +12,15 @@ RUN cmake -DCMAKE_BUILD_TYPE=Release .. && make -j $(nproc) && make test
 
 FROM base as runtime
 WORKDIR /app
+RUN apt-get update && apt-get install -y wget python3-yaml unzip curl
 ARG UID=1000
 RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app
-USER qlever
-ENV PATH=/app/:$PATH
+
 COPY --from=builder /app/build/*Main /app/src/web/* /app/
+COPY --from=builder /app/e2e/* /app/e2e/
+ENV PATH=/app/:$PATH
 
+USER qlever
 EXPOSE 7001
 VOLUME ["/input", "/index"]
 
@@ -34,7 +37,7 @@ CMD ["-t", "-a", "-P"]
 # # index.* or
 # # set the envirionment variable "INDEX_PREFIX" during `docker run` using `-e INDEX_PREFIX=<prefix>`
 # # To build an index run a bash inside the container as follows
-# docker run -it --rm -v "<path_to_input>:/input" -v "$(pwd)/index:/index" qlever-<name> bash
+# docker run -it --rm --entrypoint bash -v "<path_to_input>:/input" -v "$(pwd)/index:/index" qlever-<name>
 # # Then inside that shell IndexBuilder is in the path and can be used like
 # # described in the README.md with the files in /input
 # # To run a server use

docker-compose.test.yml

@@ -0,0 +1,3 @@
+sut:
+  build: .
+  entrypoint: e2e/e2e.sh

e2e/e2e.sh

@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 set -e
-PROJECT_DIR="$(dirname ${BASH_SOURCE[0]})/.."
+PROJECT_DIR=$(readlink -f -- "$(dirname ${BASH_SOURCE[0]})/..")
 # Change to the project directory so we can use simple relative paths
-cd "$PROJECT_DIR"
+echo "Changing to project directory: $PROJECT_DIR"
+pushd $PROJECT_DIR
+BINARY_DIR=$(readlink -f -- ./build)
+if [ ! -e $BINARY_DIR ]; then
+	BINARY_DIR=$(readlink -f -- .)
+fi
 function bail {
 	echo "$*"
 	exit 1
 }
 
 function cleanup_server {
 	echo "The Server Log follows:"
-	cat "build/server_log.txt"
+	cat "$BINARY_DIR/server_log.txt"
 	# Killing 0 sends the signal to all processes in the current
 	# process group
 	kill $SERVER_PID
@@ -25,34 +30,44 @@ else
 	export PYTHON_BINARY=`which python3`
 fi
 
-mkdir -p "e2e_data"
+INDEX_DIR="$PROJECT_DIR/e2e_data"
+INPUT_DIR="$INDEX_DIR/scientist-collection"
+INPUT_PREFIX="scientists"
+INPUT="$INPUT_DIR/$INPUT_PREFIX"
+mkdir -p "$INDEX_DIR"
 # Can't check for the scientist-collection directory because
 # Travis' caching creates it
-if [ ! -e "e2e_data/scientist-collection/scientists.nt" ]; then
+if [ ! -e "$INPUT.nt" ]; then
 	# Why the hell is this a ZIP that can't easily be decompressed from stdin?!?
-	wget -O "e2e_data/scientist-collection.zip" \
+	echo "Downloading $INPUT_PREFIX KB input files"
+	wget --quiet -O "$INDEX_DIR/scientist-collection.zip" \
 		"http://filicudi.informatik.uni-freiburg.de/bjoern-data/scientist-collection.zip"
-	unzip "e2e_data/scientist-collection.zip" -d "e2e_data"
+	unzip -j "$INDEX_DIR/scientist-collection.zip" -d "$INPUT_DIR/"
+	rm "$INDEX_DIR/scientist-collection.zip"
 fi;
 
-INDEX="e2e_data/scientists-index"
+
+INDEX_PREFIX="scientists-index"
+INDEX="$INDEX_DIR/$INDEX_PREFIX"
 
 # Delete and rebuild the index
 if [ "$1" != "no-index" ]; then
 	rm -f "$INDEX.*"
-	pushd "./build"
-	./IndexBuilderMain -a -l -i "../$INDEX" \
-		-n "../e2e_data/scientist-collection/scientists.nt" \
-		-w "../e2e_data/scientist-collection/scientists.wordsfile.tsv" \
-		-d "../e2e_data/scientist-collection/scientists.docsfile.tsv" \
+	pushd "$BINARY_DIR"
+	echo "Building index $INDEX"
+	./IndexBuilderMain -a -l -i "$INDEX" \
+		-n "$INPUT.nt" \
+		-w "$INPUT.wordsfile.tsv" \
+		-d "$INPUT.docsfile.tsv" \
 		--patterns || bail "Building Index failed"
 	popd
 fi
 
 # Launch the Server using the freshly baked index. Can't simply use a subshell here because
 # then we can't easily get the SERVER_PID out of that subshell
-pushd "./build"
-./ServerMain -i "../$INDEX" -p 9099 -t -a --patterns &> server_log.txt &
+pushd "$BINARY_DIR"
+echo "Launching server from path $(pwd)"
+./ServerMain -i "$INDEX" -p 9099 -t -a --patterns &> server_log.txt &
 SERVER_PID=$!
 popd
 
@@ -62,4 +77,5 @@ echo "Waiting for ServerMain to launch and open port"
 while ! curl --max-time 1 --output /dev/null --silent http://localhost:9099/; do
 	sleep 1
 done
-$PYTHON_BINARY e2e/queryit.py "e2e/scientists_queries.yaml" "http://localhost:9099" || bail "Querying Server failed"
+$PYTHON_BINARY "$PROJECT_DIR/e2e/queryit.py" "$PROJECT_DIR/e2e/scientists_queries.yaml" "http://localhost:9099" || bail "Querying Server failed"
+popd