Merge remote-tracking branch 'remotes/stsquad/tags/pull-ci-updates-fo…

…r-softfreeze-180717-2' into staging

Final CI updates for soft-freeze

Tweaks from Paolo for J=x Travis compiles
Bunch of updated cross-compile targets from Philippe
Additional debug tools in travis image from Me

# gpg: Signature made Tue 18 Jul 2017 11:00:26 BST
# gpg:                using RSA key 0xFBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <[email protected]>"
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-ci-updates-for-softfreeze-180717-2: (32 commits)
  docker: install clang since Shippable setup_ve() verify it is available
  docker: warn users to use newer debian8/debian9 base image
  docker: add debian Ports base image
  shippable: add win32/64 targets
  docker: add MXE (M cross environment) base image for MinGW-w64
  shippable: add mips64el targets
  docker: add debian/mips64el image
  shippable: use debian/mips[eb] targets
  docker: add debian/mips[eb] images
  shippable: add powerpc target
  docker: add debian/powerpc based on Jessie
  docker: add 'apt-fake' script which generate fake debian packages
  docker: add qemu:debian-jessie based on outdated jessie release
  shippable: add x86_64 targets
  shippable: add ppc64el targets
  shippable: add armel targets
  docker: enable nettle to extend code coverage on arm64
  docker: enable gcrypt to extend code coverage on amd64
  docker: enable netmap to extend code coverage on amd64
  docker: enable virgl to extend code coverage on amd64
  ...

Signed-off-by: Peter Maydell <[email protected]>

.shippable.yml

@@ -5,15 +5,28 @@ env:
   global:
     - LC_ALL=C
   matrix:
+    - IMAGE=debian-amd64
+      TARGET_LIST=x86_64-softmmu,x86_64-linux-user
+    - IMAGE=debian-win32-cross
+      TARGET_LIST=arm-softmmu,i386-softmmu,lm32-softmmu
+    - IMAGE=debian-win64-cross
+      TARGET_LIST=aarch64-softmmu,sparc64-softmmu,x86_64-softmmu
+    - IMAGE=debian-armel-cross
+      TARGET_LIST=arm-softmmu,arm-linux-user,armeb-linux-user
     - IMAGE=debian-armhf-cross
       TARGET_LIST=arm-softmmu,arm-linux-user,armeb-linux-user
     - IMAGE=debian-arm64-cross
       TARGET_LIST=aarch64-softmmu,aarch64-linux-user
     - IMAGE=debian-s390x-cross
       TARGET_LIST=s390x-softmmu,s390x-linux-user
-    # mips64el-softmmu disabled due to libfdt problem
-    - IMAGE=debian-mipsel-cross
-      TARGET_LIST=mipsel-softmmu,mipsel-linux-user,mips64el-linux-user
+    - IMAGE=debian-mips-cross
+      TARGET_LIST=mips-softmmu,mipsel-linux-user
+    - IMAGE=debian-mips64el-cross
+      TARGET_LIST=mips64el-softmmu,mips64el-linux-user
+    - IMAGE=debian-powerpc-cross
+      TARGET_LIST=ppc-softmmu,ppcemb-softmmu,ppc-linux-user
+    - IMAGE=debian-ppc64el-cross
+      TARGET_LIST=ppc64-softmmu,ppc64-linux-user,ppc64abi32-linux-user
 build:
   pre_ci:
     - make docker-image-${IMAGE} V=1

.travis.yml

@@ -46,6 +46,7 @@ notifications:
 env:
   global:
     - TEST_CMD="make check"
+    - MAKEFLAGS="-j3"
   matrix:
     - CONFIG=""
     - CONFIG="--enable-debug --enable-debug-tcg --enable-trace-backends=log"
@@ -64,7 +65,7 @@ before_install:
 before_script:
   - ./configure ${CONFIG}
 script:
-  - make -j3 && ${TEST_CMD}
+  - make ${MAKEFLAGS} && ${TEST_CMD}
 matrix:
   include:
     # Test with CLang for compile portability

tests/docker/Makefile.include

@@ -55,10 +55,22 @@ docker-image-%: $(DOCKER_FILES_DIR)/%.docker
 		$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
 		"BUILD","$*")
 
+docker-image-debian-powerpc-cross: EXTRA_FILES:=tests/docker/dockerfiles/debian-apt-fake.sh
+
 # Enforce dependancies for composite images
-docker-image-debian-armhf-cross: docker-image-debian
-docker-image-debian-arm64-cross: docker-image-debian
-docker-image-debian-mipsel-cross: docker-image-debian
+docker-image-debian: docker-image-debian9
+docker-image-debian8-mxe: docker-image-debian8
+docker-image-debian-amd64: docker-image-debian9
+docker-image-debian-armel-cross: docker-image-debian9
+docker-image-debian-armhf-cross: docker-image-debian9
+docker-image-debian-arm64-cross: docker-image-debian9
+docker-image-debian-mips-cross: docker-image-debian9
+docker-image-debian-mips64el-cross: docker-image-debian9
+docker-image-debian-powerpc-cross: docker-image-debian8
+docker-image-debian-ppc64el-cross: docker-image-debian9
+docker-image-debian-s390x-cross: docker-image-debian9
+docker-image-debian-win32-cross: docker-image-debian8-mxe
+docker-image-debian-win64-cross: docker-image-debian8-mxe
 
 # Expand all the pre-requistes for each docker image and test combination
 $(foreach i,$(DOCKER_IMAGES), \

tests/docker/dockerfiles/debian-amd64.docker

@@ -0,0 +1,39 @@
+#
+# Docker x86_64 target
+#
+# This docker target builds on the debian Stretch base image. Further
+# libraries which are not widely available are installed by hand.
+#
+FROM qemu:debian9
+MAINTAINER Philippe Mathieu-Daudé <[email protected]>
+
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get build-dep -yy qemu
+
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        libbz2-dev \
+        liblzo2-dev \
+        librdmacm-dev \
+        libsnappy-dev \
+        libvte-dev
+
+# virgl
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        libegl1-mesa-dev \
+        libepoxy-dev \
+        libgbm-dev
+RUN git clone git://anongit.freedesktop.org/virglrenderer /usr/src/virglrenderer
+RUN cd /usr/src/virglrenderer && ./autogen.sh && ./configure --with-glx --disable-tests && make install
+
+# netmap
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        linux-headers-amd64
+RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap
+RUN cd /usr/src/netmap/LINUX && ./configure --no-drivers --no-apps --kernel-dir=$(ls -d /usr/src/linux-headers-*-amd64) && make install
+ENV QEMU_CONFIGURE_OPTS --enable-netmap
+
+# gcrypt
+ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-gcrypt

tests/docker/dockerfiles/debian-apt-fake.sh

@@ -0,0 +1,46 @@
+#! /bin/sh
+#
+# Generate fake debian package to resolve unimportant unmet dependencies held
+# by upstream multiarch broken packages.
+#
+# Copyright (c) 2017 Philippe Mathieu-Daudé <[email protected]>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+test $1 = "install" && shift 1
+
+fake_install()
+{
+    echo "Generating fake $2 $1 $3 ..."
+    (cd /var/cache/apt/archives
+        (cat << 'EOF'
+Section: misc
+Priority: optional
+Standards-Version: 3.9.2
+
+Package: NAME
+Version: VERSION
+Maintainer: [email protected]
+Architecture: any
+Multi-Arch: same
+Description: fake NAME
+EOF
+        ) | sed s/NAME/$2/g | sed s/VERSION/$3/g > $2.control
+        equivs-build -a $1 $2.control 1>/dev/null 2>/dev/null
+        dpkg -i --force-overwrite $2_$3_$1.deb
+    )
+}
+
+try_install()
+{
+    name=$(echo $1|sed "s/\(.*\):\(.*\)=\(.*\)/\1/")
+    arch=$(echo $1|sed "s/\(.*\):\(.*\)=\(.*\)/\2/")
+    vers=$(echo $1|sed "s/\(.*\):\(.*\)=\(.*\)/\3/")
+    apt-get install -q -yy $1 || fake_install $arch $name $vers
+}
+
+for package in $*; do
+    try_install $package
+done

tests/docker/dockerfiles/debian-arm64-cross.docker

@@ -1,9 +1,9 @@
 #
 # Docker arm64 cross-compiler target
 #
-# This docker target builds on the base debian image.
+# This docker target builds on the debian Stretch base image.
 #
-FROM qemu:debian
+FROM qemu:debian9
 
 # Add the foreign architecture we want and install dependencies
 RUN dpkg --add-architecture arm64
@@ -19,11 +19,11 @@ ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu-
 
 RUN DEBIAN_FRONTEND=noninteractive eatmydata \
     apt-get install -y --no-install-recommends \
-        glusterfs-common:arm64 \
         libbz2-dev:arm64 \
         liblzo2-dev:arm64 \
-        libncursesw5-dev:arm64 \
-        libnfs-dev:arm64 \
         librdmacm-dev:arm64 \
         libsnappy-dev:arm64 \
         libxen-dev:arm64
+
+# nettle
+ENV QEMU_CONFIGURE_OPTS $QEMU_CONFIGURE_OPTS --enable-nettle

tests/docker/dockerfiles/debian-armel-cross.docker

@@ -0,0 +1,24 @@
+#
+# Docker armel cross-compiler target
+#
+# This docker target builds on the debian Stretch base image.
+#
+FROM qemu:debian9
+MAINTAINER Philippe Mathieu-Daudé <[email protected]>
+
+# Add the foreign architecture we want and install dependencies
+RUN dpkg --add-architecture armel && \
+    apt update
+RUN apt install -yy crossbuild-essential-armel
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get build-dep -yy -a armel qemu
+
+# Specify the cross prefix for this image (see tests/docker/common.rc)
+ENV QEMU_CONFIGURE_OPTS --cross-prefix=arm-linux-gnueabi-
+
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        libbz2-dev:armel \
+        liblzo2-dev:armel \
+        librdmacm-dev:armel \
+        libsnappy-dev:armel

tests/docker/dockerfiles/debian-armhf-cross.docker

@@ -1,9 +1,9 @@
 #
 # Docker armhf cross-compiler target
 #
-# This docker target builds on the base debian image.
+# This docker target builds on the debian Stretch base image.
 #
-FROM qemu:debian
+FROM qemu:debian9
 
 # Add the foreign architecture we want and install dependencies
 RUN dpkg --add-architecture armhf
@@ -19,11 +19,8 @@ ENV QEMU_CONFIGURE_OPTS --cross-prefix=arm-linux-gnueabihf-
 
 RUN DEBIAN_FRONTEND=noninteractive eatmydata \
     apt-get install -y --no-install-recommends \
-        glusterfs-common:armhf \
         libbz2-dev:armhf \
         liblzo2-dev:armhf \
-        libncursesw5-dev:armhf \
-        libnfs-dev:armhf \
         librdmacm-dev:armhf \
         libsnappy-dev:armhf \
         libxen-dev:armhf

tests/docker/dockerfiles/debian-mips-cross.docker

@@ -0,0 +1,29 @@
+#
+# Docker mips cross-compiler target
+#
+# This docker target builds on the debian Stretch base image.
+#
+FROM qemu:debian9
+
+MAINTAINER Philippe Mathieu-Daudé <[email protected]>
+
+# Add the foreign architecture we want and install dependencies
+RUN dpkg --add-architecture mips
+RUN apt-get update
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        gcc-mips-linux-gnu
+
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get build-dep -yy -a mips qemu
+
+# Specify the cross prefix for this image (see tests/docker/common.rc)
+ENV QEMU_CONFIGURE_OPTS --cross-prefix=mips-linux-gnu-
+
+# Install extra libraries to increase code coverage
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        libbz2-dev:mips \
+        liblzo2-dev:mips \
+        librdmacm-dev:mips \
+        libsnappy-dev:mips

tests/docker/dockerfiles/debian-mips64el-cross.docker

@@ -0,0 +1,30 @@
+#
+# Docker mips64el cross-compiler target
+#
+# This docker target builds on the debian Stretch base image.
+#
+
+FROM qemu:debian9
+
+MAINTAINER Philippe Mathieu-Daudé <[email protected]>
+
+# Add the foreign architecture we want and install dependencies
+RUN dpkg --add-architecture mips64el && \
+    apt-get update
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        gcc-mips64el-linux-gnuabi64
+
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get build-dep -yy -a mips64el qemu
+
+# Specify the cross prefix for this image (see tests/docker/common.rc)
+ENV QEMU_CONFIGURE_OPTS --cross-prefix=mips64el-linux-gnuabi64-
+
+# Install extra libraries to increase code coverage
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        libbz2-dev:mips64el \
+        liblzo2-dev:mips64el \
+        librdmacm-dev:mips64el \
+        libsnappy-dev:mips64el

tests/docker/dockerfiles/debian-mipsel-cross.docker

@@ -1,29 +1,29 @@
 #
 # Docker mipsel cross-compiler target
 #
-# This docker target builds on the base debian image.
+# This docker target builds on the debian Stretch base image.
 #
-FROM qemu:debian
+FROM qemu:debian9
+
 MAINTAINER Philippe Mathieu-Daudé <[email protected]>
 
 # Add the foreign architecture we want and install dependencies
 RUN dpkg --add-architecture mipsel
 RUN apt-get update
 RUN DEBIAN_FRONTEND=noninteractive eatmydata \
     apt-get install -y --no-install-recommends \
-        crossbuild-essential-mipsel
+        gcc-mipsel-linux-gnu
+
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get build-dep -yy -a mipsel qemu
 
 # Specify the cross prefix for this image (see tests/docker/common.rc)
 ENV QEMU_CONFIGURE_OPTS --cross-prefix=mipsel-linux-gnu-
 
-RUN DEBIAN_FRONTEND=noninteractive eatmydata \
-    apt-get build-dep -yy -a mipsel qemu
+# Install extra libraries to increase code coverage
 RUN DEBIAN_FRONTEND=noninteractive eatmydata \
     apt-get install -y --no-install-recommends \
-        glusterfs-common:mipsel \
         libbz2-dev:mipsel \
         liblzo2-dev:mipsel \
-        libncursesw5-dev:mipsel \
-        libnfs-dev:mipsel \
         librdmacm-dev:mipsel \
         libsnappy-dev:mipsel

tests/docker/dockerfiles/debian-ports.docker

@@ -0,0 +1,35 @@
+#
+# Docker multiarch cross-compiler target
+#
+# This docker target is builds on Debian Ports cross compiler targets
+# to build distro with a selection of cross compilers for building test binaries.
+#
+# On its own you can't build much but the docker-foo-cross targets
+# build on top of the base debian image.
+#
+FROM debian:unstable
+
+MAINTAINER Philippe Mathieu-Daudé <[email protected]>
+
+RUN echo "deb [arch=amd64] http://deb.debian.org/debian unstable main" > /etc/apt/sources.list
+
+# Duplicate deb line as deb-src
+RUN cat /etc/apt/sources.list | sed -ne "s/^deb\ \(\[.*\]\ \)\?\(.*\)/deb-src \2/p" >> /etc/apt/sources.list
+
+# Setup some basic tools we need
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -yy eatmydata
+RUN DEBIAN_FRONTEND=noninteractive eatmydata \
+    apt-get install -y --no-install-recommends \
+        bison \
+        build-essential \
+        ca-certificates \
+        clang \
+        debian-ports-archive-keyring \
+        flex \
+        git \
+        pkg-config \
+        psmisc \
+        python \
+        texinfo \
+        $(apt-get -s build-dep qemu | egrep ^Inst | fgrep '[all]' | cut -d\  -f2)