ci: migrate the protocol test on docker image

Signed-off-by: Vincenzo Palazzo <[email protected]>

.github/workflows/ci.yaml

@@ -9,6 +9,7 @@ jobs:
   smoke-test:
     name: Smoke Test ${{ matrix.cfg }}
     runs-on: ubuntu-20.04
+    timeout-minutes: 300
     env:
       DEVELOPER: 1
       VALGRIND: 0
@@ -98,15 +99,8 @@ jobs:
   proto-test:
     name: Protocol Test Config
     runs-on: ubuntu-20.04
+    timeout-minutes: 300
     needs: [smoke-test]
-    env:
-      DEVELOPER: 1
-      EXPERIMENTAL_FEATURES: 1
-      COMPAT: 0
-      PYTEST_PAR: 2
-      TEST_CMD: "make check-protos"
-      TEST_GROUP: 1
-      TEST_GROUP_COUNT: 1
     strategy:
       fail-fast: true
       matrix:
@@ -116,27 +110,23 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/[email protected]
-
-      - name: Setup Python 3.7
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Install dependencies
+      - name: Build and run
         run: |
-          bash -x .github/scripts/setup.sh
-
-      - name: Build
-        env:
-          ARCH: ${{ matrix.arch }}
-          COMPILER: ${{ matrix.compiler }}
-          DB: ${{ matrix.db }}
-          NETWORK: ${{ matrix.network }}
-          TARGET_HOST: ${{ matrix.TARGET_HOST }}
-          VALGRIND: ${{ matrix.valgrind }}
-        run: |
-          bash -x .github/scripts/build.sh
-
+          docker build -f contrib/docker/Dockerfile.ubuntu -t cln-ci-ubuntu .
+          docker run -e ARCH=${{ matrix.arch }} \
+          -e COMPILER=${{ matrix.compiler }} \
+          -e DB=${{ matrix.db }} \
+          -e NETWORK=${{ matrix.network }} \
+          -e TARGET_HOST=${{ matrix.TARGET_HOST }} \
+          -e VALGRIND=${{ matrix.valgrind }} \
+          -e DEVELOPER=1 \
+          -e EXPERIMENTAL_FEATURES=1 \
+          -e COMPA=0 \
+          -e PYTEST_PAR=2 \
+          -e TEST_CMD="make check-protos" \
+          -e TEST_GROUP=1 \
+          -e TEST_GROUP_COUNT=1 \
+          cln-ci-ubuntu
       - name: Upload Unit Test Results
         if: always()
         uses: actions/upload-artifact@v2

contrib/docker/Dockerfile.ubuntu

@@ -0,0 +1,36 @@
+FROM ubuntu:20.04
+LABEL mantainer="Vincenzo Palazzo [email protected]"
+
+WORKDIR /work
+
+COPY . .
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANGUAGE=en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+ENV TZ="Europe/London"
+
+RUN apt-get -qq update && \
+    apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
+    sudo \
+    locales \
+    tzdata
+
+RUN ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime
+
+RUN locale-gen en_US.UTF-8 && dpkg-reconfigure --frontend noninteractive tzdata
+
+COPY . .
+
+# install package for pytho cryptography lib
+# https://cryptography.io/en/latest/installation/#debian-ubuntu
+
+RUN apt-get -qq update && \
+    apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
+    build-essential libssl-dev libffi-dev \
+    python3-dev cargo
+
+RUN ls -la
+
+CMD ["./contrib/docker/scripts/entrypoint.sh"]

contrib/docker/scripts/build.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+echo "Running in $(pwd)"
+export ARCH=${ARCH:-64}
+export BOLTDIR=lightning-rfc
+export CC=${COMPILER:-gcc}
+export COMPAT=${COMPAT:-1}
+export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
+export DEVELOPER=${DEVELOPER:-1}
+export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
+export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}"
+export PYTEST_PAR=${PYTEST_PAR:-10}
+export PYTEST_SENTRY_ALWAYS_REPORT=1
+export SLOW_MACHINE=1
+export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
+export TEST_DB_PROVIDER=${TEST_DB_PROVIDER:-"sqlite3"}
+export TEST_NETWORK=${NETWORK:-"regtest"}
+export TIMEOUT=900
+export VALGRIND=${VALGRIND:-0}
+export FUZZING=${FUZZING:-0}
+export LIGHTNINGD_POSTGRES_NO_VACUUM=1
+
+pip3 install --user poetry
+poetry config virtualenvs.create false --local
+poetry install
+
+git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
+git submodule update --init --recursive
+
+./configure CC="$CC"
+cat config.vars
+
+cat << EOF > pytest.ini
+[pytest]
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3
+markers =
+    slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+EOF
+
+if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
+then
+    export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
+    export MAKE_HOST="$TARGET_HOST"
+    export BUILD=x86_64-pc-linux-gnu
+    export AR="$TARGET_HOST"-ar
+    export AS="$TARGET_HOST"-as
+    export CC="$TARGET_HOST"-gcc
+    export CXX="$TARGET_HOST"-g++
+    export LD="$TARGET_HOST"-ld
+    export STRIP="$TARGET_HOST"-strip
+    export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
+
+    wget -q https://zlib.net/zlib-1.2.11.tar.gz
+    tar xf zlib-1.2.11.tar.gz
+    cd zlib-1.2.11 || exit 1
+    ./configure --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
+
+    wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
+    unzip -q sqlite-src-3260000.zip
+    cd sqlite-src-3260000 || exit 1
+    automake --add-missing --force-missing --copy || true
+    ./configure --disable-tcl \
+     --enable-static \
+     --disable-readline \
+     --disable-threadsafe \
+     --disable-load-extension \
+     --host="$TARGET_HOST" \
+     --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm sqlite-src-3260000.zip
+    rm -rf sqlite-src-3260000
+
+    wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
+    tar xf gmp-6.1.2.tar.xz
+    cd gmp-6.1.2 || exit 1
+    ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST"
+    make
+    sudo make install
+    cd ..
+    rm gmp-6.1.2.tar.xz
+    rm -rf gmp-6.1.2
+
+    ./configure CC="$TARGET_HOST-gcc" --enable-static
+
+    make -j32 CC="$TARGET_HOST-gcc" > /dev/null
+else
+    eatmydata make -j32
+    # shellcheck disable=SC2086
+    eatmydata $TEST_CMD
+fi

contrib/docker/scripts/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd contrib/docker/scripts && \
+    ./setup.sh && ./build.sh

contrib/docker/scripts/setup.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+export BITCOIN_VERSION=0.20.1
+export ELEMENTS_VERSION=0.18.1.8
+export RUST_VERSION=nightly
+export TZ="Europe/London"
+
+sudo useradd -ms /bin/bash tester
+
+sudo apt-get update -qq
+
+sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
+     autoconf \
+     automake \
+     binfmt-support \
+     build-essential \
+     clang \
+     cppcheck \
+     docbook-xml \
+     eatmydata \
+     gcc-aarch64-linux-gnu \
+     gcc-arm-linux-gnueabihf \
+     gcc-arm-none-eabi \
+     gettext \
+     git \
+     libc6-dev-arm64-cross \
+     libc6-dev-armhf-cross \
+     libgmp-dev \
+     libpq-dev \
+     libprotobuf-c-dev \
+     libsqlite3-dev \
+     libtool \
+     libxml2-utils \
+     locales \
+     net-tools \
+     postgresql \
+     python-pkg-resources \
+     python3 \
+     python3-dev \
+     python3-pip \
+     python3-setuptools \
+     qemu \
+     qemu-system-arm \
+     qemu-user-static \
+     shellcheck \
+     software-properties-common \
+     sudo \
+     tcl \
+     unzip \
+     valgrind \
+     wget \
+     xsltproc \
+     zlib1g-dev
+
+
+echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
+sudo chmod 0440 /etc/sudoers.d/tester
+
+(
+    cd /tmp/ || exit 1
+    wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+    sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin
+    rm -rf \
+       bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
+       bitcoin-$BITCOIN_VERSION \
+       elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
+       elements-$ELEMENTS_VERSION
+)
+
+if [ "$RUST" == "1" ]; then
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
+      -y --default-toolchain ${RUST_VERSION}
+fi