Skip to content

Commit

Permalink
build test executive package and image the same way the daemon is bui…
Browse files Browse the repository at this point in the history
…lt in CI
  • Loading branch information
QuiteStochastic committed Jan 4, 2023
1 parent 018a758 commit 4efb0ac
Show file tree
Hide file tree
Showing 6 changed files with 159 additions and 13 deletions.
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,9 @@ deb_optimized:
@mkdir -p /tmp/artifacts
@cp _build/mina*.deb /tmp/artifacts/.

test_executive_deb:
./scripts/rebuild_test_executive_deb.sh

build_pv_keys: ocaml_checks
$(info Building keys)
ulimit -s 65532 && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only
Expand Down
10 changes: 5 additions & 5 deletions buildkite/scripts/build-artifact.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,14 @@ dune build "--profile=${DUNE_PROFILE}" \
src/app/replayer/replayer.exe \
src/app/swap_bad_balances/swap_bad_balances.exe \
src/app/rosetta/rosetta_mainnet_signatures.exe \
src/app/rosetta/rosetta_testnet_signatures.exe # 2>&1 | tee /tmp/buildocaml.log
src/app/rosetta/rosetta_testnet_signatures.exe \
src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log

echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
echo " Includes mina daemon, archive-node, rosetta, generate keypair for mainnet and devnet"
make deb

echo "--- Upload debs to amazon s3 repo"
make publish_debs
make test_executive_deb

echo "--- Copy artifacts to cloud"
# buildkite-agent artifact upload occurs outside of docker after this script exits
echo "--- Upload debs to amazon s3 repo"
make publish_debs
18 changes: 10 additions & 8 deletions buildkite/src/Command/MinaArtifact.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="daemon-devnet-${DebianVersions.lowerName debVersion}-docker-image"
}

in

DockerImage.generateStep daemonDevnetSpec,

-- daemon mainnet image
Expand All @@ -69,21 +67,27 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="daemon-mainnet-${DebianVersions.lowerName debVersion}-docker-image"
}

in

DockerImage.generateStep daemonMainnetSpec,

-- test_executive image
let testExecutiveSpec = DockerImage.ReleaseSpec::{
deps=DebianVersions.dependsOn debVersion,
service="mina-test-executive",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="test-executive-${DebianVersions.lowerName debVersion}-docker-image"
}
in
DockerImage.generateStep testExecutiveSpec,

-- archive image
let archiveSpec = DockerImage.ReleaseSpec::{
deps=DebianVersions.dependsOn debVersion,
service="mina-archive",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="archive-${DebianVersions.lowerName debVersion}-docker-image"
}

in

DockerImage.generateStep archiveSpec,

-- rosetta image
Expand All @@ -93,9 +97,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image"
}

in

DockerImage.generateStep rosettaSpec

]
Expand Down
76 changes: 76 additions & 0 deletions dockerfiles/Dockerfile-mina-test-executive
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
ARG image=debian:stretch-slim
FROM ${image}
# Run with `docker build --build-arg deb_version=<version>`
ARG deb_version
ARG deb_release=unstable
ARG deb_codename=stretch

ENV DEBIAN_FRONTEND noninteractive

# Dependencies
RUN apt-get -y update && \
DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \
DEBIAN_FRONTEND=noninteractive apt-get -y install \
apt-transport-https \
ca-certificates \
curl \
dnsutils \
dumb-init \
gettext \
gnupg2 \
jq \
libgmp10 \
libgomp1 \
libssl1.1 \
libpq-dev \
procps \
python3 \
tzdata && \
rm -rf /var/lib/apt/lists/*

# --- Terraform tools
RUN curl -sL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -o terraform.zip \
&& unzip terraform.zip \
&& sudo mv terraform /usr/bin \
&& sudo ln -s /usr/bin/terraform /usr/local/bin/terraform

# --- Google Cloud tools
RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
&& curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - \
&& sudo apt-get update -y \
&& sudo apt-get install -y google-cloud-sdk kubectl google-cloud-sdk-gke-gcloud-auth-plugin

# --- Helm tools
RUN curl https://baltocdn.com/helm/signing.asc | sudo apt-key add - \
&& echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list \
&& sudo apt-get update -y \
&& sudo apt-get install -y helm

# Get yarn + nodejs
RUN curl -sL https://deb.nodesource.com/setup_lts.x | sudo -E bash - \
&& curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - \
&& echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list \
&& sudo apt update \
&& sudo apt install -y nodejs yarn

# Mina test executive package
RUN echo "Building test_executive image with version $deb_version from repo $deb_release $deb_codename " \
&& echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \
&& apt-get update \
&& apt-get install -y --allow-downgrades "mina-test-executive=$deb_version"

# Move to a non-root UID in the future (specifically 50000, as it is memorable and safely within the bounds of most systems)
# for now stick to root for compatibility
ARG UID=0

## Reset workdir, USER, and ${UID} for root-owned version
WORKDIR /root/
USER 0

# these env vars need to be overriden when the image is `docker run`
ENV test_name=""
ENV mina_image=""
ENV archiver_image=""
ENV debug_bool=true

ENTRYPOINT ["/usr/bin/dumb-init", "mina-test-executive cloud $test_name --mina-image $mina_image --archive_image $archiver_image $( if [[ $debug_bool ]] ; then echo --debug ; fi ) | tee test.log | logproc -i inline -f '!(.level in ["Spam", "Debug"])'"]
62 changes: 62 additions & 0 deletions rebuild_test_executive_deb.sh.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/bin/bash

# Script builds the debian package for the test_executive

set -euo pipefail

SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
cd "${SCRIPTPATH}/../_build"

GITHASH=$(git rev-parse --short=7 HEAD)
GITHASH_CONFIG=$(git rev-parse --short=8 --verify HEAD)

set +u
BUILD_NUM=${BUILDKITE_BUILD_NUM}
BUILD_URL=${BUILDKITE_BUILD_URL}
set -u

# Load in env vars for githash/branch/etc.
source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh"

cd "${SCRIPTPATH}/../_build"

BUILDDIR="deb_build"

##################################### GENERATE TEST_EXECUTIVE PACKAGE #######################################

mkdir -p "${BUILDDIR}/DEBIAN"
cat << EOF > "${BUILDDIR}/DEBIAN/control"
Package: mina-test-executive
Version: ${MINA_DEB_VERSION}
License: Apache-2.0
Vendor: none
Architecture: amd64
Maintainer: o(1)Labs <[email protected]>
Installed-Size:
Depends: asadsfasdfasdfasdfqewrWQERQWEFASDFVXCVXCZ
Section: base
Priority: optional
Homepage: https://minaprotocol.com/
Description: Utility to run automated tests against a full mina testnet with multiple nodes.
Built from ${GITHASH} by ${BUILD_URL}
EOF

echo "------------------------------------------------------------"
echo "Control File:"
cat "${BUILDDIR}/DEBIAN/control"

# Binaries
mkdir -p "${BUILDDIR}/usr/local/bin"
cp ./default/src/app/test_executive/test_executive.exe "${BUILDDIR}/usr/local/bin/mina-test-executive"


# echo contents of deb
echo "------------------------------------------------------------"
echo "Deb Contents:"
find "${BUILDDIR}"

# Build the package
echo "------------------------------------------------------------"
fakeroot dpkg-deb --build "${BUILDDIR}" mina-test-executive-${MINA_DEB_VERSION}.deb
ls -lh mina*.deb
3 changes: 3 additions & 0 deletions scripts/release-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ mina-daemon)
mina-toolchain)
DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-toolchain"
;;
mina-test-executive)
DOCKERFILE_PATH="dockerfiles/Dockerfile-test-executive"
;;
mina-rosetta)
DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-builder dockerfiles/stages/4-production"
;;
Expand Down

0 comments on commit 4efb0ac

Please sign in to comment.