Skip to content

Commit

Permalink
Extract dependency installation steps into a dedicated script.
Browse files Browse the repository at this point in the history
There are other cleanups and optimizations that can be made, but this is a mostly direct copy.

Tested via `docker run` and direct invocation from a Linux VM.
  • Loading branch information
remoun committed Jan 18, 2022
1 parent 9d80ecf commit 9b6a10b
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 113 deletions.
2 changes: 1 addition & 1 deletion BUILD.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ run the tests in `SGX_MODE=HW`. (See below for an explanation.)

A docker-less build also works fine for development:
- Follow instructions [consensus/service/BUILD.md](consensus/service/BUILD.md)
- Set up your environment like the [Dockerfile](docker/Dockerfile)
- Set up your environment with `docker/init_debian.sh`

## Build configuration

Expand Down
123 changes: 12 additions & 111 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,134 +8,35 @@
# if it isn't done, bad things will happen to local builds and CI using mob tool
# See mob tool comments for extended discussion

# builder-install contains dependencies but doesn't copy our sources in, so it can be
# cached in google cloud.
#
# builder-install contains dependencies but doesn't copy
# our sources in, so it can be cached in google cloud.


# ##################################### #
# builder-install - Build environment #
# ##################################### #

# Ubuntu 18.04
FROM ubuntu@sha256:538529c9d229fb55f50e6746b119e899775205d62c0fc1b7e679b30d02ecb6e8 AS builder-install

# Party like it's June 8th, 1989.
#
SHELL ["/bin/bash", "-c"]

# Certain Installers make 'installations' easier by having a nice front-end. While this is great when you have a manual install, this becomes an issue during automated installations.
ENV DEBIAN_FRONTEND=noninteractive
# Specify the rust-toolchain for init script.
COPY rust-toolchain /tmp
ENV RUST_TOOLCHAIN_PATH=/tmp/rust-toolchain

# Install build tools and dependencies
#
RUN apt-get update -q -q && \
apt-get upgrade --yes --force-yes && \
apt-get install --yes --force-yes \
alien \
autoconf \
automake \
binutils-dev \
build-essential \
clang \
cmake \
curl \
git \
jq \
libclang-dev \
libcurl4-openssl-dev \
libdw-dev \
libiberty-dev \
libpq-dev \
libprotobuf-c-dev \
libprotobuf-dev \
libssl-dev \
libssl1.1 \
libsystemd-dev \
libtool \
libxml2-dev \
llvm-dev \
nano \
nginx \
ninja-build \
ocaml-native-compilers \
ocamlbuild \
patch \
pkg-config \
postgresql-10 \
prometheus \
protobuf-c-compiler \
protobuf-compiler \
psmisc \
python \
python3-pip \
systemd \
unzip \
uuid-dev \
wget \
zlib1g-dev
# psmisc = killall
# prometheus = helps with running slam scripts locally
COPY init_debian.sh /tmp
RUN bash /tmp/init_debian.sh

# Install go 1.16 release
RUN wget https://golang.org/dl/go1.16.4.linux-amd64.tar.gz && \
mv go1.16.4.linux-amd64.tar.gz go.tgz && \
tar -C /usr/local -xzf go.tgz && \
rm -rf go.tgz
ENV PATH="/usr/local/go/bin:$PATH"
# Unfortunately we cannot update ENV from the init script.
# See https://github.com/docker/docker/issues/29110
ENV GOPATH=/opt/go/
ENV PATH=$PATH:$GOPATH/bin

# Install SQLite latest release.
ENV SQLITE=sqlite-autoconf-3350400
RUN wget https://www.sqlite.org/2021/$SQLITE.tar.gz && \
tar xf $SQLITE.tar.gz && \
mkdir bld && \
pushd bld && \
../$SQLITE/configure && \
make && \
make install && \
popd && \
rm -r bld $SQLITE.tar.gz

# For use in CI
RUN pip3 install awscli black

# filebeat is used for logs when running slam scripts locally
RUN curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.2-amd64.deb && \
dpkg -i filebeat-7.4.2-amd64.deb && \
rm filebeat-7.4.2-amd64.deb

# set rust nightly
#
RUN mkdir -p /tmp/rustup
WORKDIR /tmp/rustup
COPY --chown=root:root rust-toolchain .
RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init && \
chmod +x rustup-init && \
./rustup-init --default-toolchain $(cat rust-toolchain) -y && \
/root/.cargo/bin/rustup component add \
clippy \
llvm-tools-preview \
rust-analysis \
rust-src \
rustfmt && \
/root/.cargo/bin/cargo install sccache cargo-cache cargo2junit cargo-tree cargo-feature-analyst cbindgen && \
/root/.cargo/bin/cargo install diesel_cli --no-default-features --features postgres

ENV PATH="/usr/local/go/bin:$GOPATH/bin:$PATH"
# Set PATH in the container environment to include cargo bin
ENV PATH="/root/.cargo/bin:$PATH"

# Install kcov. So that we don't have to do this again with every build in ci.
RUN mkdir -p /tmp/kcov
WORKDIR /tmp/kcov
RUN wget https://github.com/SimonKagstrom/kcov/archive/v36.tar.gz && \
tar xvf v36.tar.gz && \
cd kcov-36 && \
cmake . && \
make && \
make install

# Conditionally install intel SGX libs

ARG DOCKER_NO_SGX=false

COPY --chown=root:root install_sgx.sh /tmp
Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1_24
1_25
126 changes: 126 additions & 0 deletions docker/init_debian.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#!/usr/bin/env bash

# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Note: When modifying this file, increment the Dockerfile-version minor version number
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#
# This is needed for mob tool to be able to pull the right image from the farm,
# if it isn't done, bad things will happen to local builds and CI using mob tool
# See mob tool comments for extended discussion

set -e # Die on any errors

cd /tmp

# Certain Installers make 'installations' easier by having a nice front-end. While this is great when you have a manual install, this becomes an issue during automated installations.
export DEBIAN_FRONTEND=noninteractive

# Install build tools and dependencies
apt-get update -q -q
apt-get upgrade --yes
apt-get install --yes \
alien \
apt-transport-https \
autoconf \
automake \
binutils-dev \
build-essential \
clang \
cmake \
curl \
git \
jq \
libclang-dev \
libcurl4-openssl-dev \
libdw-dev \
libiberty-dev \
libpq-dev \
libprotobuf-c-dev \
libprotobuf-dev \
libssl-dev \
libssl1.1 \
libsystemd-dev \
libtool \
libxml2-dev \
llvm-dev \
nano \
nginx \
ninja-build \
ocaml-native-compilers \
ocamlbuild \
patch \
pkg-config \
postgresql-10 \
prometheus \
protobuf-c-compiler \
protobuf-compiler \
psmisc \
python \
python3-pip \
sqlite3 \
systemd \
unzip \
uuid-dev \
wget \
zlib1g-dev
# psmisc = killall
# prometheus = helps with running slam scripts locally

# For use in CI
# TODO: Do we need this outside CI?
pip3 install awscli black

# filebeat is used for logs when running slam scripts locally
# via https://www.elastic.co/guide/en/beats/filebeat/current/setup-repositories.html
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" >> /etc/apt/sources.list.d/elastic-7.x.list
apt-get update && apt-get install filebeat
systemctl enable filebeat

# Install go 1.16 release
GO_PKG=go1.16.4.linux-amd64.tar.gz
wget https://golang.org/dl/$GO_PKG -O go.tgz
tar -C /usr/local -xzf go.tgz
rm -rf go.tgz

# Install SQLite release.
SQLITE=sqlite-autoconf-3350400
SQLITE_PKG=$SQLITE.tar.gz
wget https://www.sqlite.org/2021/$SQLITE_PKG
tar xf $SQLITE_PKG
pushd $SQLITE
./configure
make install
popd
rm -r $SQLITE*

# set rust toolchain, defaulting to nightly
RUST_TOOLCHAIN=${RUST_TOOLCHAIN:-nightly}
if [ -f "$RUST_TOOLCHAIN_PATH" ]; then
RUST_TOOLCHAIN=`cat "$RUST_TOOLCHAIN_PATH"`
fi

# Fetch rustup, and tell it to install $RUST_TOOLCHAIN
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain $RUST_TOOLCHAIN -y
source $HOME/.cargo/env
rustup component add \
clippy \
llvm-tools-preview \
rust-analysis \
rust-src \
rustfmt
cargo install sccache cargo-cache cargo2junit cargo-tree cargo-feature-analyst cbindgen && \
cargo install diesel_cli --no-default-features --features postgres

# Install kcov. So that we don't have to do this again with every build in ci.
# TODO: Replace with `apt-get install kcov` when we upgrade builder image to
# Ubuntu 20
mkdir -p /tmp/kcov
cd /tmp/kcov
wget https://github.com/SimonKagstrom/kcov/archive/v36.tar.gz
tar xvf v36.tar.gz
cd kcov-36
cmake .
make install

echo "Successfully installed packages."

0 comments on commit 9b6a10b

Please sign in to comment.