Skip to content

Commit

Permalink
Reduce pulsar-test-latest-version docker image size (apache#9627)
Browse files Browse the repository at this point in the history
- smaller image size makes it feasible to build the image once and share
  it between the different parallel integration test jobs. This will
  help improve the Pulsar CI in the future.

Solution:
- build go-lang examples in a separate temporary layer
- cleanup files as last steps
  - requires "-Ddockerfile.build.squash=true" (uses "docker build --squash")
    to reduce the final image size. This is not part of this change
    since the image size isn't important until images are transferred over
    the network. Squashing the image would just add overhead to the current build.
  • Loading branch information
lhotari authored Feb 19, 2021
1 parent 41e118a commit 3ad9d32
Showing 1 changed file with 51 additions and 25 deletions.
76 changes: 51 additions & 25 deletions tests/docker-images/latest-version-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,36 @@
# under the License.
#

# build go lang examples first in a separate layer

FROM apachepulsar/pulsar-all:latest as pulsar-function-go

USER root

RUN rm -rf /var/lib/apt/lists/* && apt-get update
RUN apt-get install -y procps curl git

ENV GOLANG_VERSION 1.13.3

RUN curl -sSL https://storage.googleapis.com/golang/go$GOLANG_VERSION.linux-amd64.tar.gz \
| tar -v -C /usr/local -xz

# RUN wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz && tar -xvf go1.13.3.linux-amd64.tar.gz && mv go /usr/local
# RUN export GOROOT=/usr/local/go && export GOPATH=$HOME/go && export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
# RUN echo "export GOROOT=/usr/local/go" >> ~/.profile && echo "export GOPATH=$HOME/go" >> ~/.profile && echo "export PATH=$GOPATH/bin:$GOROOT/bin:$PATH" >> ~/.profile

ENV PATH /usr/local/go/bin:$PATH

RUN mkdir -p /go/src /go/bin && chmod -R 777 /go
ENV GOROOT /usr/local/go
ENV GOPATH /go
ENV PATH /go/bin:$PATH

COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./...
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./...

FROM apachepulsar/pulsar-all:latest

# Switch to run as the root user to simplify building container and then running
Expand All @@ -27,7 +57,7 @@ USER root

RUN rm -rf /var/lib/apt/lists/* && apt update

RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl git
RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl

RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ && mkdir -p /pulsar/ssl

Expand All @@ -48,36 +78,32 @@ COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \
/pulsar/bin/

# copy python test examples

RUN mkdir -p /pulsar/instances/deps

ENV GOLANG_VERSION 1.13.3

RUN curl -sSL https://storage.googleapis.com/golang/go$GOLANG_VERSION.linux-amd64.tar.gz \
| tar -v -C /usr/local -xz

# RUN wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz && tar -xvf go1.13.3.linux-amd64.tar.gz && mv go /usr/local
# RUN export GOROOT=/usr/local/go && export GOPATH=$HOME/go && export PATH=$GOPATH/bin:$GOROOT/bin:$PATH
# RUN echo "export GOROOT=/usr/local/go" >> ~/.profile && echo "export GOPATH=$HOME/go" >> ~/.profile && echo "export PATH=$GOPATH/bin:$GOROOT/bin:$PATH" >> ~/.profile

ENV PATH /usr/local/go/bin:$PATH

RUN mkdir -p /go/src /go/bin && chmod -R 777 /go
ENV GOROOT /usr/local/go
ENV GOPATH /go
ENV PATH /go/bin:$PATH

COPY python-examples/exclamation_lib.py /pulsar/instances/deps/
COPY python-examples/exclamation_with_extra_deps.py /pulsar/examples/python-examples/
COPY python-examples/exclamation.zip /pulsar/examples/python-examples/
COPY python-examples/producer_schema.py /pulsar/examples/python-examples/
COPY python-examples/consumer_schema.py /pulsar/examples/python-examples/
COPY python-examples/exception_function.py /pulsar/examples/python-examples/

COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./...
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install
RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./...

# copy java test examples
COPY target/java-test-functions.jar /pulsar/examples/
RUN cp -a /go/bin/. /pulsar/examples/go-examples/

# copy go test examples
COPY --from=pulsar-function-go /go/bin /pulsar/examples/go-examples

# change ownership of files
RUN chown -R pulsar:0 /pulsar && chmod -R g=u /pulsar

# remove static libpulsar*.a libraries to reduce image size
RUN rm /usr/lib/libpulsar*.a
# remove gcc
RUN apt-get -y remove --autoremove --purge build-essential

# cleanup
RUN apt-get -y autoremove \
&& apt-get clean \
&& apt-get autoclean \
&& rm -rf /var/lib/apt/lists/*

CMD bash

0 comments on commit 3ad9d32

Please sign in to comment.