diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfa02cc85 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +/*.*/**/Dockerfile* linguist-generated +/*.*/**/docker-entrypoint.sh linguist-generated +Dockerfile* linguist-language=Dockerfile diff --git a/.github/workflows/verify-templating.yml b/.github/workflows/verify-templating.yml new file mode 100644 index 000000000..7e833f1c7 --- /dev/null +++ b/.github/workflows/verify-templating.yml @@ -0,0 +1,22 @@ +name: Verify Templating + +on: + pull_request: + push: + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + apply-templates: + name: Check For Uncomitted Changes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Apply Templates + run: ./apply-templates.sh + - name: Check Git Status + run: | + status="$(git status --short)" + [ -z "$status" ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..d548f66de --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.jq-template.awk diff --git a/5.6/Dockerfile b/5.6/Dockerfile.debian similarity index 90% rename from 5.6/Dockerfile rename to 5.6/Dockerfile.debian index bfdf1b19b..883438148 100644 --- a/5.6/Dockerfile +++ b/5.6/Dockerfile.debian @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:stretch-slim # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added @@ -56,7 +62,7 @@ RUN set -ex; \ ENV MYSQL_MAJOR 5.6 ENV MYSQL_VERSION 5.6.50-1debian9 -RUN echo "deb http://repo.mysql.com/apt/debian/ stretch mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list +RUN echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' > /etc/apt/sources.list.d/mysql.list # the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) # also, we set debconf keys to make APT a little quieter @@ -66,17 +72,20 @@ RUN { \ echo mysql-community-server mysql-community-server/re-root-pass password ''; \ echo mysql-community-server mysql-community-server/remove-test-db select false; \ } | debconf-set-selections \ - && apt-get update && apt-get install -y mysql-server="${MYSQL_VERSION}" && rm -rf /var/lib/apt/lists/* \ - && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \ - && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \ -# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime - && chmod 1777 /var/run/mysqld /var/lib/mysql \ + && apt-get update \ + && apt-get install -y \ + mysql-server="${MYSQL_VERSION}" \ # comment out a few problematic configuration values && find /etc/mysql/ -name '*.cnf' -print0 \ | xargs -0 grep -lZE '^(bind-address|log)' \ | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/' \ # don't reverse lookup hostnames, they are usually another container - && echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf + && echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \ + && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \ +# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + && chmod 1777 /var/run/mysqld /var/lib/mysql VOLUME /var/lib/mysql diff --git a/5.7/Dockerfile b/5.7/Dockerfile.debian similarity index 90% rename from 5.7/Dockerfile rename to 5.7/Dockerfile.debian index ec29229a8..c4261bced 100644 --- a/5.7/Dockerfile +++ b/5.7/Dockerfile.debian @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:buster-slim # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added @@ -58,7 +64,7 @@ RUN set -ex; \ ENV MYSQL_MAJOR 5.7 ENV MYSQL_VERSION 5.7.32-1debian10 -RUN echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list +RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list # the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) # also, we set debconf keys to make APT a little quieter @@ -68,17 +74,20 @@ RUN { \ echo mysql-community-server mysql-community-server/re-root-pass password ''; \ echo mysql-community-server mysql-community-server/remove-test-db select false; \ } | debconf-set-selections \ - && apt-get update && apt-get install -y mysql-server="${MYSQL_VERSION}" && rm -rf /var/lib/apt/lists/* \ - && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \ - && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \ -# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime - && chmod 1777 /var/run/mysqld /var/lib/mysql \ + && apt-get update \ + && apt-get install -y \ + mysql-server="${MYSQL_VERSION}" \ # comment out a few problematic configuration values && find /etc/mysql/ -name '*.cnf' -print0 \ | xargs -0 grep -lZE '^(bind-address|log)' \ | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/' \ # don't reverse lookup hostnames, they are usually another container - && echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf + && echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \ + && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \ +# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + && chmod 1777 /var/run/mysqld /var/lib/mysql VOLUME /var/lib/mysql diff --git a/8.0/Dockerfile b/8.0/Dockerfile.debian similarity index 90% rename from 8.0/Dockerfile rename to 8.0/Dockerfile.debian index 16f5d9dc7..d3ba3cacf 100644 --- a/8.0/Dockerfile +++ b/8.0/Dockerfile.debian @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM debian:buster-slim # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added @@ -58,7 +64,7 @@ RUN set -ex; \ ENV MYSQL_MAJOR 8.0 ENV MYSQL_VERSION 8.0.22-1debian10 -RUN echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list +RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sources.list.d/mysql.list # the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) # also, we set debconf keys to make APT a little quieter @@ -68,13 +74,18 @@ RUN { \ echo mysql-community-server mysql-community-server/re-root-pass password ''; \ echo mysql-community-server mysql-community-server/remove-test-db select false; \ } | debconf-set-selections \ - && apt-get update && apt-get install -y mysql-community-client="${MYSQL_VERSION}" mysql-community-server-core="${MYSQL_VERSION}" && rm -rf /var/lib/apt/lists/* \ + && apt-get update \ + && apt-get install -y \ + mysql-community-client="${MYSQL_VERSION}" \ + mysql-community-server-core="${MYSQL_VERSION}" \ + && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \ && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \ # ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime && chmod 1777 /var/run/mysqld /var/lib/mysql VOLUME /var/lib/mysql + # Config files COPY config/ /etc/mysql/ COPY docker-entrypoint.sh /usr/local/bin/ diff --git a/apply-templates.sh b/apply-templates.sh new file mode 100755 index 000000000..f80f7670b --- /dev/null +++ b/apply-templates.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +[ -f versions.json ] # run "versions.sh" first + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +jqt='.jq-template.awk' +if [ -n "${BASHBREW_SCRIPTS:-}" ]; then + jqt="$BASHBREW_SCRIPTS/jq-template.awk" +elif [ "$BASH_SOURCE" -nt "$jqt" ]; then + wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/5f0c26381fb7cc78b2d217d58007800bdcfbcfa1/scripts/jq-template.awk' +fi + +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi + +generated_warning() { + cat <<-EOH + # + # NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" + # + # PLEASE DO NOT EDIT IT DIRECTLY. + # + + EOH +} + +for version; do + export version + + debianVersion="$(jq -r '.[env.version].debian // ""' versions.json)" + if [ -n "$debianVersion" ]; then + dockerfile='Dockerfile.debian' + + { + generated_warning + gawk -f "$jqt" "template/$dockerfile" + } > "$version/$dockerfile" + fi + + cp -a template/docker-entrypoint.sh "$version/docker-entrypoint.sh" +done diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index a4a7f55d6..a6940e74a 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -Eeuo pipefail declare -A aliases=( [5.7]='5' @@ -9,31 +9,35 @@ declare -A aliases=( self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -versions=( */ ) -versions=( "${versions[@]%/}" ) +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi # sort version numbers with highest first -IFS=$'\n'; versions=( $(echo "${versions[*]}" | sort -rV) ); unset IFS +IFS=$'\n'; set -- $(sort -rV <<<"$*"); unset IFS # get the most recent commit which modified any of "$@" fileCommit() { git log -1 --format='format:%H' HEAD -- "$@" } -# get the most recent commit which modified "$1/Dockerfile" or any file COPY'd from "$1/Dockerfile" +# get the most recent commit which modified "dir/dockerfile" or any file COPY'd from it dirCommit() { local dir="$1"; shift + local df="$1"; shift ( cd "$dir" - fileCommit \ - Dockerfile \ - $(git show HEAD:./Dockerfile | awk ' + local files; files="$( + git show "HEAD:./$df" | awk ' toupper($1) == "COPY" { for (i = 2; i < NF; i++) { print $i } } - ') + ' + )" + fileCommit "$df" $files ) } @@ -52,10 +56,12 @@ join() { echo "${out#$sep}" } -for version in "${versions[@]}"; do - commit="$(dirCommit "$version")" +for version; do + export version + df='Dockerfile.debian' + commit="$(dirCommit "$version" "$df")" - fullVersion="$(git show "$commit":"$version/Dockerfile" | awk '$1 == "ENV" && $2 == "MYSQL_VERSION" { gsub(/-[0-9].*$/, "", $3); print $3; exit }')" + fullVersion="$(jq -r '.[env.version].version' versions.json)" versionAliases=() while [ "$fullVersion" != "$version" -a "${fullVersion%[.-]*}" != "$fullVersion" ]; do @@ -72,5 +78,6 @@ for version in "${versions[@]}"; do Tags: $(join ', ' "${versionAliases[@]}") GitCommit: $commit Directory: $version + File: $df EOE done diff --git a/template/Dockerfile.debian b/template/Dockerfile.debian new file mode 100644 index 000000000..092cc57e0 --- /dev/null +++ b/template/Dockerfile.debian @@ -0,0 +1,110 @@ +FROM debian:{{ .debian.suite }}-slim + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN groupadd -r mysql && useradd -r -g mysql mysql + +RUN apt-get update && apt-get install -y --no-install-recommends gnupg dirmngr && rm -rf /var/lib/apt/lists/* + +# add gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.12 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN mkdir /docker-entrypoint-initdb.d + +RUN apt-get update && apt-get install -y --no-install-recommends \ +# for MYSQL_RANDOM_ROOT_PASSWORD + pwgen \ +{{ + if env.version != "5.6" then ( +-}} +# for mysql_ssl_rsa_setup + openssl \ +{{ ) else "" end -}} +# FATAL ERROR: please install the following Perl modules before executing /usr/local/mysql/scripts/mysql_install_db: +# File::Basename +# File::Copy +# Sys::Hostname +# Data::Dumper + perl \ +# install "xz-utils" for .sql.xz docker-entrypoint-initdb.d files + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN set -ex; \ +# gpg: key 5072E1F5: public key "MySQL Release Engineering " imported + key='A4A9406876FCBD3C456770C88C718D3B5072E1F5'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \ + gpg --batch --export "$key" > /etc/apt/trusted.gpg.d/mysql.gpg; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + apt-key list > /dev/null + +ENV MYSQL_MAJOR {{ env.version }} +ENV MYSQL_VERSION {{ .debian.version }} + +RUN echo 'deb http://repo.mysql.com/apt/debian/ {{ .debian.suite }} mysql-{{ env.version }}' > /etc/apt/sources.list.d/mysql.list + +# the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) +# also, we set debconf keys to make APT a little quieter +RUN { \ + echo mysql-community-server mysql-community-server/data-dir select ''; \ + echo mysql-community-server mysql-community-server/root-pass password ''; \ + echo mysql-community-server mysql-community-server/re-root-pass password ''; \ + echo mysql-community-server mysql-community-server/remove-test-db select false; \ + } | debconf-set-selections \ + && apt-get update \ + && apt-get install -y \ +{{ if env.version == "5.6" or env.version == "5.7" then ( -}} + mysql-server="${MYSQL_VERSION}" \ +# comment out a few problematic configuration values + && find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/' \ +# don't reverse lookup hostnames, they are usually another container + && echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf \ +{{ ) else ( -}} + mysql-community-client="${MYSQL_VERSION}" \ + mysql-community-server-core="${MYSQL_VERSION}" \ +{{ ) end -}} + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \ + && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \ +# ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + && chmod 1777 /var/run/mysqld /var/lib/mysql + +VOLUME /var/lib/mysql + +{{ if env.version != "5.6" and env.version != "5.7" then ( -}} +# Config files +COPY config/ /etc/mysql/ +{{ ) else "" end -}} +COPY docker-entrypoint.sh /usr/local/bin/ +RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat +ENTRYPOINT ["docker-entrypoint.sh"] + +{{ if env.version != "5.6" then ( -}} +EXPOSE 3306 33060 +{{ ) else ( -}} +EXPOSE 3306 +{{ ) end -}} +CMD ["mysqld"] diff --git a/.template.Debian/docker-entrypoint.sh b/template/docker-entrypoint.sh similarity index 100% rename from .template.Debian/docker-entrypoint.sh rename to template/docker-entrypoint.sh diff --git a/update.sh b/update.sh index 75a2ecdf2..bac2d7581 100755 --- a/update.sh +++ b/update.sh @@ -1,46 +1,7 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash +set -Eeuo pipefail cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -versions=( "$@" ) -if [ ${#versions[@]} -eq 0 ]; then - versions=( */ ) -fi -versions=( "${versions[@]%/}" ) - -defaultDebianVariant='buster-slim' -declare -A debianVariants=( - [5.6]='stretch-slim' -) - -for version in "${versions[@]}"; do - debianVariant="${debianVariants[$version]:-$defaultDebianVariant}" - debianSuite="${debianVariant%%-*}" # "buster", etc - - cp -a .template.Debian/docker-entrypoint.sh "$version/docker-entrypoint.sh" - - fullVersion="$( - curl -fsSL "https://repo.mysql.com/apt/debian/dists/$debianSuite/mysql-$version/binary-amd64/Packages.gz" \ - | gunzip \ - | awk -F ': ' ' - $1 == "Package" { - pkg = $2 - next - } - pkg == "mysql-server" && $1 == "Version" { - print $2 - } - ' - )" - - ( - set -x - sed -ri \ - -e 's/^(ENV MYSQL_VERSION) .*/\1 '"$fullVersion"'/' \ - -e 's/^(ENV MYSQL_MAJOR) .*/\1 '"$version"'/' \ - -e 's/^(FROM) .*/\1 debian:'"$debianVariant"'/' \ - -e 's!(http://repo.mysql.com/apt/debian/) [^ ]+!\1 '"$debianSuite"'!' \ - "$version/Dockerfile" - ) -done +./versions.sh "$@" +./apply-templates.sh "$@" diff --git a/versions.json b/versions.json new file mode 100644 index 000000000..cceb6192c --- /dev/null +++ b/versions.json @@ -0,0 +1,32 @@ +{ + "5.6": { + "debian": { + "architectures": [ + "amd64" + ], + "suite": "stretch", + "version": "5.6.50-1debian9" + }, + "version": "5.6.50" + }, + "5.7": { + "debian": { + "architectures": [ + "amd64" + ], + "suite": "buster", + "version": "5.7.32-1debian10" + }, + "version": "5.7.32" + }, + "8.0": { + "debian": { + "architectures": [ + "amd64" + ], + "suite": "buster", + "version": "8.0.22-1debian10" + }, + "version": "8.0.22" + } +} diff --git a/versions.sh b/versions.sh new file mode 100755 index 000000000..e475d0b78 --- /dev/null +++ b/versions.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +defaultDebianSuite='buster' +declare -A debianSuites=( + [5.6]='stretch' +) + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +versions=( "$@" ) +if [ ${#versions[@]} -eq 0 ]; then + versions=( *.*/ ) + json='{}' +else + json="$(< versions.json)" +fi +versions=( "${versions[@]%/}" ) + +for version in "${versions[@]}"; do + export version + + debianSuite="${debianSuites[$version]:-$defaultDebianSuite}" + debianVersion="$( + curl -fsSL "https://repo.mysql.com/apt/debian/dists/$debianSuite/mysql-$version/binary-amd64/Packages.gz" \ + | gunzip \ + | awk -F ': ' ' + $1 == "Package" { + pkg = $2 + next + } + pkg == "mysql-server" && $1 == "Version" { + print $2 + } + ' + )" + + # example 8.0.22-1debian10 => 8.0.22 + baseVersion="${debianVersion%-*}" + + export baseVersion debianSuite debianVersion + json="$( + jq <<<"$json" -c \ + '.[env.version] = { + version: env.baseVersion, + debian: { + architectures: [ "amd64" ], + suite: env.debianSuite, + version: env.debianVersion, + }, + }' + )" +done + +jq <<<"$json" -S . > versions.json