Skip to content

Commit

Permalink
GEODE-5864 - Refactor and parameterize CI env creation
Browse files Browse the repository at this point in the history
Makes it easier for developers to deploy their own CI instances for
testing of code and CI itself.

* Use concourse-pipeline-resource to deploy instead of hand-rolling
  `fly` scripts
* Add pararmeters for fork/branch/repo name
* Rename all pipelines to be consistent
* Rename all pipelines to contain fork/branch information
* Reorganize artifact storage to include fork/branch information
* `deploy_meta.sh` auto-starts Concourse tasks in required bootstrap order
* Instance images names are created consistently and are unique
* Require fewer VAULT secrets
* Improve behavior of packer on windows GCP instances
* Check to see if a previous iteration of start_instance created an
  instance and kill it before trying to create a new one.
* Ensure we stop instances that cannot be reached via SSH
* Add ssh server keep alives to prevent firewall timeouts

Co-authored-by: Sean Goller <[email protected]>
Co-authored-by: Patrick Rhomberg <[email protected]>
Co-authored-by: Robert Houghton <[email protected]>
Co-authored-by: Jacob Barrett <[email protected]>
  • Loading branch information
4 people committed Oct 15, 2018
1 parent 4e8ea1c commit afdc9cb
Show file tree
Hide file tree
Showing 31 changed files with 494 additions and 381 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -241,3 +241,4 @@ The following provides more details on the included cryptographic software:
The [JCE Unlimited Strength Jurisdiction Policy](https://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
may need to be installed separately to use keystore passwords with 7 or more characters.
* Apache Geode links to and uses [OpenSSL](https://www.openssl.org/) ciphers.

3 changes: 2 additions & 1 deletion ci/images/alpine-tools/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM alpine:3.7
FROM alpine:latest

COPY --from=google/cloud-sdk:alpine /google-cloud-sdk /google-cloud-sdk
COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer
Expand All @@ -31,6 +31,7 @@ RUN apk --no-cache add \
python \
py2-pip \
rsync \
util-linux \
&& gcloud config set core/disable_usage_reporting true \
&& gcloud config set component_manager/disable_update_check true \
&& gcloud config set metrics/environment github_docker_image \
Expand Down
42 changes: 25 additions & 17 deletions ci/images/google-geode-builder/build_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,30 +24,38 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

pushd ${SCRIPTDIR}

. ${SCRIPTDIR}/../../pipelines/shared/utilities.sh

pushd ${SCRIPTDIR}

IMAGE_FAMILY_PREFIX=""
GEODE_DOCKER_IMAGE=${GEODE_DOCKER_IMAGE:-"gcr.io/apachegeode-ci/test-container"}
if [[ -z "${GEODE_FORK}" ]]; then
echo "GEODE_FORK environment variable must be set for this script to work."
exit 1
if [[ -n "${CONCOURSE_GCP_KEY}" ]]; then
dd of=credentials.json <<< "${CONCOURSE_GCP_KEY}"
export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPTDIR}/credentials.json
fi

GEODE_BRANCH=${GEODE_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}

SANITIZED_GEODE_BRANCH=$(getSanitizedBranch ${GEODE_BRANCH})
SANITIZED_GEODE_FORK=$(getSanitizedFork ${GEODE_FORK})

if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
GCP_NETWORK="default"
GCP_SUBNETWORK="default"

MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
if [[ -n "${MY_NAME}" ]]; then
MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
MY_ZONE=${MY_ZONE##*/}
NETWORK_INTERFACE_INFO="$(gcloud compute instances describe ${MY_NAME} --zone ${MY_ZONE} --format="json(networkInterfaces)")"
GCP_NETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].network')
GCP_NETWORK=${GCP_NETWORK##*/}
GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
fi

HASHED_PIPELINE_PREFIX="i$(uuidgen -n @dns -s -N "${PIPELINE_PREFIX}")-"

echo "Running packer"
packer build \
PACKER_LOG=1 packer build \
--var "geode_docker_image=${GEODE_DOCKER_IMAGE}" \
--var "image_family_prefix=${IMAGE_FAMILY_PREFIX}" \
--var "pipeline_prefix=${PIPELINE_PREFIX}" \
--var "hashed_pipeline_prefix=${HASHED_PIPELINE_PREFIX}" \
--var "gcp_project=${GCP_PROJECT}" \
--var "java_build_version=${JAVA_BUILD_VERSION}" \
--var "gcp_network=${GCP_NETWORK}" \
--var "gcp_subnetwork=${GCP_SUBNETWORK}" \
--var "use_internal_ip=true" \
packer.json
13 changes: 8 additions & 5 deletions ci/images/google-geode-builder/packer.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"variables": {
"image_family_prefix": "local-testing-",
"geode_docker_image": "gcr.io/apachegeode-ci/test-container",
"pipeline_prefix": "local-testing-",
"hashed_pipeline_prefix": "0b7ad56c-671b-5244-9f36-01f92928abf8",
"geode_docker_image": "unset",
"gcp_project": "unset",
"java_build_version": "8"
},
"provisioners": [
Expand Down Expand Up @@ -29,12 +31,13 @@
"builders": [
{
"type": "googlecompute",
"project_id": "apachegeode-ci",
"project_id": "{{user `gcp_project`}}",
"source_image_family": "debian-9",
"ssh_username": "packer",
"zone": "us-central1-f",
"image_family": "{{user `image_family_prefix`}}geode-builder",
"image_name": "{{user `image_family_prefix`}}geode-builder-{{timestamp}}"
"image_family": "{{user `pipeline_prefix`}}geode-builder",
"image_name": "{{user `hashed_pipeline_prefix`}}gb-{{timestamp}}",
"tags": ["packer"]
}
]
}
37 changes: 26 additions & 11 deletions ci/images/google-windows-geode-builder/build_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x

SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
Expand All @@ -26,21 +27,35 @@ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

pushd ${SCRIPTDIR}

IMAGE_FAMILY_PREFIX=""
if [[ -z "${GEODE_FORK}" ]]; then
echo "GEODE_FORK environment variable must be set for this script to work."
exit 1
if [[ -n "${CONCOURSE_GCP_KEY}" ]]; then
dd of=credentials.json <<< "${CONCOURSE_GCP_KEY}"
export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPTDIR}/credentials.json
fi

GEODE_BRANCH=${GEODE_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}
SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]' '[:lower:]' | cut -c 1-20)
SANITIZED_GEODE_FORK=$(echo ${GEODE_FORK} | tr "/" "-" | tr '[:upper:]' '[:lower:]' | cut -c 1-16)
GCP_NETWORK="default"
GCP_SUBNETWORK="default"

if [[ "${SANITIZED_GEODE_FORK}" != "apache" ]]; then
IMAGE_FAMILY_PREFIX="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-"
MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
if [[ -n "${MY_NAME}" ]]; then
MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
MY_ZONE=${MY_ZONE##*/}
NETWORK_INTERFACE_INFO="$(gcloud compute instances describe ${MY_NAME} --zone ${MY_ZONE} --format="json(networkInterfaces)")"
GCP_NETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].network')
GCP_NETWORK=${GCP_NETWORK##*/}
GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
fi

HASHED_PIPELINE_PREFIX="i$(uuidgen -n @dns -s -N "${PIPELINE_PREFIX}")-"

echo "Running packer"
packer build \
--var "image_family_prefix=${IMAGE_FAMILY_PREFIX}" \
PACKER_LOG=1 packer build \
--var "geode_docker_image=${GEODE_DOCKER_IMAGE}" \
--var "pipeline_prefix=${PIPELINE_PREFIX}" \
--var "hashed_pipeline_prefix=${HASHED_PIPELINE_PREFIX}" \
--var "java_build_version=${JAVA_BUILD_VERSION}" \
--var "gcp_project=${GCP_PROJECT}" \
--var "gcp_network=${GCP_NETWORK}" \
--var "gcp_subnetwork=${GCP_SUBNETWORK}" \
--var "use_internal_ip=true" \
windows-packer.json
24 changes: 17 additions & 7 deletions ci/images/google-windows-geode-builder/windows-packer.json
Original file line number Diff line number Diff line change
@@ -1,27 +1,37 @@
{
"variables": {
"image_family_prefix": "local-testing-",
"geode_docker_image": "gcr.io/apachegeode-ci/test-container"
"pipeline_prefix": "local-testing-",
"hashed_pipeline_prefix": "0b7ad56c-671b-5244-9f36-01f92928abf8",
"geode_docker_image": "unset",
"gcp_project": "unset",
"gcp_network": "default",
"gcp_subnetwork": "default",
"use_internal_ip": "false",
"java_build_version": "8"
},
"builders": [
{
"image_name": "{{user `image_family_prefix`}}windows-geode-builder-{{timestamp}}",
"image_family": "{{user `image_family_prefix`}}windows-geode-builder",
"image_name": "{{user `hashed_pipeline_prefix`}}wgb-{{timestamp}}",
"image_family": "{{user `pipeline_prefix`}}windows-geode-builder",
"type": "googlecompute",
"project_id": "apachegeode-ci",
"source_image": "windows-server-1709-dc-core-for-containers-v20180814",
"project_id": "{{user `gcp_project`}}",
"network": "{{user `gcp_network`}}",
"subnetwork": "{{user `gcp_subnetwork`}}",
"source_image_family": "windows-1709-core-for-containers",
"disk_size": "100",
"machine_type": "n1-standard-1",
"communicator": "winrm",
"winrm_username": "geode",
"winrm_insecure": true,
"winrm_use_ssl": true,
"state_timeout": "10m",
"metadata": {
"windows-startup-script-cmd": "winrm quickconfig -quiet & net user /add geode & net localgroup administrators geode /add & winrm set winrm/config/service/auth @{Basic=\"true\"}"
},
"zone": "us-central1-a",
"tags": ["packer"],
"omit_external_ip": false,
"use_internal_ip": false
"use_internal_ip": "{{user `use_internal_ip`}}"
}
],
"provisioners": [
Expand Down
8 changes: 4 additions & 4 deletions ci/images/meta-mini/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM alpine:3.7
FROM alpine:latest

COPY --from=google/cloud-sdk:alpine /google-cloud-sdk /google-cloud-sdk
ENV PATH /google-cloud-sdk/bin:$PATH
RUN apk --no-cache add \
bash \
curl \
jq \
python3 \
&& curl -L -o /usr/local/bin/fly "https://concourse.apachegeode-ci.info/api/v1/cli?arch=amd64&platform=linux" \
&& pip3 install --upgrade pip \
&& pip3 install Jinja2 pyYAML \
&& chmod +x /usr/local/bin/* \
&& pip3 install Jinja2 pyYAML
4 changes: 2 additions & 2 deletions ci/pipelines/examples/jinja.template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ resources:
source:
username: ((!docker-username))
password: ((!docker-password))
repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
repository: gcr.io/((gcp-project))/((!docker-image-prefix))((!docker-image-name))
tag: latest
- name: 24h
type: time
Expand Down Expand Up @@ -78,7 +78,7 @@ jobs:
params:
MAINTENANCE_VERSION: {{repository.branch}}
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
ARTIFACT_BUCKET: ((artifact-bucket))
run:
args:
path: geode-ci/ci/scripts/build-examples.sh
78 changes: 31 additions & 47 deletions ci/pipelines/geode-build/deploy_pipeline.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,9 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
GEODEBUILDDIR="${SCRIPTDIR}/../geode-build"

for cmd in Jinja2 PyYAML; do
if ! [[ $(pip3 list |grep ${cmd}) ]]; then
echo "${cmd} must be installed for pipeline deployment to work."
echo " 'pip3 install ${cmd}'"
echo ""
exit 1
fi
done

set -ex
set -e

if [ -z "${GEODE_BRANCH}" ]; then
GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
Expand All @@ -44,47 +36,39 @@ if [ "${GEODE_BRANCH}" = "HEAD" ]; then
exit 1
fi

. ${SCRIPTDIR}/../shared/utilities.sh

SANITIZED_GEODE_BRANCH=$(getSanitizedBranch ${GEODE_BRANCH})
SANITIZED_GEODE_FORK=$(getSanitizedFork ${GEODE_FORK})

BIN_DIR=${OUTPUT_DIRECTORY}/bin
TMP_DIR=${OUTPUT_DIRECTORY}/tmp
mkdir -p ${BIN_DIR} ${TMP_DIR}
curl -o ${BIN_DIR}/fly "https://concourse.apachegeode-ci.info/api/v1/cli?arch=amd64&platform=linux"
chmod +x ${BIN_DIR}/fly
echo "Sanitized Geode Fork = ${SANITIZED_GEODE_FORK}"
echo "Sanitized Goede Branch = ${SANITIZED_GEODE_BRANCH}"

PATH=${PATH}:${BIN_DIR}

TARGET="geode"

if [[ "${SANITIZED_GEODE_FORK}" == "apache" ]]; then
PIPELINE_NAME=${SANITIZED_GEODE_BRANCH}
DOCKER_IMAGE_PREFIX=""
else
PIPELINE_NAME="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}"
DOCKER_IMAGE_PREFIX="${PIPELINE_NAME}-"
fi
MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
MY_ZONE=${MY_ZONE##*/}
NETWORK_INTERFACE_INFO="$(gcloud compute instances describe ${MY_NAME} --zone ${MY_ZONE} --format="json(networkInterfaces)")"
GCP_NETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].network')
GCP_NETWORK=${GCP_NETWORK##*/}
GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
ENV_ID=$(echo ${GCP_NETWORK} | awk -F- '{ print $1}')
VERSION_BUCKET="concourse-${ENV_ID}-version"

#echo "DEBUG INFO *****************************"
#echo "Pipeline prefix = ${PIPELINE_PREFIX}"
#echo "Docker image prefix = ${DOCKER_IMAGE_PREFIX}"]
pushd ${SCRIPTDIR} 2>&1 > /dev/null
# Template and output share a directory with this script, but variables are shared in the parent directory.
python3 ../render.py $(basename ${SCRIPTDIR}) || exit 1

grep -n . generated-pipeline.yml
python3 ../render.py $(basename ${SCRIPTDIR}) ${GEODE_FORK} ${GEODE_BRANCH} ${UPSTREAM_FORK} || exit 1
popd 2>&1 > /dev/null
cp ${SCRIPTDIR}/generated-pipeline.yml ${OUTPUT_DIRECTORY}/generated-pipeline.yml

fly login -t ${TARGET} \
-c https://concourse.apachegeode-ci.info \
-u ${CONCOURSE_USERNAME} \
-p ${CONCOURSE_PASSWORD}
cat > ${OUTPUT_DIRECTORY}/pipeline-vars.yml <<YML
geode-build-branch: ${GEODE_BRANCH}
geode-fork: ${GEODE_FORK}
geode-repo-name: ${GEODE_REPO_NAME}
upstream-fork: ${UPSTREAM_FORK}
pipeline-prefix: "${PIPELINE_PREFIX}"
public-pipelines: ${PUBLIC_PIPELINES}
gcp-project: ${GCP_PROJECT}
version-bucket: ${VERSION_BUCKET}
artifact-bucket: ${ARTIFACT_BUCKET}
YML

fly -t ${TARGET} set-pipeline \
--non-interactive \
--pipeline ${PIPELINE_NAME} \
--var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
--config ${SCRIPTDIR}/generated-pipeline.yml

if [[ "${SANITIZED_GEODE_FORK}" == "apache" ]]; then
fly -t ${TARGET} expose-pipeline -p ${PIPELINE_NAME}
fi
popd 2>&1 > /dev/null
Loading

0 comments on commit afdc9cb

Please sign in to comment.