import submodules into project

builder/Dockerfile

@@ -0,0 +1,53 @@
+FROM deis/base:latest
+MAINTAINER Gabriel Monroy <[email protected]>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# install ssh server
+RUN apt-get install -yq openssh-server
+RUN rm /etc/ssh/ssh_host_*
+RUN dpkg-reconfigure openssh-server
+RUN mkdir -p /var/run/sshd
+
+# install docker in docker deps
+RUN apt-get install -yq aufs-tools iptables ca-certificates lxc
+RUN echo "deb http://get.docker.io/ubuntu docker main" > /etc/apt/sources.list.d/docker.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
+RUN apt-get update -q
+RUN apt-get install -yq lxc-docker-0.8.0
+
+# install hook dependencies
+RUN apt-get install -yq python-pip
+RUN pip install pyyaml requests
+
+# install hook utilities
+RUN apt-get install -yq curl vim
+
+# install all i18n locales
+RUN ln -s /usr/share/i18n/SUPPORTED /var/lib/locales/supported.d/all && locale-gen
+
+# install git and configure gituser
+ENV GITHOME /home/git
+ENV GITUSER git
+RUN apt-get install -yq git
+RUN useradd -d $GITHOME $GITUSER
+RUN mkdir -p $GITHOME/.ssh && chown git:git $GITHOME/.ssh
+RUN chown -R $GITUSER:$GITUSER $GITHOME
+
+# let the git user run `sudo /home/git/builder` (not writeable)
+RUN apt-get install -yq sudo
+RUN echo "%git    ALL=(ALL:ALL) NOPASSWD:/home/git/builder" >> /etc/sudoers
+
+# install custom confd
+RUN wget -q https://s3-us-west-2.amazonaws.com/deis/confd -O /usr/local/bin/confd
+RUN chmod +x /usr/local/bin/confd
+
+# add the current build context to /app
+ADD . /app
+RUN chown -R root:root /app
+
+# define the execution environment
+VOLUME /var/lib/docker
+ENTRYPOINT ["/app/bin/entry"]
+CMD ["/app/bin/boot"]
+EXPOSE 22

builder/Makefile

@@ -0,0 +1,16 @@
+build:
+	docker build -t deis/builder .
+
+config:
+	-etcdctl -C $${ETCD:-127.0.0.1:4001} setdir /deis
+	-etcdctl -C $${ETCD:-127.0.0.1:4001} setdir /deis/builder
+	etcdctl -C $${ETCD:-127.0.0.1:4001} set /deis/builder/port $${PORT:-22}
+
+run:
+	docker run -privileged -e ETCD=$${ETCD:-127.0.0.1:4001} -p $${PORT:-2222}:$${PORT:-22} -rm deis/builder ; exit 0
+
+shell:
+	docker run -privileged -e $${ETCD:-127.0.0.1:4001} -t -i -rm deis/builder /bin/bash
+
+clean:
+	-docker rmi deis/builder

builder/README.md

@@ -0,0 +1,23 @@
+# Deis Builder
+
+A Docker image that builds Docker images, for use in the [Deis](http://deis.io) open source PaaS.
+
+[![image](https://d207aa93qlcgug.cloudfront.net/img/icons/framed-icon-checked-repository.svg)](https://index.docker.io/u/deis/builder/)
+
+[**Trusted Build**](https://index.docker.io/u/deis/builder/)
+
+This Docker image is based on the trusted build [deis/base](https://index.docker.io/u/deis/base/), which itself is based on the official [ubuntu:12.04](https://index.docker.io/_/ubuntu/) base image.
+
+Please add any issues you find with this software to the parent [Deis project](https://github.com/opdemand/deis/issues).
+
+## Usage
+
+Coming Soon!
+
+## License
+
+Copyright 2014 OpDemand LLC
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0>
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

builder/authorized_keys

@@ -0,0 +1,2 @@
+command="/app/gitreceive run gabrtv 88:25:ed:67:56:91:3d:c6:1b:7f:42:c6:9b:41:24:80",no-agent-forwarding,no-pty,no-user-rc,no-X11-forwarding,no-port-forwarding ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDYU+u7FKdFoRq2BFyAVVznMhGmI4IiaqS3ewqpaaoSzMYoQ0UyUI7OOYHxDM1QI9E8ubX/j9+UJQFbGquT8cOO2pBqsjNQtumqzKgehzoDhjKomEC0KgqrCUjCA7JaLx5F+Lo1D8Vjco4K0CC1fjy6Lm8gbZxNC0D77xgyL2mmBbXO+LTm/CE+AmsHXh8OnLRzUVFdoOZYDJpCS7FlQixYhhd08nPjUqa9aAhpyURBwxLFojtdBFK/5lgOPlZBq0JSCn3lAPdrrZCA3zw6cs5AoPy3keOffbqoMs+/r9QqTPUrkfkt5AqPOXZaFkt3Xkaz53Tb6y696KiPr+2U1gSkuVnjxUUTXzukYGa6/r/CB6E7m8d70Sw1j3QGC1diohyEnYYuOWYv39GsOmfjjA0pTGwi3OiB5dWD1REfY6dJXbf2gamuULwy8q/C26fFNuTilg1JlBuxK8PzeWTuSCoeWdhXna1Z8BWQ4b55oV8puJeBiAl4NMgZ1k4vKGT4kBzMdhptYxHEt3IvnK7w/vzioLwBnhqORtiLheIsLCLKc2dWRDxbFq5540OiogqQMPYNOVboFxpbd5l0Z3mknULlmEXumgecg8UTwK4TE4eZkohhKEGG7EX65YqJOeQuKPSZjKC319Zs7h2fc6qS9tQO2M8/o6cQ2IYkTdVTyjz3OQ== [email protected]
+

builder/bin/boot

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# This script is designed to be run inside the container
+#
+
+# configure etcd
+export ETCD=${ETCD:-127.0.0.1:4001}
+export ETCD_PATH=${ETCD_PATH:-/deis/builder}
+export ETCD_TTL=${ETCD_TTL:-10}
+
+# fail hard and fast even on pipelines
+set -eo pipefail
+
+# configure service discovery
+export HOST=${HOST:-localhost}
+export PORT=${PORT:-22}
+export PROTO=${PROTO:-tcp}
+
+# wait for etcd to be available
+until etcdctl -C $ETCD ls >/dev/null; do
+	echo "waiting for etcd at $ETCD..."
+	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
+done
+
+# wait until etcd has discarded potentially stale values
+sleep $(($ETCD_TTL+1))
+
+# seed initial service configuration if necessary
+$(dirname ${BASH_SOURCE[0]})/seed >/dev/null
+
+# wait for confd to run once and install initial templates
+until confd -onetime -node $ETCD -config-file /app/confd.toml; do
+	echo "waiting for confd to write initial templates..."
+	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
+done
+
+# spawn the service in the background
+$(dirname ${BASH_SOURCE[0]})/start &
+SERVICE_PID=$!
+
+# smart shutdown on SIGINT and SIGTERM
+function on_exit() {
+    rm -f /var/run/docker.pid
+	kill -TERM $SERVICE_PID
+	wait $SERVICE_PID 2>/dev/null
+}
+trap on_exit INT TERM EXIT
+
+# spawn confd in the background to update services based on etcd changes
+confd -node $ETCD -config-file /app/confd.toml &
+CONFD_PID=$!
+
+# wait for the service to become available
+sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
+
+# as long as the service remains up, keep publishing to etcd with a TTL
+$(dirname ${BASH_SOURCE[0]})/publish &
+
+wait

builder/bin/entry

@@ -0,0 +1,85 @@
+#!/bin/bash
+set -eo pipefail
+
+# START jpetazzo/dind wrapper
+
+# First, make sure that cgroups are mounted correctly.
+CGROUP=/sys/fs/cgroup
+
+[ -d $CGROUP ] || 
+	mkdir $CGROUP
+
+mountpoint -q $CGROUP || 
+	mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
+		echo "Could not make a tmpfs mount. Did you use -privileged?"
+		exit 1
+	}
+
+if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
+then
+    mount -t securityfs none /sys/kernel/security || {
+        echo "Could not mount /sys/kernel/security."
+        echo "AppArmor detection and -privileged mode might break."
+    }
+fi
+
+# Mount the cgroup hierarchies exactly as they are in the parent system.
+for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+do
+        [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+        mountpoint -q $CGROUP/$SUBSYS || 
+                mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+        # The two following sections address a bug which manifests itself
+        # by a cryptic "lxc-start: no ns_cgroup option specified" when
+        # trying to start containers withina container.
+        # The bug seems to appear when the cgroup hierarchies are not
+        # mounted on the exact same directories in the host, and in the
+        # container.
+
+        # Named, control-less cgroups are mounted with "-o name=foo"
+        # (and appear as such under /proc/<pid>/cgroup) but are usually
+        # mounted on a directory named "foo" (without the "name=" prefix).
+        # Systemd and OpenRC (and possibly others) both create such a
+        # cgroup. To avoid the aforementioned bug, we symlink "foo" to
+        # "name=foo". This shouldn't have any adverse effect.
+        echo $SUBSYS | grep -q ^name= && {
+                NAME=$(echo $SUBSYS | sed s/^name=//)
+                ln -s $SUBSYS $CGROUP/$NAME
+        }
+
+        # Likewise, on at least one system, it has been reported that
+        # systemd would mount the CPU and CPU accounting controllers
+        # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
+        # but on a directory called "cpu,cpuacct" (note the inversion
+        # in the order of the groups). This tries to work around it.
+        [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+done
+
+# Note: as I write those lines, the LXC userland tools cannot setup
+# a "sub-container" properly if the "devices" cgroup is not in its
+# own hierarchy. Let's detect this and issue a warning.
+grep -q :devices: /proc/1/cgroup ||
+	echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
+grep -qw devices /proc/1/cgroup ||
+	echo "WARNING: it looks like the 'devices' cgroup is not mounted."
+
+# Now, close extraneous file descriptors.
+pushd /proc/self/fd >/dev/null
+for FD in *
+do
+	case "$FD" in
+	# Keep stdin/stdout/stderr
+	[012])
+		;;
+	# Nuke everything else
+	*)
+		eval exec "$FD>&-"
+		;;
+	esac
+done
+popd >/dev/null
+
+# END jpetazzo/dind wrapper
+
+exec $@

builder/bin/publish

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# configure etcd
+ETCD=${ETCD:-127.0.0.1:4001}
+ETCD_PATH=${ETCD_PATH:-/deis/builder}
+ETCD_TTL=${ETCD_TTL:-10}
+
+# while the port is listening, publish to etcd
+while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
+	etcdctl -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
+	etcdctl -C $ETCD set $ETCD_PATH/port $PORT --ttl $ETCD_TTL >/dev/null
+	sleep $(($ETCD_TTL/2)) # sleep for half the TTL
+done
+
+# if the loop quits, something went wrong
+exit 1

builder/bin/seed

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eo pipefail
+
+# if the keyspace already exists, exit early
+etcdctl -C $ETCD ls $ETCD_PATH >/dev/null && exit 0
+
+etcdctl -C $ETCD mkdir $ETCD_PATH/users || true

builder/bin/start

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eo pipefail
+
+# remove any pre-existing docker.sock
+rm -f /var/run/docker.sock
+
+# spawn a docker daemon to run builds
+docker -d &
+
+# wait for docker to start
+while [[ ! -e /var/run/docker.sock ]]; do
+  sleep 1
+done
+
+# pull required images
+docker pull deis/slugbuilder:latest
+docker pull deis/slugrunner:latest
+
+# start an SSH daemon to process `git push` requests
+/usr/sbin/sshd -D -e

builder/conf.d/authorized_keys.toml

@@ -0,0 +1,11 @@
+[template]
+src   = "authorized_keys"
+dest  = "/home/git/.ssh/authorized_keys"
+uid = 1000
+git = 1000
+mode  = "0600"
+keys = [
+  "/deis/builder/users",
+]
+#check_cmd  = "cat {{ .src }}"
+#reload_cmd = "/usr/sbin/service nginx restart"

builder/conf.d/builder.toml

@@ -0,0 +1,11 @@
+[template]
+src   = "builder"
+dest  = "/home/git/builder"
+uid = 0
+gid = 0
+mode  = "0755"
+keys = [
+  "/deis/controller",
+  "/deis/builder",
+  "/deis/registry",
+]

builder/conf.d/gitreceive.toml

@@ -0,0 +1,6 @@
+[template]
+src   = "gitreceive"
+dest  = "/usr/local/bin/gitreceive"
+uid = 0
+gid = 0
+mode  = "0755"

builder/conf.d/receiver.toml

@@ -0,0 +1,9 @@
+[template]
+src   = "receiver"
+dest  = "/home/git/receiver"
+uid = 0
+gid = 0
+mode  = "0755"
+keys = [
+  "/deis/controller",
+]

builder/confd.toml

@@ -0,0 +1,5 @@
+[confd]
+confdir  = "/app"
+interval = 5
+prefix   = "/"
+quiet = true

builder/templates/authorized_keys

@@ -0,0 +1,3 @@
+{{ range $user := .deis_builder_users }}{{ range $key := $user.Nodes }}
+command="/usr/local/bin/gitreceive run {{ Base $user.Key }} {{ Base $key.Key }}",no-agent-forwarding,no-pty,no-user-rc,no-X11-forwarding,no-port-forwarding {{ $key.Value }}
+{{ end }}{{ end }}