Skip to content

Commit

Permalink
revive the liveness/readiness probe config (istio#3335)
Browse files Browse the repository at this point in the history
The real problem of 'crash loop' of mixer is failure of initialization;
the mixer CRD store will wait for up to 30 seconds for init.
(see
https://github.com/istio/istio/blob/master/mixer/pkg/config/crd/store.go#L52)

Considering this, this PR sets the initial delay for liveness
to be 60 seconds for safety.
  • Loading branch information
jmuk authored Feb 22, 2018
1 parent 9f9f816 commit c87983a
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 0 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ install/kubernetes/addons/zipkin-to-stackdriver.yaml
install/kubernetes/addons/zipkin.yaml
install/kubernetes/helm/istio/values.yaml
install/kubernetes/istio-auth.yaml
install/kubernetes/istio-mixer-with-health-check.yaml
install/kubernetes/istio-ca-plugin-certs.yaml
install/kubernetes/istio-ca-with-health-check.yaml
install/kubernetes/istio-one-namespace-auth.yaml
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#
# This file only contains the deployment configuration for mixer
# with liveness / readiness. This file is supposed to be used after
# installing istio normally (like istio.yaml) to update mixer setup.

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: istio-mixer
namespace: {ISTIO_NAMESPACE}
spec:
replicas: 1
template:
metadata:
labels:
istio: mixer
annotations:
sidecar.istio.io/inject: "false"
spec:
serviceAccountName: istio-mixer-service-account
containers:
- name: statsd-to-prometheus
image: prom/statsd-exporter:v0.5.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9102
- containerPort: 9125
protocol: UDP
args:
- '-statsd.mapping-config=/etc/statsd/mapping.conf'
volumeMounts:
- name: config-volume
mountPath: /etc/statsd
- name: mixer
image: {MIXER_HUB}/mixer:{MIXER_TAG}
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9091
- containerPort: 9093
- containerPort: 9094
- containerPort: 42422
args:
- --configStoreURL=k8s://
- --configDefaultNamespace={ISTIO_NAMESPACE}
- --zipkinURL=http://zipkin:9411/api/v1/spans
- --livenessProbePath=/live
- --livenessProbeInterval=5s
- --readinessProbePath=/ready
- --readinessProbeInterval=5s
- --logtostderr
- -v
- "2"
livenessProbe:
exec:
command:
- /usr/local/bin/mixs
- probe
- --probe-path=/live
- --interval=5s
initialDelaySeconds: 60
periodSeconds: 5
readinessProbe:
exec:
command:
- /usr/local/bin/mixs
- probe
- --probe-path=/ready
- --interval=5s
initialDelaySeconds: 5
periodSeconds: 5
- name: istio-proxy
image: {PROXY_HUB}/{PROXY_IMAGE}:{PROXY_TAG}
imagePullPolicy: IfNotPresent
ports:
- containerPort: 15004
args:
- proxy
- mixer
- -v
- "2"
- --controlPlaneAuthPolicy
- NONE #--controlPlaneAuthPolicy
- --customConfigFile
- /etc/istio/proxy/envoy_mixer.json
volumeMounts:
- name: istio-certs
mountPath: /etc/certs
readOnly: true
volumes:
- name: istio-certs
secret:
secretName: istio.istio-mixer-service-account
optional: true
- name: config-volume
configMap:
name: istio-mixer
7 changes: 7 additions & 0 deletions install/updateVersion.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ function merge_files() {
ISTIO_SIDECAR_INJECTOR_CONFIGMAP_RELEASE=$DEST/istio-sidecar-injector-configmap-release.yaml
ISTIO_CA_PLUGIN_CERTS=$DEST/istio-ca-plugin-certs.yaml
ISTIO_CA_HEALTH_CHECK=$DEST/istio-ca-with-health-check.yaml
ISTIO_MIXER_HEALTH_CHECK=$DEST/istio-mixer-with-health-check.yaml


if [ "$COMPONENT_FILES" = true ]; then
Expand Down Expand Up @@ -202,6 +203,10 @@ function merge_files() {
echo "# GENERATED FILE. Use with Kubernetes 1.7+" > $ISTIO_CA_HEALTH_CHECK
echo "# TO UPDATE, modify files in install/kubernetes/templates and run install/updateVersion.sh" >> $ISTIO_CA_HEALTH_CHECK
cat $SRC/istio-ca-with-health-check.yaml.tmpl >> $ISTIO_CA_HEALTH_CHECK

echo "# GENERATED FILE. Use with Kubernetes 1.7+" > $ISTIO_MIXER_HEALTH_CHECK
echo "# TO UPDATE, modify files in install/kubernetes/templates and run install/updateVersion.sh" >> $ISTIO_MIXER_HEALTH_CHECK
cat $SRC/istio-mixer-with-health-check.yaml.tmpl >> $ISTIO_MIXER_HEALTH_CHECK
}

function update_version_file() {
Expand Down Expand Up @@ -261,6 +266,7 @@ function update_istio_install() {
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-pilot.yaml.tmpl
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-ingress.yaml.tmpl
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-mixer.yaml.tmpl
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-mixer-with-health-check.yaml.tmpl
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-ca.yaml.tmpl
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-ca-one-namespace.yaml.tmpl
execute_sed "s|{ISTIO_NAMESPACE}|${ISTIO_NAMESPACE}|" istio-ca-plugin-certs.yaml.tmpl
Expand All @@ -272,6 +278,7 @@ function update_istio_install() {
execute_sed "s|image: {PILOT_HUB}/\(.*\):{PILOT_TAG}|image: ${PILOT_HUB}/\1:${PILOT_TAG}|" istio-pilot.yaml.tmpl
execute_sed "s|image: {PROXY_HUB}/{PROXY_IMAGE}:{PROXY_TAG}|image: ${PILOT_HUB}/${PROXY_IMAGE}:${PILOT_TAG}|" istio-pilot.yaml.tmpl
execute_sed "s|image: {MIXER_HUB}/\(.*\):{MIXER_TAG}|image: ${MIXER_HUB}/\1:${MIXER_TAG}|" istio-mixer.yaml.tmpl
execute_sed "s|image: {MIXER_HUB}/\(.*\):{MIXER_TAG}|image: ${MIXER_HUB}/\1:${MIXER_TAG}|" istio-mixer-with-health-check.yaml.tmpl
execute_sed "s|image: {PROXY_HUB}/{PROXY_IMAGE}:{PROXY_TAG}|image: ${PILOT_HUB}/${PROXY_IMAGE}:${PILOT_TAG}|" istio-mixer.yaml.tmpl
execute_sed "s|image: {CA_HUB}/\(.*\):{CA_TAG}|image: ${CA_HUB}/\1:${CA_TAG}|" istio-ca.yaml.tmpl
execute_sed "s|image: {CA_HUB}/\(.*\):{CA_TAG}|image: ${CA_HUB}/\1:${CA_TAG}|" istio-ca-one-namespace.yaml.tmpl
Expand Down

0 comments on commit c87983a

Please sign in to comment.