refactor container status check (solana-labs#30998)

* refactor container status check

* remove blank line at EOF

* add pagerduty integration

Co-authored-by: axleiro <[email protected]>

* fix discord webhook reference

* remove webhook references

---------

Co-authored-by: axleiro <[email protected]>

Author: joedenis01

metrics/metrics-main/alertmanager.sh

@@ -39,6 +39,6 @@ sudo docker run -it -d \
   --user root:root \
   --publish 9093:9093 \
   --name=alertmanager \
-  --volume "PWD"/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
+  --volume "$PWD"/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
   --volume /etc/hosts:/etc/hosts \
   $ALERTMANAGER_IMAGE

metrics/metrics-main/status.sh

@@ -1,32 +1,40 @@
-#!/bin/bash -ex
-#
-# Status of the InfluxDB/Chronograf/Grafana/Chronograf_8889 containers
-#
-cd "$(dirname "$0")"
-
-if [[ -z $HOST ]]; then
-  HOST=metrics.solana.com
-fi
-echo "HOST: $HOST"
-
-echo +++ status
-(
-  set -x
-  pwd
-  sudo docker ps --no-trunc --size
-  df -h
-  free -h
-  uptime
-)
-
-# If the container is not running state or exited state, then sent the notification on slack and redeploy the container again
-
-for container in chronograf_8889 grafana alertmanager alertmanager-discord prometheus chronograf kapacitor ; do
-          if [ "$(sudo docker inspect --format='{{.State.Status}}' $container)" != "running" ] || [ "$(sudo docker inspect --format='{{.State.Status}}' $container)" = "exited" ]; then
-        curl -X POST -H 'Content-type: application/json' --data '{"text": "'"$container"' container is down in the metrics-mainsystem server. Restarting..."}' "$SLACK_WEBHOOK"
-        curl -X POST -H 'Content-type: application/json' --data '{"content": "'"$container"' container is down in the metrics-mainsystem server. Restarting..."}' "$DISCORD_WEBHOOK"
-        echo "Starting up script"
-        sudo bash $container.sh
-        sleep 30
-     fi
-    done
+#!/bin/bash
+
+# List of containers
+containers=("chronograf_8889" "grafana" "alertmanager" "alertmanager-discord" "prometheus" "chronograf" "kapacitor")
+
+# Send a message to Discord
+send_discord_message() {
+  local message="$1"
+  curl -sS -H "Content-Type: application/json" -X POST -d "{\"content\": \"$message\"}" "$DISCORD_WEBHOOK"
+}
+
+# Send a critical alert to PagerDuty
+send_pagerduty_alert() {
+  local description="$1"
+  curl -sS -H "Content-Type: application/json" -X POST -d "{\"event_action\": \"trigger\", \"payload\": {\"summary\": \"$description\", \"source\": \"Docker Monitor\", \"severity\": \"critical\"}}" "$PAGERDUTY_WEBHOOK"
+}
+
+# Iterate over the containers and check their status
+for container in "${containers[@]}"; do
+  container_status=$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)
+
+  if [ "$container_status" != "running" ]; then
+    send_discord_message "$container is down and it's being redeployed..."
+
+    # Run the container.sh script to redeploy the container
+    chmod +x "$container.sh"
+    ./"$container.sh"
+    sleep 10
+
+    # Check the container status again
+    container_status=$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)
+
+    if [ "$container_status" != "running" ]; then
+      send_discord_message "$container failed to redeploy and manual intervention is required"
+      send_pagerduty_alert "$container failed to redeploy and manual intervention is required."
+    else
+      send_discord_message "$container has been redeployed successfully"
+    fi
+  fi
+done