Issue F1bonacc1#107 - Fix goroutine leak when readiness probe on

issues/issue_107/after.4h.alloc_space.txt

@@ -0,0 +1,22 @@
+> go tool pprof -alloc_space heap.out
+File: process-compose
+Type: alloc_space
+Time: Dec 2, 2023 at 2:32am (IST)
+Duration: 30.01s, Total samples = 206.33MB
+Entering interactive mode (type "help" for commands, "o" for options)
+(pprof) top
+Showing nodes accounting for 199.44MB, 96.66% of 206.33MB total
+Dropped 72 nodes (cum <= 1.03MB)
+Showing top 10 nodes out of 26
+      flat  flat%   sum%        cum   cum%
+  187.04MB 90.65% 90.65%   187.04MB 90.65%  github.com/rivo/tview.(*TextView).parseAhead
+    5.90MB  2.86% 93.51%     5.90MB  2.86%  strings.(*Builder).WriteString (inline)
+       3MB  1.45% 94.97%     3.50MB  1.70%  github.com/rivo/tview.step
+       3MB  1.45% 96.42%   196.44MB 95.21%  github.com/rivo/tview.(*TextView).Draw
+    0.50MB  0.24% 96.66%     3.50MB  1.70%  github.com/rivo/tview.(*Table).Draw
+         0     0% 96.66%        2MB  0.97%  github.com/InVisionApp/go-health/v2.(*Health).startRunner.func1
+         0     0% 96.66%        2MB  0.97%  github.com/InVisionApp/go-health/v2.(*Health).startRunner.func2
+         0     0% 96.66%   200.44MB 97.15%  github.com/f1bonacc1/process-compose/src/cmd.startTui
+         0     0% 96.66%   140.93MB 68.30%  github.com/f1bonacc1/process-compose/src/tui.(*pcView).updateLogs.(*Application).QueueUpdateDraw.func4
+         0     0% 96.66%    59.51MB 28.84%  github.com/f1bonacc1/process-compose/src/tui.(*pcView).updateTable.(*Application).QueueUpdateDraw.func4
+(pprof)

issues/issue_107/after.4h.txt

@@ -0,0 +1,20 @@
+> go tool pprof heap.out
+File: process-compose
+Type: inuse_space
+Time: Dec 2, 2023 at 2:32am (IST)
+Duration: 30.01s, Total samples = 1.52MB
+Entering interactive mode (type "help" for commands, "o" for options)
+(pprof) top
+Showing nodes accounting for -528.06kB, 34.02% of 1552.40kB total
+Showing top 10 nodes out of 30
+      flat  flat%   sum%        cum   cum%
+ -528.17kB 34.02% 34.02%  -528.17kB 34.02%  regexp.(*bitState).reset
+  512.17kB 32.99%  1.03%   512.17kB 32.99%  internal/profile.(*Profile).postDecode
+ -512.06kB 32.99% 34.02%  -512.06kB 32.99%  github.com/rivo/tview.(*TextView).parseAhead
+         0     0% 34.02%  -528.17kB 34.02%  github.com/f1bonacc1/process-compose/src/app.(*Process).handleInfo
+         0     0% 34.02%  -528.17kB 34.02%  github.com/f1bonacc1/process-compose/src/app.(*Process).handleOutput
+         0     0% 34.02%  -512.06kB 32.99%  github.com/f1bonacc1/process-compose/src/cmd.startTui
+         0     0% 34.02%  -528.17kB 34.02%  github.com/f1bonacc1/process-compose/src/pclog.(*ProcessLogBuffer).Write
+         0     0% 34.02%  -528.17kB 34.02%  github.com/f1bonacc1/process-compose/src/tui.(*LogView).WriteString
+         0     0% 34.02%  -512.06kB 32.99%  github.com/f1bonacc1/process-compose/src/tui.(*pcView).updateLogs.(*Application).QueueUpdateDraw.func4
+         0     0% 34.02%  -512.06kB 32.99%  github.com/f1bonacc1/process-compose/src/tui.SetupTui

issues/issue_107/process-compose.yaml

@@ -0,0 +1,104 @@
+version: "0.5"
+
+log_level: debug
+log_length: 1000
+
+processes:
+  redis:
+    command: "sleep 999999999"
+    readiness_probe:
+      exec:
+        command: "pidof process-compose"
+      initial_delay_seconds: 1
+      period_seconds: 1
+      timeout_seconds: 1
+      success_threshold: 1
+      failure_threshold: 20
+
+  webserver-a:
+    command: "sleep 999999999"
+    readiness_probe:
+      http_get:
+        host: localhost
+        scheme: http
+        path: "/live"
+        port: 8080
+      initial_delay_seconds: 1
+      period_seconds: 1
+      timeout_seconds: 1
+      success_threshold: 1
+      failure_threshold: 20
+    depends_on:
+      redis:
+        condition: process_healthy
+
+  webserver-b:
+    command: "sleep 999999999"
+    readiness_probe:
+      http_get:
+        host: localhost
+        scheme: http
+        path: "/live"
+        port: 8080
+      initial_delay_seconds: 1
+      period_seconds: 1
+      timeout_seconds: 1
+      success_threshold: 1
+      failure_threshold: 20
+    depends_on:
+      redis:
+        condition: process_started
+    shutdown:
+      parent_only: true
+
+  rpyc-client:
+    command: "sleep 999999999"
+    depends_on:
+      webserver-b:
+        condition: process_healthy
+    shutdown:
+      signal: 1 # SIGHUP
+      timeout_seconds: 5
+    availability:
+      restart: on_failure
+      backoff_seconds: 10
+      max_restarts: 3
+
+  webserver-c:
+    command: "sleep 999999999"
+    readiness_probe:
+      http_get:
+        host: localhost
+        scheme: http
+        path: "/live"
+        port: 8080
+      initial_delay_seconds: 20
+      period_seconds: 30
+      timeout_seconds: 10
+      success_threshold: 1
+      failure_threshold: 4
+    depends_on:
+      webserver-b:
+        condition: process_healthy
+    availability:
+      restart: on_failure
+      backoff_seconds: 10
+      max_restarts: 3
+
+  httpclient:
+    command: "sleep 999999999"
+    depends_on:
+      webserver-b:
+        condition: process_healthy
+
+  pc_log:
+    command: "tail -f -n100 process-compose-${USER}.log"
+    working_dir: "/tmp"
+    namespace: debug
+
+  memory:
+    command: "./bin/process-compose project state --with-memory"
+    availability:
+      restart: always
+      backoff_seconds: 60
+    namespace: debug

src/app/process.go

@@ -38,9 +38,10 @@ type Process struct {
 	stateMtx      sync.Mutex
 	procCond      sync.Cond
 	procStateChan chan string
-	procReadyChan chan string
 	procReadyCtx  context.Context
 	readyCancelFn context.CancelFunc
+	procRunCtx    context.Context
+	runCancelFn   context.CancelFunc
 	procColor     func(a ...interface{}) string
 	noColor       func(a ...interface{}) string
 	redColor      func(a ...interface{}) string
@@ -83,13 +84,13 @@ func NewProcess(
 		logBuffer:     procLog,
 		shellConfig:   shellConfig,
 		procStateChan: make(chan string, 1),
-		procReadyChan: make(chan string, 1),
 		printLogs:     printLogs,
 		isMain:        isMain,
 		extraArgs:     extraArgs,
 	}
 
 	proc.procReadyCtx, proc.readyCancelFn = context.WithCancel(context.Background())
+	proc.procRunCtx, proc.runCancelFn = context.WithCancel(context.Background())
 	proc.setUpProbes()
 	proc.procCond = *sync.NewCond(proc)
 	return proc
@@ -153,7 +154,14 @@ func (p *Process) run() int {
 		log.Info().Msgf("Restarting %s in %v second(s)... Restarts: %d",
 			p.getName(), p.getBackoff().Seconds(), p.procState.Restarts)
 
-		time.Sleep(p.getBackoff())
+		select {
+		case <-p.procRunCtx.Done():
+			log.Debug().Str("process", p.getName()).Msg("process stopped while waiting to restart")
+			break
+		case <-time.After(p.getBackoff()):
+			p.handleInfo("\n")
+			continue
+		}
 	}
 	p.onProcessEnd(types.ProcessStateCompleted)
 	return p.procState.ExitCode
@@ -262,19 +270,17 @@ func (p *Process) waitUntilReady() bool {
 	for {
 		select {
 		case <-p.procReadyCtx.Done():
-			log.Error().Msgf("Process %s was aborted and won't become ready", p.getName())
-			return false
-		case ready := <-p.procReadyChan:
-			if ready == types.ProcessHealthReady {
+			if p.procState.Health == types.ProcessHealthReady {
 				return true
 			}
+			log.Error().Msgf("Process %s was aborted and won't become ready", p.getName())
+			return false
 		}
 	}
 }
 
 func (p *Process) wontRun() {
 	p.onProcessEnd(types.ProcessStateCompleted)
-
 }
 
 // perform graceful process shutdown if defined in configuration
@@ -285,6 +291,7 @@ func (p *Process) shutDownNoRestart() error {
 
 // perform graceful process shutdown if defined in configuration
 func (p *Process) shutDown() error {
+	p.runCancelFn()
 	if !p.isRunning() {
 		state := p.getState()
 		log.Debug().Msgf("process %s is in state %s not shutting down", p.getName(), state.Status)
@@ -579,7 +586,7 @@ func (p *Process) onReadinessCheckEnd(isOk, isFatal bool, err string) {
 		_ = p.shutDown()
 	} else if isOk {
 		p.procState.Health = types.ProcessHealthReady
-		p.procReadyChan <- types.ProcessHealthReady
+		p.readyCancelFn()
 	} else {
 		p.procState.Health = types.ProcessHealthNotReady
 	}

src/health/health_checks.go

@@ -81,6 +81,9 @@ func (p *Prober) healthCheckCompleted(state *health.State) {
 	if state.Status == OK {
 		ok = true
 	}
+	if p.stopped {
+		return
+	}
 	p.onCheckEndFunc(ok, fatal, state.Err)
 }
 

test_loop.bash

@@ -4,7 +4,7 @@
 #trap "echo ERROR: The program is terminated ; exit" SIGTERM
 trap 'echo CODE: $?; exit $EXIT_CODE' 1 2 3 15
 
-LOOPS=30000
+LOOPS=3000000
 for (( i=1; i<=LOOPS; i++ ))
 do
   sleep 0.1