Additional TDE configuration for Patroni pg_rewind

Provide a wrapper script to allow Patroni to invoke pg_rewind as
required for TDE and update configuration to enable pg_rewind in
Patroni for all versions > 10.

Issue: PGO-785

internal/patroni/config.go

@@ -206,6 +206,15 @@ func DynamicConfiguration(
 		// TODO(cbandy): explain this. requires an archive, perhaps.
 		"use_slots": false,
 	}
+
+	// When TDE is configured, override the pg_rewind binary name to point
+	// to the wrapper script.
+	if config.FetchKeyCommand(&cluster.Spec) != "" {
+		postgresql["bin_name"] = map[string]any{
+			"pg_rewind": "/tmp/pg_rewind_tde.sh",
+		}
+	}
+
 	if section, ok := root["postgresql"].(map[string]any); ok {
 		for k, v := range section {
 			postgresql[k] = v
@@ -279,12 +288,7 @@ func DynamicConfiguration(
 	// Recent versions of `pg_rewind` can run with limited permissions granted
 	// by Patroni to the user defined in "postgresql.authentication.rewind".
 	// PostgreSQL v10 and earlier require superuser access over the network.
-	//
-	// Additionally, Patroni does not currently provide an easy way to set the
-	// "encryption_key_command" value for `pg_rewind`, so if TDE is enabled,
-	// `use_pg_rewind` is set to false.
-	postgresql["use_pg_rewind"] = cluster.Spec.PostgresVersion > 10 &&
-		config.FetchKeyCommand(&cluster.Spec) == ""
+	postgresql["use_pg_rewind"] = cluster.Spec.PostgresVersion > 10
 
 	if cluster.Spec.Standby != nil && cluster.Spec.Standby.Enabled {
 		// Copy the "standby_cluster" section before making any changes.

internal/patroni/config_test.go

@@ -752,9 +752,10 @@ func TestDynamicConfiguration(t *testing.T) {
 				"loop_wait": int32(10),
 				"ttl":       int32(30),
 				"postgresql": map[string]any{
+					"bin_name":      map[string]any{"pg_rewind": string("/tmp/pg_rewind_tde.sh")},
 					"parameters":    map[string]any{},
 					"pg_hba":        []string{},
-					"use_pg_rewind": bool(false),
+					"use_pg_rewind": bool(true),
 					"use_slots":     bool(false),
 				},
 			},

internal/postgres/config.go

@@ -21,6 +21,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 
+	"github.com/crunchydata/postgres-operator/internal/config"
 	"github.com/crunchydata/postgres-operator/internal/naming"
 	"github.com/crunchydata/postgres-operator/internal/util"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
@@ -246,6 +247,18 @@ func startupCommand(
 		}
 	}
 
+	pg_rewind_override := ""
+	if config.FetchKeyCommand(&cluster.Spec) != "" {
+		// Quoting "EOF" disables parameter substitution during write.
+		// - https://tldp.org/LDP/abs/html/here-docs.html#EX71C
+		pg_rewind_override = `cat << "EOF" > /tmp/pg_rewind_tde.sh
+#!/bin/sh
+pg_rewind -K "$(postgres -C encryption_key_command)" "$@"
+EOF
+chmod +x /tmp/pg_rewind_tde.sh
+`
+	}
+
 	args := []string{version, walDir, naming.PGBackRestPGDataLogPath}
 	script := strings.Join([]string{
 		`declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3"`,
@@ -332,6 +345,9 @@ func startupCommand(
 			naming.ReplicationCert, naming.ReplicationPrivateKey,
 			naming.ReplicationCACert),
 
+		// Add the pg_rewind wrapper script, if TDE is enabled.
+		pg_rewind_override,
+
 		tablespaceCmd,
 		// When the data directory is empty, there's nothing more to do.
 		`[ -f "${postgres_data_directory}/PG_VERSION" ] || exit 0`,

internal/postgres/config_test.go

@@ -495,4 +495,24 @@ func TestStartupCommand(t *testing.T) {
 		assert.Assert(t, strings.HasPrefix(string(b), `|`),
 			"expected literal block scalar, got:\n%s", b)
 	})
+
+	t.Run("EnableTDE", func(t *testing.T) {
+
+		cluster.Spec.Patroni = &v1beta1.PatroniSpec{
+			DynamicConfiguration: map[string]any{
+				"postgresql": map[string]any{
+					"parameters": map[string]any{
+						"encryption_key_command": "echo test",
+					},
+				},
+			},
+		}
+		command := startupCommand(cluster, instance)
+		assert.Assert(t, len(command) > 3)
+		assert.Assert(t, strings.Contains(command[3], `cat << "EOF" > /tmp/pg_rewind_tde.sh
+#!/bin/sh
+pg_rewind -K "$(postgres -C encryption_key_command)" "$@"
+EOF
+chmod +x /tmp/pg_rewind_tde.sh`))
+	})
 }

internal/postgres/reconcile_test.go

@@ -251,6 +251,7 @@ initContainers:
     halt "$(permissions "${pgbrLog_directory}" ||:)"
     install -D --mode=0600 -t "/tmp/replication" "/pgconf/tls/replication"/{tls.crt,tls.key,ca.crt}
 
+
     [ -f "${postgres_data_directory}/PG_VERSION" ] || exit 0
     results 'data version' "${postgres_data_version:=$(< "${postgres_data_directory}/PG_VERSION")}"
     [[ "${postgres_data_version}" == "${expected_major_version}" ]] ||