fix: database backups

app/Http/Livewire/Project/Database/BackupEdit.php

@@ -8,6 +8,7 @@ class BackupEdit extends Component
 {
     public $backup;
     public $s3s;
+    public ?string $status = null;
     public array $parameters;
 
     protected $rules = [

app/Http/Livewire/Team/Storage/Create.php

@@ -64,7 +64,7 @@ public function submit()
             }
             $this->storage->team_id = currentTeam()->id;
             $this->storage->testConnection();
-            $this->emit('success', 'Connection is working. Tested with "ListObjectsV2" action.');
+            $this->storage->is_usable = true;
             $this->storage->save();
             return redirect()->route('team.storages.show', $this->storage->uuid);
         } catch (\Throwable $e) {

app/Http/Livewire/Team/Storage/Form.php

@@ -9,6 +9,7 @@ class Form extends Component
 {
     public S3Storage $storage;
     protected $rules = [
+        'storage.is_usable' => 'nullable|boolean',
         'storage.name' => 'nullable|min:3|max:255',
         'storage.description' => 'nullable|min:3|max:255',
         'storage.region' => 'required|max:255',
@@ -18,6 +19,7 @@ class Form extends Component
         'storage.endpoint' => 'required|url|max:255',
     ];
     protected $validationAttributes = [
+        'storage.is_usable' => 'Is Usable',
         'storage.name' => 'Name',
         'storage.description' => 'Description',
         'storage.region' => 'Region',

app/Jobs/DatabaseBackupJob.php

@@ -31,7 +31,7 @@ class DatabaseBackupJob implements ShouldQueue, ShouldBeEncrypted
 
     public ?string $container_name = null;
     public ?ScheduledDatabaseBackupExecution $backup_log = null;
-    public string $backup_status;
+    public string $backup_status = 'failed';
     public ?string $backup_location = null;
     public string $backup_dir;
     public string $backup_file;
@@ -74,7 +74,7 @@ public function handle(): void
                 $ip = Str::slug($this->server->ip);
                 $this->backup_dir = backup_dir() . "/coolify" . "/coolify-db-$ip";
             }
-            $this->backup_file = "/pg_dump-" . Carbon::now()->timestamp . ".dump";
+            $this->backup_file = "/pg-backup-customformat-" . Carbon::now()->timestamp . ".backup";
             $this->backup_location = $this->backup_dir . $this->backup_file;
 
             $this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -90,10 +90,17 @@ public function handle(): void
                 $this->upload_to_s3();
             }
             $this->save_backup_logs();
+            $this->team->notify(new BackupSuccess($this->backup, $this->database));
+            $this->backup_status = 'success';
         } catch (\Throwable $e) {
-            ray($e->getMessage());
+            $this->backup_status = 'failed';
             send_internal_notification('DatabaseBackupJob failed with: ' . $e->getMessage());
+            $this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
             throw $e;
+        } finally {
+            $this->backup_log->update([
+                'status' => $this->backup_status,
+            ]);
         }
     }
 
@@ -103,28 +110,15 @@ private function backup_standalone_postgresql(): void
             ray($this->backup_dir);
             $commands[] = "mkdir -p " . $this->backup_dir;
             $commands[] = "docker exec $this->container_name pg_dump -Fc -U {$this->database->postgres_user} > $this->backup_location";
-
             $this->backup_output = instant_remote_process($commands, $this->server);
-
             $this->backup_output = trim($this->backup_output);
-
             if ($this->backup_output === '') {
                 $this->backup_output = null;
             }
-
             ray('Backup done for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location);
-
-            $this->backup_status = 'success';
-            $this->team->notify(new BackupSuccess($this->backup, $this->database));
         } catch (\Throwable $e) {
-            $this->backup_status = 'failed';
             $this->add_to_backup_output($e->getMessage());
             ray('Backup failed for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location . '\n\nError:' . $e->getMessage());
-            $this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
-        } finally {
-            $this->backup_log->update([
-                'status' => $this->backup_status,
-            ]);
         }
     }
 
@@ -163,19 +157,24 @@ private function upload_to_s3(): void
             }
             $key = $this->s3->key;
             $secret = $this->s3->secret;
-            //            $region = $this->s3->region;
+            // $region = $this->s3->region;
             $bucket = $this->s3->bucket;
             $endpoint = $this->s3->endpoint;
+            $this->s3->testConnection();
+            if (isDev()) {
+                $commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v coolify_coolify-data-dev:/data/coolify:ro ghcr.io/coollabsio/coolify-helper >/dev/null 2>&1";
+            } else {
+                $commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro ghcr.io/coollabsio/coolify-helper >/dev/null 2>&1";
+            }
 
-            $commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro ghcr.io/coollabsio/coolify-helper";
             $commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret";
             $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
             instant_remote_process($commands, $this->server);
             $this->add_to_backup_output('Uploaded to S3.');
             ray('Uploaded to S3. ' . $this->backup_location . ' to s3://' . $bucket . $this->backup_dir);
         } catch (\Throwable $e) {
             $this->add_to_backup_output($e->getMessage());
-            ray($e->getMessage());
+            throw $e;
         } finally {
             $command = "docker rm -f backup-of-{$this->backup->uuid}";
             instant_remote_process([$command], $this->server);

app/Models/S3Storage.php

@@ -3,13 +3,16 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Support\Facades\Storage;
 
 class S3Storage extends BaseModel
 {
     use HasFactory;
 
     protected $guarded = [];
     protected $casts = [
+        'is_usable' => 'boolean',
         'key' => 'encrypted',
         'secret' => 'encrypted',
     ];
@@ -19,15 +22,50 @@ static public function ownedByCurrentTeam(array $select = ['*'])
         $selectArray = collect($select)->concat(['id']);
         return S3Storage::whereTeamId(currentTeam()->id)->select($selectArray->all())->orderBy('name');
     }
+    public function isUsable()
+    {
+        return $this->is_usable;
+    }
 
+    public function team()
+    {
+        return $this->belongsTo(Team::class);
+    }
     public function awsUrl()
     {
         return "{$this->endpoint}/{$this->bucket}";
     }
 
     public function testConnection()
     {
-        set_s3_target($this);
-        return \Storage::disk('custom-s3')->files();
+        try {
+            set_s3_target($this);
+            Storage::disk('custom-s3')->files();
+            $this->unusable_email_sent = false;
+            $this->is_usable = true;
+            return;
+        } catch (\Throwable $e) {
+            $this->is_usable = false;
+            if ($this->unusable_email_sent === false) {
+                $mail = new MailMessage();
+                $mail->subject('Coolify: S3 Storage Connection Error');
+                $mail->view('emails.s3-connection-error', ['name' => $this->name, 'reason' => $e->getMessage(), 'url' => route('team.storages.show', ['storage_uuid' => $this->uuid])]);
+                $users = collect([]);
+                $members = $this->team->members()->get();
+                foreach ($members as $user) {
+                    if ($user->isAdmin()) {
+                        $users->push($user);
+                    }
+                }
+                foreach ($users as $user) {
+                    send_user_an_email($mail, $user->email);
+                }
+                $this->unusable_email_sent = true;
+            }
+
+            throw $e;
+        } finally {
+            $this->save();
+        }
     }
 }

app/Models/Team.php

@@ -48,6 +48,7 @@ public function getRecepients($notification)
         }
         return explode(',', $recipients);
     }
+
     public function limits(): Attribute
     {
         return Attribute::make(
@@ -125,7 +126,7 @@ public function sources()
 
     public function s3s()
     {
-        return $this->hasMany(S3Storage::class);
+        return $this->hasMany(S3Storage::class)->where('is_usable', true);
     }
     public function trialEnded() {
         foreach ($this->servers as $server) {

app/Notifications/Application/DeploymentFailed.php

@@ -52,10 +52,10 @@ public function toMail(): MailMessage
         $pull_request_id = data_get($this->preview, 'pull_request_id', 0);
         $fqdn = $this->fqdn;
         if ($pull_request_id === 0) {
-            $mail->subject('❌ Deployment failed of ' . $this->application_name . '.');
+            $mail->subject('Coolify: Deployment failed of ' . $this->application_name . '.');
         } else {
             $fqdn = $this->preview->fqdn;
-            $mail->subject('❌ Deployment failed of pull request #' . $this->preview->pull_request_id . ' of ' . $this->application_name . '.');
+            $mail->subject('Coolify: Deployment failed of pull request #' . $this->preview->pull_request_id . ' of ' . $this->application_name . '.');
         }
         $mail->view('emails.application-deployment-failed', [
             'name' => $this->application_name,
@@ -69,20 +69,20 @@ public function toMail(): MailMessage
     public function toDiscord(): string
     {
         if ($this->preview) {
-            $message = '❌ Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
+            $message = 'Coolify:  Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
             $message .= '[View Deployment Logs](' . $this->deployment_url . ')';
         } else {
-            $message = '❌ Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
+            $message = 'Coolify: Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
             $message .= '[View Deployment Logs](' . $this->deployment_url . ')';
         }
         return $message;
     }
     public function toTelegram(): array
     {
         if ($this->preview) {
-            $message = '❌ Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
+            $message = 'Coolify: Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
         } else {
-            $message = '❌ Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
+            $message = 'Coolify: Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
         }
         return [
             "message" => $message,

app/Notifications/Application/DeploymentSuccess.php

@@ -52,10 +52,10 @@ public function toMail(): MailMessage
         $pull_request_id = data_get($this->preview, 'pull_request_id', 0);
         $fqdn = $this->fqdn;
         if ($pull_request_id === 0) {
-            $mail->subject("✅ New version is deployed of {$this->application_name}");
+            $mail->subject("Coolify: New version is deployed of {$this->application_name}");
         } else {
             $fqdn = $this->preview->fqdn;
-            $mail->subject("✅ Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");
+            $mail->subject("Coolify: Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");
         }
         $mail->view('emails.application-deployment-success', [
             'name' => $this->application_name,
@@ -69,15 +69,15 @@ public function toMail(): MailMessage
     public function toDiscord(): string
     {
         if ($this->preview) {
-            $message = '✅ New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '
+            $message = 'Coolify:  New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '
 
 ';
             if ($this->preview->fqdn) {
                 $message .= '[Open Application](' . $this->preview->fqdn . ') | ';
             }
             $message .= '[Deployment logs](' . $this->deployment_url . ')';
         } else {
-            $message = '✅ New version successfully deployed of ' . $this->application_name . '
+            $message = 'Coolify: New version successfully deployed of ' . $this->application_name . '
 
 ';
             if ($this->fqdn) {
@@ -90,7 +90,7 @@ public function toDiscord(): string
     public function toTelegram(): array
     {
         if ($this->preview) {
-            $message = '✅ New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '';
+            $message = 'Coolify: New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '';
             if ($this->preview->fqdn) {
                 $buttons[] = [
                     "text" => "Open Application",

app/Notifications/Application/StatusChanged.php

@@ -45,7 +45,7 @@ public function toMail(): MailMessage
     {
         $mail = new MailMessage();
         $fqdn = $this->fqdn;
-        $mail->subject("⛔ {$this->application_name} has been stopped");
+        $mail->subject("Coolify: {$this->application_name} has been stopped");
         $mail->view('emails.application-status-changes', [
             'name' => $this->application_name,
             'fqdn' => $fqdn,
@@ -56,15 +56,15 @@ public function toMail(): MailMessage
 
     public function toDiscord(): string
     {
-        $message = '⛔ ' . $this->application_name . ' has been stopped.
+        $message = 'Coolify: ' . $this->application_name . ' has been stopped.
 
 ';
         $message .= '[Open Application in Coolify](' . $this->application_url . ')';
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = '⛔ ' . $this->application_name . ' has been stopped.';
+        $message = 'Coolify: ' . $this->application_name . ' has been stopped.';
         return [
             "message" => $message,
             "buttons" => [

app/Notifications/Container/ContainerRestarted.php

@@ -27,7 +27,7 @@ public function via(object $notifiable): array
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}");
+        $mail->subject("Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}");
         $mail->view('emails.container-restarted', [
             'containerName' => $this->name,
             'serverName' => $this->server->name,
@@ -38,12 +38,12 @@ public function toMail(): MailMessage
 
     public function toDiscord(): string
     {
-        $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+        $message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+        $message = "Coolify:  Container ({$this->name}) has been restarted automatically on {$this->server->name}";
         $payload = [
             "message" => $message,
         ];

app/Notifications/Container/ContainerStopped.php

@@ -26,7 +26,7 @@ public function via(object $notifiable): array
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("⛔ Container {$this->name} has been stopped on {$this->server->name}");
+        $mail->subject("Coolify: Container ({$this->name}) has been stopped on {$this->server->name}");
         $mail->view('emails.container-stopped', [
             'containerName' => $this->name,
             'serverName' => $this->server->name,
@@ -37,12 +37,12 @@ public function toMail(): MailMessage
 
     public function toDiscord(): string
     {
-        $message = "⛔ Container {$this->name} has been stopped on {$this->server->name}";
+        $message = "Coolify:  Container ({$this->name}) has been stopped on {$this->server->name}";
         return $message;
     }
     public function toTelegram(): array
     {
-        $message = "⛔ Container ($this->name} has been stopped on {$this->server->name}";
+        $message = "Coolify:  Container ($this->name} has been stopped on {$this->server->name}";
         $payload = [
             "message" => $message,
         ];

app/Notifications/Database/BackupFailed.php

@@ -30,7 +30,7 @@ public function via(object $notifiable): array
     public function toMail(): MailMessage
     {
         $mail = new MailMessage();
-        $mail->subject("❌ [ACTION REQUIRED] Backup FAILED for {$this->database->name}");
+        $mail->subject("Coolify: [ACTION REQUIRED] Backup FAILED for {$this->database->name}");
         $mail->view('emails.backup-failed', [
             'name' => $this->name,
             'frequency' => $this->frequency,
@@ -41,11 +41,11 @@ public function toMail(): MailMessage
 
     public function toDiscord(): string
     {
-        return "❌ Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
+        return "Coolify: Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
     }
     public function toTelegram(): array
     {
-        $message = "❌ Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
+        $message = "Coolify:  Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
         return [
             "message" => $message,
         ];