Skip to content

Commit

Permalink
Add keydb, dragonfly and clickhouse
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrasbacsai
andrasbacsai committed Apr 10, 2024
1 parent 605a630 commit c6844ff
Show file tree
Hide file tree
Showing 54 changed files with 2,405 additions and 130 deletions.
162 changes: 162 additions & 0 deletions app/Actions/Database/StartClickhouse.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
<?php

namespace App\Actions\Database;

use App\Models\StandaloneClickhouse;
use Illuminate\Support\Str;
use Symfony\Component\Yaml\Yaml;
use Lorisleiva\Actions\Concerns\AsAction;

class StartClickhouse
{
use AsAction;

public StandaloneClickhouse $database;
public array $commands = [];
public string $configuration_dir;

public function handle(StandaloneClickhouse $database)
{
$this->database = $database;

$startCommand = "clickhouse-server";

$container_name = $this->database->uuid;
$this->configuration_dir = database_configuration_dir() . '/' . $container_name;

$this->commands = [
"echo 'Starting {$database->name}.'",
"mkdir -p $this->configuration_dir",
];

$persistent_storages = $this->generate_local_persistent_volumes();
$volume_names = $this->generate_local_persistent_volumes_only_volume_names();
$environment_variables = $this->generate_environment_variables();

$docker_compose = [
'version' => '3.8',
'services' => [
$container_name => [
'image' => $this->database->image,
'command' => $startCommand,
'container_name' => $container_name,
'environment' => $environment_variables,
'restart' => RESTART_MODE,
'networks' => [
$this->database->destination->network,
],
'ulimits' => [
'nofile' => [
'soft' => 262144,
'hard' => 262144,
],
],
'labels' => [
'coolify.managed' => 'true',
],
'user' => $this->database->clickhouse_user,
'healthcheck' => [
'test' => "wget -qO- http://localhost:8123/ping || exit 1",
'interval' => '5s',
'timeout' => '5s',
'retries' => 10,
'start_period' => '5s'
],
'mem_limit' => $this->database->limits_memory,
'memswap_limit' => $this->database->limits_memory_swap,
'mem_swappiness' => $this->database->limits_memory_swappiness,
'mem_reservation' => $this->database->limits_memory_reservation,
'cpus' => (float) $this->database->limits_cpus,
'cpu_shares' => $this->database->limits_cpu_shares,
]
],
'networks' => [
$this->database->destination->network => [
'external' => true,
'name' => $this->database->destination->network,
'attachable' => true,
]
]
];
if (!is_null($this->database->limits_cpuset)) {
data_set($docker_compose, "services.{$container_name}.cpuset", $this->database->limits_cpuset);
}
if ($this->database->destination->server->isLogDrainEnabled() && $this->database->isLogDrainEnabled()) {
$docker_compose['services'][$container_name]['logging'] = [
'driver' => 'fluentd',
'options' => [
'fluentd-address' => "tcp://127.0.0.1:24224",
'fluentd-async' => "true",
'fluentd-sub-second-precision' => "true",
]
];
}
if (count($this->database->ports_mappings_array) > 0) {
$docker_compose['services'][$container_name]['ports'] = $this->database->ports_mappings_array;
}
if (count($persistent_storages) > 0) {
$docker_compose['services'][$container_name]['volumes'] = $persistent_storages;
}
if (count($volume_names) > 0) {
$docker_compose['volumes'] = $volume_names;
}
$docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($docker_compose);
$this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml";
$readme = generate_readme_file($this->database->name, now());
$this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
$this->commands[] = "echo 'Pulling {$database->image} image.'";
$this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
$this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
$this->commands[] = "echo 'Database started.'";
return remote_process($this->commands, $database->destination->server, callEventOnFinish: 'DatabaseStatusChanged');
}

private function generate_local_persistent_volumes()
{
$local_persistent_volumes = [];
foreach ($this->database->persistentStorages as $persistentStorage) {
$volume_name = $persistentStorage->host_path ?? $persistentStorage->name;
$local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
}
return $local_persistent_volumes;
}

private function generate_local_persistent_volumes_only_volume_names()
{
$local_persistent_volumes_names = [];
foreach ($this->database->persistentStorages as $persistentStorage) {
if ($persistentStorage->host_path) {
continue;
}
$name = $persistentStorage->name;
$local_persistent_volumes_names[$name] = [
'name' => $name,
'external' => false,
];
}
return $local_persistent_volumes_names;
}

private function generate_environment_variables()
{
$environment_variables = collect();
foreach ($this->database->runtime_environment_variables as $env) {
$environment_variables->push("$env->key=$env->real_value");
}

if ($environment_variables->filter(fn ($env) => Str::of($env)->contains('CLICKHOUSE_DB'))->isEmpty()) {
$environment_variables->push("CLICKHOUSE_DB={$this->database->clickhouse_db}");
}

if ($environment_variables->filter(fn ($env) => Str::of($env)->contains('CLICKHOUSE_USER'))->isEmpty()) {
$environment_variables->push("CLICKHOUSE_USER={$this->database->clickhouse_user}");
}

if ($environment_variables->filter(fn ($env) => Str::of($env)->contains('CLICKHOUSE_PASSWORD'))->isEmpty()) {
$environment_variables->push("CLICKHOUSE_PASSWORD={$this->database->clickhouse_password}");
}

return $environment_variables->all();
}
}
23 changes: 22 additions & 1 deletion app/Actions/Database/StartDatabaseProxy.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
namespace App\Actions\Database;

use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
Expand All @@ -15,7 +18,7 @@ class StartDatabaseProxy
{
use AsAction;

public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|ServiceDatabase $database)
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse|ServiceDatabase $database)
{
$internalPort = null;
$type = $database->getMorphClass();
Expand Down Expand Up @@ -50,6 +53,18 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St
$type = 'App\Models\StandaloneRedis';
$containerName = "redis-{$database->service->uuid}";
break;
case 'standalone-keydb':
$type = 'App\Models\StandaloneKeydb';
$containerName = "keydb-{$database->service->uuid}";
break;
case 'standalone-dragonfly':
$type = 'App\Models\StandaloneDragonfly';
$containerName = "dragonfly-{$database->service->uuid}";
break;
case 'standalone-clickhouse':
$type = 'App\Models\StandaloneClickhouse';
$containerName = "clickhouse-{$database->service->uuid}";
break;
}
}
if ($type === 'App\Models\StandaloneRedis') {
Expand All @@ -62,6 +77,12 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St
$internalPort = 3306;
} else if ($type === 'App\Models\StandaloneMariadb') {
$internalPort = 3306;
} else if ($type === 'App\Models\StandaloneKeydb') {
$internalPort = 6379;
} else if ($type === 'App\Models\StandaloneDragonfly') {
$internalPort = 6379;
} else if ($type === 'App\Models\StandaloneClickhouse') {
$internalPort = 9000;
}
$configuration_dir = database_proxy_dir($database->uuid);
$nginxconf = <<<EOF
Expand Down
173 changes: 173 additions & 0 deletions app/Actions/Database/StartDragonfly.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
<?php

namespace App\Actions\Database;

use App\Models\StandaloneDragonfly;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Symfony\Component\Yaml\Yaml;
use Lorisleiva\Actions\Concerns\AsAction;

class StartDragonfly
{
use AsAction;

public StandaloneDragonfly $database;
public array $commands = [];
public string $configuration_dir;


public function handle(StandaloneDragonfly $database)
{
$this->database = $database;

$startCommand = "dragonfly --requirepass {$this->database->dragonfly_password}";

$container_name = $this->database->uuid;
$this->configuration_dir = database_configuration_dir() . '/' . $container_name;

$this->commands = [
"echo 'Starting {$database->name}.'",
"mkdir -p $this->configuration_dir",
];

$persistent_storages = $this->generate_local_persistent_volumes();
$volume_names = $this->generate_local_persistent_volumes_only_volume_names();
$environment_variables = $this->generate_environment_variables();
$this->add_custom_dragonfly();

$docker_compose = [
'version' => '3.8',
'services' => [
$container_name => [
'image' => $this->database->image,
'command' => $startCommand,
'container_name' => $container_name,
'environment' => $environment_variables,
'restart' => RESTART_MODE,
'networks' => [
$this->database->destination->network,
],
'ulimits' => [
'memlock'=> '-1'
],
'labels' => [
'coolify.managed' => 'true',
],
'healthcheck' => [
'test' => "redis-cli -a {$this->database->dragonfly_password} ping",
'interval' => '5s',
'timeout' => '5s',
'retries' => 10,
'start_period' => '5s'
],
'mem_limit' => $this->database->limits_memory,
'memswap_limit' => $this->database->limits_memory_swap,
'mem_swappiness' => $this->database->limits_memory_swappiness,
'mem_reservation' => $this->database->limits_memory_reservation,
'cpus' => (float) $this->database->limits_cpus,
'cpu_shares' => $this->database->limits_cpu_shares,
]
],
'networks' => [
$this->database->destination->network => [
'external' => true,
'name' => $this->database->destination->network,
'attachable' => true,
]
]
];
if (!is_null($this->database->limits_cpuset)) {
data_set($docker_compose, "services.{$container_name}.cpuset", $this->database->limits_cpuset);
}
if ($this->database->destination->server->isLogDrainEnabled() && $this->database->isLogDrainEnabled()) {
$docker_compose['services'][$container_name]['logging'] = [
'driver' => 'fluentd',
'options' => [
'fluentd-address' => "tcp://127.0.0.1:24224",
'fluentd-async' => "true",
'fluentd-sub-second-precision' => "true",
]
];
}
if (count($this->database->ports_mappings_array) > 0) {
$docker_compose['services'][$container_name]['ports'] = $this->database->ports_mappings_array;
}
if (count($persistent_storages) > 0) {
$docker_compose['services'][$container_name]['volumes'] = $persistent_storages;
}
if (count($volume_names) > 0) {
$docker_compose['volumes'] = $volume_names;
}
if (!is_null($this->database->dragonfly_conf)) {
$docker_compose['services'][$container_name]['volumes'][] = [
'type' => 'bind',
'source' => $this->configuration_dir . '/dragonfly.conf',
'target' => '/etc/dragonfly/dragonfly.conf',
'read_only' => true,
];
$docker_compose['services'][$container_name]['command'] = "dragonfly /etc/dragonfly/dragonfly.conf --requirepass {$this->database->dragonfly_password}";
}
$docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($docker_compose);
$this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml";
$readme = generate_readme_file($this->database->name, now());
$this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
$this->commands[] = "echo 'Pulling {$database->image} image.'";
$this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml pull";
$this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
$this->commands[] = "echo 'Database started.'";
return remote_process($this->commands, $database->destination->server, callEventOnFinish: 'DatabaseStatusChanged');
}

private function generate_local_persistent_volumes()
{
$local_persistent_volumes = [];
foreach ($this->database->persistentStorages as $persistentStorage) {
$volume_name = $persistentStorage->host_path ?? $persistentStorage->name;
$local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
}
return $local_persistent_volumes;
}

private function generate_local_persistent_volumes_only_volume_names()
{
$local_persistent_volumes_names = [];
foreach ($this->database->persistentStorages as $persistentStorage) {
if ($persistentStorage->host_path) {
continue;
}
$name = $persistentStorage->name;
$local_persistent_volumes_names[$name] = [
'name' => $name,
'external' => false,
];
}
return $local_persistent_volumes_names;
}

private function generate_environment_variables()
{
$environment_variables = collect();
foreach ($this->database->runtime_environment_variables as $env) {
$environment_variables->push("$env->key=$env->real_value");
}

if ($environment_variables->filter(fn ($env) => Str::of($env)->contains('REDIS_PASSWORD'))->isEmpty()) {
$environment_variables->push("REDIS_PASSWORD={$this->database->dragonfly_password}");
}

return $environment_variables->all();
}
private function add_custom_dragonfly()
{
if (is_null($this->database->dragonfly_conf)) {
return;
}
$filename = 'dragonfly.conf';
Storage::disk('local')->put("tmp/dragonfly.conf_{$this->database->uuid}", $this->database->dragonfly_conf);
$path = Storage::path("tmp/dragonfly.conf_{$this->database->uuid}");
instant_scp($path, "{$this->configuration_dir}/{$filename}", $this->database->destination->server);
Storage::disk('local')->delete("tmp/dragonfly.conf_{$this->database->uuid}");
}
}
Loading

0 comments on commit c6844ff

Please sign in to comment.