Closes zammad#3450 - Add support for Redis as an optional web socket …

…session store back end.

.gitignore

@@ -16,6 +16,9 @@
 # local backup config file
 /contrib/backup/config
 
+# Dynamic environment config for Gitlab
+/.gitlab/environment.env
+
 # Third-Party ------------------------------------------------------------------
 # The config files / dev tools listed below are optional
 # and may not be present on most users' machines

.gitlab-ci.yml

@@ -59,6 +59,6 @@ cache:
 # Initialize application env
 before_script:
   - source /etc/profile.d/rvm.sh
-  - FRESHENVFILE=fresh.env && test -f $FRESHENVFILE && source $FRESHENVFILE
   - bundle install -j $(nproc) --path vendor
-  - bundle exec ruby script/build/database_config.rb
+  - bundle exec ruby .gitlab/configure_environment.rb
+  - source .gitlab/environment.env

.gitlab/ci/base.yml

@@ -64,20 +64,46 @@
   name: registry.znuny.com/docker/zammad-imap:stable
   alias: mail
 
+.docker_redis: &docker_redis
+  name: redis:latest
+  alias: redis
+
  # service templates
 .services_mysql: &services_mysql
   services:
     - <<: *docker_mysql
 
+.services_mysql_redis: &services_mysql_redis
+  variables:
+    REDIS_URL: "redis://redis:6379"
+  services:
+    - <<: *docker_mysql
+    - <<: *docker_redis
+
 .services_postgresql: &services_postgresql
   services:
     - <<: *docker_postgresql
 
+.services_postgresql_redis: &services_postgresql_redis
+  variables:
+    REDIS_URL: "redis://redis:6379"
+  services:
+    - <<: *docker_postgresql
+    - <<: *docker_redis
+
 .services_mysql_postgresql: &services_mysql_postgresql
   services:
     - <<: *docker_mysql
     - <<: *docker_postgresql
 
+.services_mysql_postgresql_redis: &services_mysql_postgresql_redis
+  variables:
+    REDIS_URL: "redis://redis:6379"
+  services:
+    - <<: *docker_mysql
+    - <<: *docker_postgresql
+    - <<: *docker_redis
+
 .services_postgresql_selenium: &services_postgresql_selenium
   services:
     - <<: *docker_postgresql
@@ -107,6 +133,18 @@
     - <<: *docker_selenium
     - <<: *docker_imap
 
+.services_mysql_postgresql_elasticsearch_selenium_imap_redis: &services_mysql_postgresql_elasticsearch_selenium_imap_redis
+  variables:
+    ELASTICSEARCH_TAG: 'stable'
+    REDIS_URL: "redis://redis:6379"
+  services:
+    - <<: *docker_mysql
+    - <<: *docker_postgresql
+    - <<: *docker_elasticsearch
+    - <<: *docker_selenium
+    - <<: *docker_imap
+    - <<: *docker_redis
+
 # we need at least one job to store and include this template
 # but we skip this via 'only' -> 'variables' -> '$IGNORE'
 # $IGNORE is not defined

.gitlab/ci/browser-core.yml

@@ -19,11 +19,10 @@ include:
     - .env_base
     - .variables_es
     - .variables_app_restart_cmd
-    - .services_mysql_postgresql_elasticsearch_selenium_imap
+    - .services_mysql_postgresql_elasticsearch_selenium_imap_redis
   variables:
     RAILS_ENV: "production"
   script:
-    - env
     - script/build/test_slice_tests.sh $TEST_SLICE
     - RAILS_ENV=test bundle exec rake db:create
     - bundle exec rake zammad:ci:test:start[with_elasticsearch]
@@ -54,7 +53,7 @@ include:
   extends:
     - .env_base
     - .variables_app_restart_cmd
-    - .services_mysql_postgresql
+    - .services_mysql_postgresql_redis
   variables:
     RAILS_ENV: "production"
 
@@ -64,9 +63,10 @@ include:
   extends:
     - .env_base
     - .variables_es
-    - .services_mysql_postgresql_elasticsearch_selenium_imap
+    - .services_mysql_postgresql_elasticsearch_selenium_imap_redis
   variables:
     RAILS_ENV: "test"
+    REDIS_URL: "redis://redis:6379"
 
 .template_browser-core_capybara: &template_browser-core_capybara
   extends:

.gitlab/ci/browser-integration.yml

@@ -15,7 +15,7 @@ include:
     - .env_base
     - .variables_app_restart_cmd
     - .variables_es
-    - .services_mysql_postgresql_elasticsearch_selenium_imap
+    - .services_mysql_postgresql_elasticsearch_selenium_imap_redis
   variables:
     RAILS_ENV: "test"
   script:

.gitlab/ci/pre.yml

@@ -28,7 +28,8 @@ zeitwerk_check:
     - .services_postgresql
   script:
     - bundle install -j $(nproc) --path vendor
-    - bundle exec ruby script/build/database_config.rb
+    - bundle exec ruby .gitlab/configure_environment.rb
+    - source .gitlab/environment.env
     - bundle exec rake zammad:db:init
     - bundle exec rails zeitwerk:check
 

.gitlab/ci/rspec.yml

@@ -36,7 +36,7 @@ rspec:integration:
   stage: test
   extends:
     - .env_base
-    - .services_mysql_postgresql
+    - .services_mysql_postgresql_redis
     - .rspec_integration_rules
   variables:
     RAILS_ENV: "test"

.gitlab/ci/rspec/mysql.yml

@@ -1,11 +1,11 @@
 rspec:mysql:
   stage: test
   extends:
-    - .services_mysql
+    - .services_mysql_redis
     - .template_rspec
 
 rspec:mysql:db_reset:
   stage: test
   extends:
-    - .services_mysql
+    - .services_mysql_redis
     - .template_rspec_db_reset

.gitlab/ci/rspec/postgresql.yml

@@ -1,11 +1,11 @@
 rspec:postgresql:
   stage: test
   extends:
-    - .services_postgresql
+    - .services_postgresql_redis
     - .template_rspec
 
 rspec:postgresql:db_reset:
   stage: test
   extends:
-    - .services_postgresql
+    - .services_postgresql_redis
     - .template_rspec_db_reset

.gitlab/ci/unit/mysql.yml

@@ -1,5 +1,5 @@
 unit:mysql:
   stage: test
   extends:
-    - .services_mysql
+    - .services_mysql_redis
     - .template_unit

.gitlab/ci/unit/postgresql.yml

@@ -1,5 +1,5 @@
 unit:postgresql:
   stage: test
   extends:
-    - .services_postgresql
+    - .services_postgresql_redis
     - .template_unit

.gitlab/configure_environment.rb

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
+
+require 'yaml'
+require 'resolv'
+
+#
+# Configures the CI system
+#   - either (randomly) mysql or postgresql, if it is available
+#   - (randomly) Redis or File as web socket session back end, if Redis is available
+#
+# Database config happens directly in config/database.yml, other settings are written to
+#   .gitlab/environment.env which must be sourced in the CI configuration.
+#
+
+class ConfigureEnvironment
+
+  @env_file_content = <<~EOF
+  #!/bin/bash
+  FRESHENVFILE=fresh.env && test -f $FRESHENVFILE && source $FRESHENVFILE
+  true
+  EOF
+
+  def self.configure_redis
+    if ENV['REDIS_URL'].nil? || ENV['REDIS_URL'].empty? # rubocop:disable Rails/Blank
+      puts 'Redis is not available, using File as web socket session back end.'
+      return
+    end
+    if [true, false].sample
+      puts 'Using Redis as web socket session back end.'
+      return
+    end
+    puts 'Using File as web socket session back end.'
+    @env_file_content += "unset REDIS_URL\n"
+  end
+
+  def self.configure_database # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+    if File.exist? File.join(__dir__, '../config/database.yml')
+      puts "'config/database.yml' already exists and will not be changed."
+      return
+    end
+
+    cnf = YAML.load_file(File.join(__dir__, '../config/database/database.yml'))
+    cnf.delete('default')
+
+    database = ENV['ENFORCE_DB_SERVICE']
+
+    # Lookup in /etc/hosts first: gitlab uses that if FF_NETWORK_PER_BUILD is not set.
+    if !database
+      hostsfile = '/etc/hosts'
+      database  = %w[postgresql mysql].shuffle.find do |possible_database|
+        File.foreach(hostsfile).any? { |l| l[possible_database] }
+      end
+    end
+
+    # Lookup via DNS if needed: gitlab uses that if FF_NETWORK_PER_BUILD is enabled.
+    if !database
+      dns = Resolv::DNS.new
+      dns.timeouts = 3
+      database = %w[postgresql mysql].shuffle.find do |possible_database|
+        # Perform a lookup of the database host to check if it is configured as a service.
+        if dns.getaddress possible_database
+          next possible_database
+        end
+      rescue Resolv::ResolvError
+        # Ignore DNS lookup errors
+      end
+    end
+
+    raise "Can't find any supported database." if database.nil?
+
+    puts "Using #{database} as database service."
+
+    db_settings_map = {
+      'postgresql' => {
+        'adapter'  => 'postgresql',
+        'username' => 'zammad',
+        'password' => 'zammad',
+        'host'     => 'postgresql', # db alias from gitlab-ci.yml
+      },
+      'mysql'      => {
+        'adapter'  => 'mysql2',
+        'username' => 'root',
+        'password' => 'zammad',
+        'host'     => 'mysql', # db alias from gitlab-ci.yml
+      }
+    }
+
+    # fetch DB settings from settings map and fallback to postgresql
+    db_settings = db_settings_map.fetch(database) { db_settings_map['postgresql'] }
+
+    %w[development test production].each do |environment|
+      cnf[environment].merge!(db_settings)
+    end
+
+    File.write(File.join(__dir__, '../config/database.yml'), Psych.dump(cnf))
+  end
+
+  def self.write_env_file
+    File.write(File.join(__dir__, 'environment.env'), @env_file_content)
+  end
+
+  def self.run
+    configure_redis
+    configure_database
+    write_env_file
+  end
+end
+
+ConfigureEnvironment.run

Gemfile

@@ -29,6 +29,7 @@ gem 'delayed_job_active_record'
 # core - websocket
 gem 'em-websocket'
 gem 'eventmachine'
+gem 'redis', require: false
 
 # core - password security
 gem 'argon2'

Gemfile.lock

@@ -442,6 +442,7 @@ GEM
     rb-inotify (0.10.0)
       ffi (~> 1.0)
     rchardet (1.8.0)
+    redis (4.2.5)
     regexp_parser (1.8.2)
     rest-client (2.0.2)
       http-cookie (>= 1.0.2, < 2.0)
@@ -666,6 +667,7 @@ DEPENDENCIES
   rails-controller-testing
   rb-fsevent
   rchardet (>= 1.8.0)
+  redis
   rspec-rails
   rszr (= 0.5.2)
   rubocop

config/application.rb

@@ -49,6 +49,9 @@ class Application < Rails::Application
     # define cache store
     config.cache_store = :zammad_file_store, Rails.root.join('tmp', "cache_file_store_#{Rails.env}"), { expires_in: 7.days }
 
+    # define websocket session store
+    config.websocket_session_store = ENV['REDIS_URL'] ? :redis : :file
+
     # Rails 6.1 returns false when the enqueuing is aborted.
     config.active_job.return_false_on_aborted_enqueue = true