Fixes zammad#2709, fixes zammad#2666, fixes zammad#2665, fixes zammad…

…#556, fixes zammad#3275 - Refactoring: Implement new translation toolchain based on gettext.

- Translations are no longer fetched from the cloud.
- Instead, they are extracted from the codebase and stored in i18n/zammad.pot.
- Translations will be managed via a public Weblate instance soon.
- The translated .po files are fed to the database as before.
- It is now possible to change "translation" strings for en-us locally via the admin GUI.
- It is no longer possible to submit local changes.

.coffeelint/rules/detect_translatable_string.coffee

@@ -0,0 +1,91 @@
+module.exports = class DetectTranslatableString
+
+  # coffeelint: disable=detect_translatable_string
+  rule:
+    name: 'detect_translatable_string'
+    level: 'ignore'
+    message: 'The following string looks like it should be marked as translatable via __(...)'
+    description: '''
+      '''
+
+  constructor: ->
+    @callTokens = []
+
+  tokens: ['STRING', 'CALL_START', 'CALL_END']
+
+  lintToken: (token, tokenApi) ->
+    [type, tokenValue] = token
+
+    if type in ['CALL_START', 'CALL_END']
+      @trackCall token, tokenApi
+      return
+
+    return false if @isInIgnoredMethod()
+
+    return @lintString(token, tokenApi)
+
+  lintString: (token, tokenApi) ->
+    [type, tokenValue] = token
+
+    # Remove quotes.
+    string = tokenValue[1..-2]
+
+    # Ignore strings with less than two words.
+    return false if string.split(' ').length < 2
+
+    # Ignore strings that are being used as exception; unlike Ruby exceptions, these should not reach the user.
+    return false if tokenApi.peek(-3)[1] == 'throw'
+    return false if tokenApi.peek(-2)[1] == 'throw'
+    return false if tokenApi.peek(-1)[1] == 'throw'
+
+    # Ignore strings that are being used for comparison
+    return false if tokenApi.peek(-1)[1] == '=='
+
+    # String interpolation is handled via concatenation, ignore such strings.
+    return false if tokenApi.peek(1)[1] == '+'
+    return false if tokenApi.peek(2)[1] == '+'
+
+    BLOCKLIST = [
+      # Only look at strings starting with upper case letters
+      /^[^A-Z]/,
+      # # Ignore strings starting with three upper case letters like SELECT, POST etc.
+      # /^[A-Z]{3}/,
+    ]
+
+    return false if BLOCKLIST.some (entry) ->
+      #console.log([string, entry, string.match(entry), token, tokenApi.peek(-1), tokenApi.peek(1)])
+      string.match(entry)
+
+    # console.log(tokenApi.peek(-3))
+    # console.log(tokenApi.peek(-2))
+    # console.log(tokenApi.peek(-1))
+    # console.log(token)
+
+    return { context: "Found: #{token[1]}" }
+
+  ignoredMethods: {
+    '__': true,
+    'log': true,
+    'T': true,
+    'controllerBind': true,
+    'error': true,  # App.Log.error
+    'set': true,  # App.Config.set
+    'translateInline': true,
+    'translateContent': true,
+    'translatePlain': true,
+  }
+
+  isInIgnoredMethod: ->
+    #console.log(@callTokens)
+    for t in @callTokens
+      return true if t.isIgnoredMethod
+    return false
+
+  trackCall: (token, tokenApi) ->
+    if token[0] is 'CALL_START'
+      p = tokenApi.peek(-1)
+      token.isIgnoredMethod = p and @ignoredMethods[p[1]]
+      @callTokens.push(token)
+    else
+      @callTokens.pop()
+    return null

.gitignore

@@ -37,8 +37,8 @@
 /db/*.sqlite3
 /db/schema.rb
 
-# translation cache files
-/config/locales*.yml
+# legacy translation cache files
+/config/locales-*.yml
 /config/translations/*.yml
 
 # NPM / Yarn

.gitlab/ci/pre.yml

@@ -22,7 +22,26 @@ shellcheck:
   script:
     - shellcheck -S warning $(find . -name "*.sh" -o -name "functions" | grep -v "/vendor/")
 
-zeitwerk_check:
+gettext lint:
+  <<: *template_pre
+  before_script:
+    - echo "Disable default before_script."
+  script:
+    - for FILE in i18n/*.pot i18n/*.po; do echo "Checking $FILE"; msgfmt -o /dev/null -c $FILE; done
+
+gettext catalog consistency:
+  <<: *template_pre
+  extends:
+    - .tags_docker
+    - .services_postgresql
+  script:
+    - bundle install -j $(nproc) --path vendor
+    - bundle exec ruby .gitlab/configure_environment.rb
+    - source .gitlab/environment.env
+    - bundle exec rake zammad:db:init
+    - bundle exec rails generate translation_catalog --check
+
+zeitwerk:check:
   <<: *template_pre
   extends:
     - .tags_docker
@@ -48,7 +67,7 @@ brakeman:
 coffeelint:
   <<: *template_pre
   script:
-    - coffeelint app/
+    - coffeelint --rules ./.coffeelint/rules/* app/
 
 stylelint:
   <<: *template_pre

.overcommit.yml

@@ -9,10 +9,18 @@ PreCommit:
     enabled: false
   RuboCop:
     enabled: true
-    on_warn: fail # Treat all warnings as failures
+    on_warn: fail
   CoffeeLint:
+    # .coffeelint/rules/* not supported in YAML, specify all rules separately.
+    flags: ['--reporter=csv', '--rules', './.coffeelint/rules/detect_translatable_string.coffee']
     enabled: true
-    on_warn: fail # Treat all warnings as failures
+    on_warn: fail
+    exclude: public/assets/chat/**/*
+  CustomScript:
+    enabled: true
+    description: 'Check if translation catalog is up-to-date'
+    required_executable: 'rails'
+    flags: ['generate', 'translation_catalog', '--check']
   Stylelint:
     enabled: true
 
@@ -43,4 +51,3 @@ PreRebase:
 PrepareCommitMsg:
   ALL:
     enabled: false
-

.rubocop/cop/zammad/detect_translatable_string.rb

@@ -0,0 +1,90 @@
+# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
+
+module RuboCop
+  module Cop
+    module Zammad
+      class DetectTranslatableString < Base
+        extend AutoCorrector
+
+        MSG = 'This string looks like it should be marked as translatable via __(...).'.freeze
+
+        def on_str(node)
+          # Constants like __FILE__ are handled as strings, but don't respond to begin.
+          return if !node.loc.respond_to?(:begin) || !node.loc.begin
+          return if part_of_ignored_node?(node)
+
+          return if !offense?(node)
+
+          add_offense(node) do |corrector|
+            corrector.replace(node, "__(#{node.source})")
+          end
+        end
+
+        def on_regexp(node)
+          ignore_node(node)
+        end
+
+        METHOD_NAME_BLOCKLIST = %i[
+          __ translate
+          include? eql? parse
+          debug info warn error fatal unknown log log_error
+        ].freeze
+
+        def on_send(node)
+          ignore_node(node) if METHOD_NAME_BLOCKLIST.include? node.method_name
+        end
+
+        private
+
+        PARENT_SOURCE_BLOCKLIST = [
+          # Ignore logged strings
+          'Rails.logger'
+        ].freeze
+
+        NODE_START_BLOCKLIST = [
+          # Only look at strings starting with upper case letters
+          %r{[^A-Z]},
+          # Ignore strings starting with three upper case letters like SELECT, POST etc.
+          %r{[A-Z]{3}},
+        ].freeze
+
+        NODE_CONTAIN_BLOCKLIST = [
+          # Ignore strings with interpolation.
+          '#{',
+          # Ignore Email addresses
+          '@'
+        ].freeze
+
+        def offense?(node) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+          # Ignore Hash Keys
+          return false if node.parent.type.eql?(:pair) && node.parent.children.first.equal?(node)
+
+          # Ignore equality checks like ... == 'My String'
+          return false if node.left_sibling.eql?(:==)
+
+          # Remove quotes
+          node_source = node.source[1..-2]
+
+          # Only match strings with at least two words
+          return false if node_source.split.count < 2
+
+          NODE_START_BLOCKLIST.each do |entry|
+            return false if node_source.start_with? entry
+          end
+
+          NODE_CONTAIN_BLOCKLIST.each do |entry|
+            return false if node_source.include? entry
+          end
+
+          parent_source = node.parent.source
+          PARENT_SOURCE_BLOCKLIST.each do |entry|
+            return false if parent_source.include? entry
+          end
+
+          true
+        end
+      end
+    end
+  end
+end

.rubocop/default.yml

@@ -317,6 +317,20 @@ Zammad/ExistsResetColumnInformation:
     - 'db/migrate/201*_*.rb'
     - 'db/migrate/2020*_*.rb'
 
+Zammad/DetectTranslatableString:
+  Enabled: true
+  Include:
+    - "app/**/*.rb"
+    - "db/**/*.rb"
+    - "lib/**/*.rb"
+  Exclude:
+    - "db/migrate/**/*.rb"
+    - "db/addon/**/*.rb"
+    - "lib/generators/**/*.rb"
+    - "lib/sequencer/**/*.rb"
+    - "lib/import/**/*.rb"
+    - "lib/tasks/**/*.rb"
+
 Zammad/ExistsDbStrategy:
   Include:
     - "spec/**/*.rb"

.rubocop/rubocop_zammad.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
 
 require_relative 'cop/zammad/exists_condition'
+require_relative 'cop/zammad/detect_translatable_string'
 require_relative 'cop/zammad/exists_date_time_precision'
 require_relative 'cop/zammad/exists_db_strategy'
 require_relative 'cop/zammad/exists_reset_column_information'

Gemfile

@@ -149,6 +149,9 @@ gem 'viewpoint', require: false
 # integrations - S/MIME
 gem 'openssl'
 
+# Translation sync
+gem 'PoParser', require: false
+
 # Gems used only for develop/test and not required
 # in production environments by default.
 group :development, :test do

Gemfile.lock

@@ -19,6 +19,8 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    PoParser (3.2.5)
+      simple_po_parser (~> 1.1.2)
     aasm (5.2.0)
       concurrent-ruby (~> 1.0)
     actioncable (6.0.4.1)
@@ -555,6 +557,7 @@ GEM
     shoulda-matchers (5.0.0)
       activesupport (>= 5.2.0)
     simple_oauth (0.3.1)
+    simple_po_parser (1.1.5)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -655,6 +658,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  PoParser
   aasm
   activerecord-import
   activerecord-nulldb-adapter

app/assets/javascripts/app/controllers/_application_controller/_base.coffee

@@ -114,7 +114,7 @@ class App.Controller extends Spine.Controller
     if window.clipboardData # IE
       window.clipboardData.setData('Text', text)
     else
-      window.prompt('Copy to clipboard: Ctrl+C, Enter', text)
+      window.prompt(__('Copy to clipboard: Ctrl+C, Enter'), text)
 
   # disable all delay's and interval's
   disconnectClient: ->

app/assets/javascripts/app/controllers/_application_controller/_modal.coffee

@@ -59,7 +59,7 @@ class App.ControllerModal extends App.Controller
     false
 
   content: ->
-    'You need to implement a one @content()!'
+    __('You need to implement a one @content()!')
 
   update: =>
     if @message
@@ -106,7 +106,7 @@ class App.ControllerModal extends App.Controller
     if @buttonSubmit is true
       @buttonSubmit = 'Submit'
     if @buttonCancel is true
-      @buttonCancel = 'Cancel & Go Back'
+      @buttonCancel = __('Cancel & Go Back')
 
     @update()