extract i18n_tasks gem

fixes CNVS-11181

test plan:
 * i18n rake tasks still operate correctly

Change-Id: I261973a947ce28840d6817f5a931f5221192e8db
Reviewed-on: https://gerrit.instructure.com/31441
Reviewed-by: Jon Jensen <[email protected]>
QA-Review: Clare Strong <[email protected]>
Tested-by: Jenkins <[email protected]>
Product-Review: Simon Williams <[email protected]>

Gemfile.d/i18n_tools.rb

@@ -1,6 +1,5 @@
 group :i18n_tools do
-  gem 'ya2yaml', '0.30'
-
   gem 'i18n_extraction', :path => 'gems/i18n_extraction', :require => false
+  gem 'i18n_tasks', :path => 'gems/i18n_tasks', :require => false
 end
 

config/initializers/i18n.rb

@@ -11,7 +11,7 @@
 
 I18n.enforce_available_locales = true
 
-I18n.send :extend, I18n::Lolcalize if ENV['LOLCALIZE']
+I18n.send :extend, I18nTasks::Lolcalize if ENV['LOLCALIZE']
 
 module I18nUtilities
   def before_label(text_or_key, default_value = nil, *args)

gems/canvas_breach_mitigation/test.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 result=0
 
-
 bundle install
 bundle exec rspec spec
 let result=$result+$?

gems/canvas_cassandra/test.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 result=0
 
-
 bundle install
 bundle exec rspec spec
 let result=$result+$?

gems/canvas_mimetype_fu/test.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 result=0
 
-
 bundle install
 bundle exec rspec spec
 let result=$result+$?

gems/i18n_tasks/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

gems/i18n_tasks/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+gemspec
+gem 'simplecov', '0.8.2', :require => false
+gem 'simplecov-rcov', '0.2.3', :require => false
+
+gem 'i18n_extraction', path: '../i18n_extraction'
+gem 'utf8_cleaner', path: '../utf8_cleaner'

gems/i18n_tasks/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

gems/i18n_tasks/i18n_tasks.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+unless defined?(CANVAS_RAILS3)
+  require File.expand_path("../../../config/canvas_rails3", __FILE__)
+end
+
+Gem::Specification.new do |spec|
+  spec.name          = "i18n_tasks"
+  spec.version       = '0.0.1'
+  spec.authors       = ["Raphael Weiner", "Stephan Hagemann"]
+  spec.email         = ["[email protected]", "[email protected]"]
+  spec.summary       = %q{Instructure i18n tasks gem}
+
+  spec.files         = Dir.glob("{lib,spec}/**/*") + %w(Rakefile test.sh)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  if CANVAS_RAILS3
+    spec.add_dependency "activesupport", "~> 3.2"
+  else
+    spec.add_dependency "activesupport", "~> 2.3"
+  end
+
+  spec.add_dependency "i18n", "0.6.8"
+  spec.add_dependency "ruby_parser", "3.1.3"
+  spec.add_dependency "ya2yaml", "0.30"
+  spec.add_dependency "i18n_extraction"
+  spec.add_dependency "utf8_cleaner"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

gems/i18n_tasks/lib/i18n_tasks.rb

@@ -0,0 +1,12 @@
+require "utf8_cleaner"
+require "i18n"
+require "sexp_processor"
+require "ruby_parser"
+require "json"
+
+module I18nTasks
+  require "i18n_tasks/hash_extensions"
+  require "i18n_tasks/lolcalize"
+  require "i18n_tasks/utils"
+  require "i18n_tasks/i18n_import"
+end

lib/i18n/hash_extensions.rb → gems/i18n_tasks/lib/i18n_tasks/hash_extensions.rb

@@ -1,6 +1,5 @@
-module I18n
+module I18nTasks
   module HashExtensions
-
     def flatten_keys(result={}, prefix='')
       each_pair do |k, v|
         if v.is_a?(Hash)
@@ -16,7 +15,7 @@ def expand_keys(result = {})
       each_pair do |k, v|
         parts = k.split('.')
         last = parts.pop
-        parts.inject(result){ |h, k2| h[k2] ||= {}}[last] = v
+        parts.inject(result) { |h, k2| h[k2] ||= {} }[last] = v
       end
       result
     end
@@ -29,5 +28,6 @@ def to_ordered
       end
     end
 
+    Hash.send(:include, HashExtensions)
   end
-end
+end

gems/i18n_tasks/lib/i18n_tasks/i18n_import.rb

@@ -0,0 +1,157 @@
+module I18nTasks
+  class I18nImport
+    attr_reader :source_translations, :new_translations, :language
+
+    def initialize(source_translations, new_translations)
+      @source_translations = init_source(source_translations)
+      @language = init_language(new_translations)
+      Utf8Cleaner.recursively_strip_invalid_utf8!(new_translations, true)
+      @new_translations = new_translations[language].flatten_keys
+    end
+
+    def compare_translations(&warning)
+      [
+          [missing_keys, "missing translations"],
+          [unexpected_keys, "unexpected translations"]
+      ].each do |keys, description|
+        if keys.present?
+          case (action = yield(keys.sort, description))
+            when :abort then
+              throw(:abort)
+            when :discard then
+              :ok # <-discard and accept are the same in this case
+            when :accept then
+              :ok # <-/
+            else
+              raise "don't know how to handle #{action}"
+          end
+        end
+      end
+    end
+
+    def compare_mismatches(&warning)
+      # Important to populate @placeholder_mismatches and @markdown_mismatches first
+      find_mismatches
+
+      [
+          [@placeholder_mismatches, "placeholder mismatches"],
+          [@markdown_mismatches, "markdown/wrapper mismatches"],
+      ].each do |mismatches, description|
+        if mismatches.size > 0
+          case (action = yield(mismatch_diff(mismatches), description))
+            when :abort then
+              throw(:abort)
+            when :discard then
+              @new_translations.delete_if do |k, v|
+                mismatches.keys.include? k
+              end
+            when :accept then
+              :ok
+            else
+              raise "don't know how to handle #{action}"
+          end
+        end
+      end
+    end
+
+    def compile_complete_translations(&warning)
+      catch(:abort) do
+        compare_translations(&warning)
+        compare_mismatches(&warning)
+        complete_translations
+      end
+    end
+
+    def complete_translations
+      I18n.available_locales
+      base = (I18n.backend.direct_lookup(language) || {})
+      translations = base.flatten_keys.merge(new_translations)
+      fix_plural_keys(translations)
+      translations.expand_keys
+    end
+
+    def fix_plural_keys(flat_hash)
+      other_keys = flat_hash.keys.grep(/\.other$/)
+      other_keys.each do |other_key|
+        one_key = other_key.gsub(/other$/, 'one')
+        if flat_hash[one_key].nil?
+          flat_hash[one_key] = flat_hash[other_key]
+        end
+      end
+    end
+
+    def missing_keys
+      source_translations.keys - new_translations.keys
+    end
+
+    def unexpected_keys
+      new_translations.keys - source_translations.keys
+    end
+
+    def find_mismatches
+      @placeholder_mismatches = {}
+      @markdown_mismatches = {}
+      new_translations.keys.each do |key|
+        next unless source_translations[key]
+        p1 = placeholders(source_translations[key].to_s)
+        p2 = placeholders(new_translations[key].to_s)
+        @placeholder_mismatches[key] = [p1, p2] if p1 != p2
+
+        m1 = markdown_and_wrappers(source_translations[key].to_s)
+        m2 = markdown_and_wrappers(new_translations[key].to_s)
+        @markdown_mismatches[key] = [m1, m2] if m1 != m2
+      end
+    end
+
+    def markdown_and_wrappers(str)
+      # Since underscores can be wrappers, and underscores can also be inside
+      # placeholders (as placeholder names) we need to be unambiguous about
+      # underscores in placeholders:
+      dashed_str = str.gsub(/%\{([^\}]+)\}/) { |x| x.gsub("_", "-") }
+      # some stuff this doesn't check (though we don't use):
+      #   blockquotes, e.g. "> some text"
+      #   reference links, e.g. "[an example][id]"
+      #   indented code
+      (
+      scan_and_report(dashed_str, /\\[\\`\*_\{\}\[\]\(\)#\+\-\.!]/) +
+          scan_and_report(dashed_str, /(\*+|_+|`+)[^\s].*?[^\s]?\1/).map { |m| "#{m[0]}-wrap" } +
+          scan_and_report(dashed_str, /(!?\[)[^\]]+\]\(([^\)"']+).*?\)/).map { |m| "link:#{m.last}" } +
+          scan_and_report(dashed_str, /^((\s*\*\s*){3,}|(\s*-\s*){3,}|(\s*_\s*){3,})$/).map { "hr" } +
+          scan_and_report(dashed_str, /^[^=\-\n]+\n^(=+|-+)$/).map { |m| m.first[0]=='=' ? 'h1' : 'h2' } +
+          scan_and_report(dashed_str, /^(\#{1,6})\s+[^#]*#*$/).map { |m| "h#{m.first.size}" } +
+          scan_and_report(dashed_str, /^ {0,3}(\d+\.|\*|\+|\-)\s/).map { |m| m.first =~ /\d/ ? "1." : "*" }
+      ).sort
+    end
+
+    def placeholders(str)
+      str.scan(/%h?\{[^\}]+\}/).sort
+    rescue ArgumentError => e
+      puts "Unable to scan string: #{str.inspect}"
+      raise e
+    end
+
+    def scan_and_report(str, re)
+      str.scan(re)
+    rescue ArgumentError => e
+      puts "Unable to scan string: #{str.inspect}"
+      raise e
+    end
+
+    private
+    def init_source(translations)
+      raise "Source does not have any English strings" unless translations.keys.include?('en')
+      translations['en'].flatten_keys
+    end
+
+    def init_language(translations)
+      raise "Translation file contains multiple languages" if translations.size > 1
+      language = translations.keys.first
+      raise "Translation file appears to have only English strings" if language == 'en'
+      language
+    end
+
+    def mismatch_diff(mismatches)
+      mismatches.map { |k, (p1, p2)| "#{k}: expected #{p1.inspect}, got #{p2.inspect}" }.sort
+    end
+  end
+end

lib/i18n/lolcalize.rb → gems/i18n_tasks/lib/i18n_tasks/lolcalize.rb

@@ -1,4 +1,4 @@
-module I18n
+module I18nTasks
   module Lolcalize
     def translate_with_lols(key, options = {})
       if options[:default]
@@ -38,4 +38,4 @@ class << self
       end
     end
   end
-end
+end

lib/i18n/utils.rb → gems/i18n_tasks/lib/i18n_tasks/utils.rb

@@ -1,10 +1,8 @@
-module I18n
+module I18nTasks
   module Utils
     CORE_KEYS = [:date, :time, :number, :datetime, :support]
 
     def self.dump_js(translations, locales = I18n.available_locales)
-      Hash.send :include, I18n::HashExtensions unless Hash.new.kind_of?(I18n::HashExtensions)
-
       # include all locales (even if untranslated) to avoid js errors
       locales.each { |locale| translations[locale.to_s] ||= {} }
 
@@ -17,4 +15,4 @@ def self.dump_js(translations, locales = I18n.available_locales)
       TRANSLATIONS
     end
   end
-end
+end