extract i18n_extraction gem

fixes CNVS-11182

test plan:
 * rake i18n tasks work correctly
 * rake jst tasks work correctly

Change-Id: I9777649e338d81cd7129c887acc18d9ef6722a92
Reviewed-on: https://gerrit.instructure.com/31440
Reviewed-by: Jon Jensen <[email protected]>
QA-Review: Clare Strong <[email protected]>
Tested-by: Jenkins <[email protected]>
Product-Review: Simon Williams <[email protected]>

Gemfile.d/i18n_tools.rb

@@ -1,6 +1,6 @@
 group :i18n_tools do
-  gem 'ruby_parser', '3.1.3'
-  gem 'sexp_processor', '4.2.1'
   gem 'ya2yaml', '0.30'
+
+  gem 'i18n_extraction', :path => 'gems/i18n_extraction', :require => false
 end
 

Rakefile

@@ -14,8 +14,10 @@ require 'rake/testtask'
 require 'rdoc/task'
 
 if CANVAS_RAILS2
+  Dir["#{RAILS_ROOT}/gems/**/lib/tasks/*.rake"].sort.each { |ext| load ext }
   require 'tasks/rails'
 else
+  Dir["#{Rails.root}/gems/**/lib/tasks/*.rake"].sort.each { |ext| load ext }
   CanvasRails::Application.load_tasks
 end
 begin; require 'parallelized_specs/lib/parallelized_specs/tasks'; rescue LoadError; end

gems/activesupport-suspend_callbacks/test.sh

@@ -8,11 +8,13 @@ bundle exec rspec spec
 let result=$result+$?
 
 echo "################ Running tests against Rails 3 ################"
-rm -f Gemfile.lock
+mv Gemfile.lock Gemfile.lock.rails2
 export CANVAS_RAILS3=true
 bundle install
 bundle exec rspec spec
 let result=$result+$?
+mv Gemfile.lock.rails2 Gemfile.lock
+
 
 if [ $result -eq 0 ]; then
   echo "SUCCESS"

gems/i18n_extraction/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

gems/i18n_extraction/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

gems/i18n_extraction/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

gems/i18n_extraction/i18n_extraction.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+unless defined?(CANVAS_RAILS3)
+  require File.expand_path("../../../config/canvas_rails3", __FILE__)
+end
+
+Gem::Specification.new do |spec|
+  spec.name = "i18n_extraction"
+  spec.version = '0.0.1'
+  spec.authors = ["Raphael Weiner"]
+  spec.email = ["[email protected]"]
+  spec.summary = %q{i18n extraction for Instructure}
+
+  spec.files = Dir.glob("{lib,spec}/**/*") + %w(Rakefile test.sh)
+  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sexp_processor", "4.2.1"
+  spec.add_dependency "ruby_parser", "3.1.3"
+  if CANVAS_RAILS3
+    spec.add_dependency "activesupport", "~> 3.2"
+  else
+    spec.add_dependency "activesupport", "~> 2.3"
+  end
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

gems/i18n_extraction/lib/i18n_extraction.rb

@@ -0,0 +1,11 @@
+require "sexp_processor"
+require "ruby_parser"
+require "json"
+require "active_support/all"
+
+module I18nExtraction
+  require "i18n_extraction/abstract_extractor"
+  require "i18n_extraction/handlebars_extractor"
+  require "i18n_extraction/js_extractor"
+  require "i18n_extraction/ruby_extractor"
+end

lib/i18n_extraction/abstract_extractor.rb → gems/i18n_extraction/lib/i18n_extraction/abstract_extractor.rb

@@ -57,6 +57,4 @@ def self.included(base)
       end
     end
   end
-end
-
-
+end

lib/i18n_extraction/handlebars_extractor.rb → gems/i18n_extraction/lib/i18n_extraction/handlebars_extractor.rb

@@ -1,5 +1,3 @@
-require 'lib/i18n_extraction/abstract_extractor'
-
 module I18nExtraction
   class HandlebarsExtractor
     include AbstractExtractor
@@ -14,7 +12,7 @@ class HandlebarsExtractor
       \}\}
     /x
     I18N_CALL = /
-      #{I18N_CALL_START}
+    #{I18N_CALL_START}
       (?<content> .*?)
       \{\{\/t\}\}
     /mx
@@ -39,7 +37,7 @@ def process(source, scope)
 
     def scan(source, options={})
       options = {
-        :method => :scan
+          :method => :scan
       }.merge(options)
 
       method = options[:method]
@@ -67,10 +65,10 @@ def scan(source, options={})
         content.gsub!(/\s+/, ' ')
         content.strip!
         yield :key => key,
-              :value => content,
-              :options => opts,
-              :wrappers => wrappers,
-              :line_number => line_number
+            :value => content,
+            :options => opts,
+            :wrappers => wrappers,
+            :line_number => line_number
       end
       raise "possibly unterminated #t call (line #{block_line_numbers.shift} or earlier)" unless block_line_numbers.empty?
       result
@@ -107,7 +105,7 @@ def extract_wrappers!(source)
 
     def balanced_tags?(open, close)
       open.scan(TAG_START).map { |tag| tag.match(TAG_NAME).to_s } ==
-      close.scan(TAG_END).map { |tag| tag.match(TAG_NAME).to_s }.reverse
+          close.scan(TAG_END).map { |tag| tag.match(TAG_NAME).to_s }.reverse
     end
 
     def check_html(source, base_line_number)

lib/i18n_extraction/js_extractor.rb → gems/i18n_extraction/lib/i18n_extraction/js_extractor.rb

@@ -1,5 +1,3 @@
-require 'lib/i18n_extraction/abstract_extractor'
-
 module I18nExtraction
   class JsExtractor
     include AbstractExtractor
@@ -55,8 +53,8 @@ class JsExtractor
     I18N_CALL_START = /I18n\.(t|translate|beforeLabel)\(/
     I18N_KEY_OR_SIMPLE_EXPRESSION = /(#{I18N_KEY}|([\w\.]+|\(['"][\w.]+['"]\))+)/
     I18N_CALL = /
-      #{I18N_CALL_START}
-        #{I18N_KEY_OR_SIMPLE_EXPRESSION}
+    #{I18N_CALL_START}
+    #{I18N_KEY_OR_SIMPLE_EXPRESSION}
         (,\s*
           ( #{STRING_CONCATENATION} | #{REALLY_SIMPLE_HASH_LITERAL} ) # default
           (,\s*
@@ -93,7 +91,7 @@ def find_matches(source, start_pattern, full_pattern=start_pattern, options = {}
         end
       end
       matches = []
-      source.scan(full_pattern){ |args| matches << [$&] + args }
+      source.scan(full_pattern) { |args| matches << [$&] + args }
       raise "expected/actual mismatch (probably a bug)" if expected.size < matches.size
       expected.each_index do |i|
         expected_string = expected[i].first.strip
@@ -280,4 +278,4 @@ def process_literal(string)
       instance_eval(string.gsub(/(^|[^\\])#/, '\1\\#'))
     end
   end
-end
+end

lib/i18n_extraction/ruby_extractor.rb → gems/i18n_extraction/lib/i18n_extraction/ruby_extractor.rb

@@ -1,8 +1,7 @@
-require 'lib/i18n_extraction/abstract_extractor'
-
 module I18nExtraction
   class RubyExtractor < SexpProcessor
     include AbstractExtractor
+
     attr_accessor :in_html_view
 
     def process_defn(exp)
@@ -69,27 +68,27 @@ def process_translate_call(receiver, method, args)
       default = process_default_translation(args.shift, key)
 
       options = if args.first.is_a?(Sexp)
-        if method == :jt
-          if args.first.sexp_type != :str
-            raise "jt options must be a javascript string: #{key.inspect} on line #{line}"
-          end
-          str = args.shift.last
-          str.scan(/['"]?(\w+)['"]?:/).flatten.map(&:to_sym)
-        else
-          if args.first.sexp_type != :hash
-            raise "translate options must be a hash: #{key.inspect} on line #{line}"
-          end
-          hash = args.shift
-          hash.shift
-          (0...(hash.size/2)).map{ |i|
-            process hash[i * 2 + 1]
-            raise "option keys must be strings or symbols on line #{line}" unless [:lit, :str].include?(hash[i * 2].sexp_type)
-            hash[i * 2].last.to_sym
-          }
-        end
-      else
-        []
-      end
+                  if method == :jt
+                    if args.first.sexp_type != :str
+                      raise "jt options must be a javascript string: #{key.inspect} on line #{line}"
+                    end
+                    str = args.shift.last
+                    str.scan(/['"]?(\w+)['"]?:/).flatten.map(&:to_sym)
+                  else
+                    if args.first.sexp_type != :hash
+                      raise "translate options must be a hash: #{key.inspect} on line #{line}"
+                    end
+                    hash = args.shift
+                    hash.shift
+                    (0...(hash.size/2)).map { |i|
+                      process hash[i * 2 + 1]
+                      raise "option keys must be strings or symbols on line #{line}" unless [:lit, :str].include?(hash[i * 2].sexp_type)
+                      hash[i * 2].last.to_sym
+                    }
+                  end
+                else
+                  []
+                end
 
       # single word count/pluralization fu
       if default.is_a?(String) && default =~ /\A[\w\-]+\z/ && options.include?(:count)
@@ -121,12 +120,12 @@ def process_label_call(receiver, method, args)
       inferred = false
       default = nil
       key_arg = if args.size == 1 || args[1] && args[1].is_a?(Sexp) && args[1].sexp_type == :hash
-        inferred = true
-        args.shift
-      elsif args[1].is_a?(Sexp)
-        args.shift
-        args.shift
-      end
+                  inferred = true
+                  args.shift
+                elsif args[1].is_a?(Sexp)
+                  args.shift
+                  args.shift
+                end
       if args.first.is_a?(Sexp) && args.first.sexp_type == :hash
         hash_args = args.shift
         hash_args.shift
@@ -175,18 +174,18 @@ def process_default_translation(exp, key)
       raise "invalid en default #{exp.inspect}" unless exp.is_a?(Sexp)
       if exp.sexp_type == :hash
         exp.shift
-        hash = Hash[*exp.map{ |e| process_possible_string_concat(e, :allow_symbols => true) }]
+        hash = Hash[*exp.map { |e| process_possible_string_concat(e, :allow_symbols => true) }]
         pluralization_keys = hash.keys
         if (pluralization_keys - allowed_pluralization_keys).size > 0
           raise "invalid :count sub-key(s) #{exp.inspect} on line #{exp.line}"
         elsif required_pluralization_keys & pluralization_keys != required_pluralization_keys
           raise "not all required :count sub-key(s) provided on line #{exp.line} (expected #{required_pluralization_keys.join(', ')})"
-        elsif hash.values.any?{ |v| !v.is_a?(String) }
+        elsif hash.values.any? { |v| !v.is_a?(String) }
           raise "invalid en count default(s) #{exp.inspect} on line #{exp.line}"
         end
         hash
       else
-        process_possible_string_concat(exp, :top_level_error => lambda{ |exp| "invalid en default #{exp.inspect} on line #{exp.line}" })
+        process_possible_string_concat(exp, :top_level_error => lambda { |exp| "invalid en default #{exp.inspect} on line #{exp.line}" })
       end
     rescue
       raise "#{$!} (#{key.inspect})"
@@ -204,4 +203,4 @@ def process_possible_string_concat(exp, options={})
       end
     end
   end
-end
+end

gems/i18n_extraction/spec/i18n_extraction/handlebars_extractor_spec.rb

@@ -0,0 +1,109 @@
+#
+# Copyright (C) 2011 Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'spec_helper'
+
+module I18nExtraction
+
+  describe HandlebarsExtractor do
+    def extract(source, scope = 'asdf', options = {})
+      scope_results = scope && (options.has_key?(:scope_results) ? options.delete(:scope_results) : true)
+
+      extractor = HandlebarsExtractor.new
+      extractor.process(source, scope)
+      (scope_results ?
+          scope.split(/\./).inject(extractor.translations) { |hash, s| hash[s] } :
+          extractor.translations) || {}
+    end
+
+    context "keys" do
+      it "should allow valid string keys" do
+        extract('{{#t "foo"}}Foo{{/t}}').should eql({'foo' => "Foo"})
+      end
+
+      it "should disallow everything else" do
+        lambda { extract '{{#t "foo foo"}}Foo{{/t}}' }.should raise_error 'invalid translation key "foo foo" on line 1'
+      end
+    end
+
+    context "well-formed-ness" do
+      it "should make sure all #t calls are closed" do
+        lambda { extract "{{#t \"foo\"}}Foo{{/t}}\n{{#t \"bar\"}}...\nruh-roh\n" }.should raise_error /possibly unterminated #t call \(line 2/
+      end
+    end
+
+    context "values" do
+      it "should strip extraneous whitespace" do
+        extract("{{#t \"foo\"}}\t Foo\n foo\r\n\ffoo!!! {{/t}}").should eql({'foo' => 'Foo foo foo!!!'})
+      end
+    end
+
+    context "placeholders" do
+      it "should allow simple placeholders" do
+        extract('{{#t "foo"}}Hello {{user.name}}{{/t}}').should eql({'foo' => 'Hello %{user.name}'})
+      end
+
+      it "should disallow helpers or anything else" do
+        lambda { extract '{{#t "foo"}}Hello {{call a helper}}{{/t}}' }.should raise_error 'helpers may not be used inside #t calls (line 1)'
+      end
+    end
+
+    context "wrappers" do
+      it "should infer wrappers" do
+        extract('{{#t "foo"}}Be sure to <a href="{{url}}">log in</a>. <b>Don\'t</b> you <b>dare</b> forget!!!{{/t}}').should eql({'foo' => 'Be sure to *log in*. **Don\'t** you **dare** forget!!!'})
+      end
+
+      it "should not infer wrappers from unbalanced tags" do
+        lambda { extract '{{#t "foo"}}you are <b><i>so cool</i></strong>{{/t}}' }.should raise_error 'translation contains un-wrapper-ed markup (line 1). hint: use a placeholder, or balance your markup'
+      end
+
+      it "should allow empty tags on either side of the wrapper" do
+        extract('{{#t "bar"}}you can <button><i class="icon-email"></i>send an email</button>{{/t}}').should eql({'bar' => 'you can *send an email*'})
+        extract('{{#t "baz"}}this is <b>so cool!<img /></b>{{/t}}').should eql({'baz' => 'this is *so cool!*'})
+      end
+
+      it "should disallow any un-wrapper-ed html" do
+        lambda { extract '{{#t "foo"}}check out this pic: <img src="pic.gif">{{/t}}' }.should raise_error 'translation contains un-wrapper-ed markup (line 1). hint: use a placeholder, or balance your markup'
+      end
+    end
+
+    context "scoping" do
+      it "should auto-scope relative keys to the current scope" do
+        extract('{{#t "foo"}}Foo{{/t}}', 'asdf', :scope_results => false).should eql({'asdf' => {'foo' => "Foo"}})
+      end
+
+      it "should not auto-scope absolute keys" do
+        extract('{{#t "#foo"}}Foo{{/t}}', 'asdf', :scope_results => false).should eql({'foo' => "Foo"})
+      end
+    end
+
+    context "collisions" do
+      it "should not let you reuse a key" do
+        lambda { extract '{{#t "foo"}}Foo{{/t}}{{#t "foo"}}foo{{/t}}' }.should raise_error 'cannot reuse key "asdf.foo"'
+      end
+
+      it "should not let you use a scope as a key" do
+        lambda { extract '{{#t "foo.bar"}}bar{{/t}}{{#t "foo"}}foo{{/t}}' }.should raise_error '"asdf.foo" used as both a scope and a key'
+      end
+
+      it "should not let you use a key as a scope" do
+        lambda { extract '{{#t "foo"}}foo{{/t}}{{#t "foo.bar"}}bar{{/t}}' }.should raise_error '"asdf.foo" used as both a scope and a key'
+      end
+    end
+  end
+end