[controllers,lib]: Lots of i18n fixes: ckan#1374 switch to english, c…

…kan#1417 browser detection, ckan#1388 etags on home page.

ckan/config/deployment.ini_tmpl

@@ -179,6 +179,16 @@ ckan.backup_dir = %(here)s/backup
 #ckan.recaptcha.publickey = 
 #ckan.recaptcha.privatekey = 
 
+# Locale/languages
+# TODO: Figure out a nicer way to get this. From the .ini? 
+# Order these by number of people speaking it in Europe:
+# http://en.wikipedia.org/wiki/Languages_of_the_European_Union#Knowledge
+# (or there abouts)
+ckan.locale_default = en
+#ckan.locales_offered = 
+ckan.locale_order = en de fr it es pl ru nl sv no cs_CZ hu pt_BR fi bg ca sq
+ckan.locales_filtered_out = el
+
 # Logging configuration
 [loggers]
 keys = root, ckan, ckanext

ckan/controllers/home.py

@@ -2,13 +2,14 @@
 import sys
 
 from pylons import cache, config
+from pylons.i18n import set_lang
 from genshi.template import NewTextTemplate
 import sqlalchemy.exc
 
 from ckan.authz import Authorizer
 from ckan.logic import NotAuthorized
 from ckan.logic import check_access, get_action
-from ckan.i18n import set_session_locale, set_lang
+from ckan.lib.i18n import set_session_locale, get_lang
 from ckan.lib.search import query_for, QueryOptions, SearchError
 from ckan.lib.cache import proxy_cache, get_cache_expires
 from ckan.lib.base import *
@@ -41,16 +42,11 @@ def __before__(self, action, **env):
 
 
     @staticmethod
-    def _home_cache_key(latest_revision_id=None):
-        '''Calculate the etag cache key for the home page. If you have
-        the latest revision id then supply it as a param.'''
-        if not latest_revision_id:
-            latest_revision_id = model.repo.youngest_revision().id
+    def _home_cache_key():
+        '''Calculate the etag cache key for the home page.'''
         user_name = c.user
-        if latest_revision_id:
-            cache_key = str(hash((latest_revision_id, user_name)))
-        else:
-            cache_key = 'fresh-install'
+        language = get_lang()
+        cache_key = str(hash((user_name, language)))
         return cache_key
 
     @proxy_cache(expires=cache_expires)
@@ -86,6 +82,11 @@ def locale(self):
                 abort(400, _('Invalid language specified'))
             try:
                 set_lang(locale)
+                # NOTE: When translating this string, substitute the word
+                # 'English' for the language being translated into.
+                # We do it this way because some Babel locales don't contain
+                # a display_name!
+                # e.g. babel.Locale.parse('no').get_display_name() returns None
                 h.flash_notice(_("Language has been set to: English"))
             except:
                 h.flash_notice("Language has been set to: English")

ckan/controllers/package.py

@@ -7,7 +7,7 @@
 from sqlalchemy.orm import eagerload_all
 import genshi
 from pylons import config, cache
-from pylons.i18n import get_lang, _
+from pylons.i18n import _
 from autoneg.accept import negotiate
 from babel.dates import format_date, format_datetime, format_time
 
@@ -25,6 +25,7 @@
 from ckan.logic import NotFound, NotAuthorized, ValidationError
 from ckan.logic import tuplize_dict, clean_dict, parse_params, flatten_to_string_key
 from ckan.lib.dictization import table_dictize
+from ckan.lib.i18n import get_lang
 import ckan.forms
 import ckan.authz
 import ckan.rating
@@ -179,7 +180,8 @@ def pager_url(q=None, page=None):
     def _pkg_cache_key(pkg):
         # note: we need pkg.id in addition to pkg.revision.id because a
         # revision may have more than one package in it.
-        return str(hash((pkg.id, pkg.latest_related_revision.id, c.user, pkg.get_average_rating())))
+        language = get_lang()
+        return str(hash((pkg.id, pkg.latest_related_revision.id, c.user, pkg.get_average_rating(), language)))
 
     @proxy_cache()
     def read(self, id):

ckan/i18n/__init__.py

@@ -1,65 +0,0 @@
-from pylons.i18n import _, add_fallback, get_lang, set_lang, gettext
-from babel import Locale
-
-
-# TODO: Figure out a nicer way to get this. From the .ini? 
-# Order these by number of people speaking it in Europe:
-# http://en.wikipedia.org/wiki/Languages_of_the_European_Union#Knowledge
-# (or there abouts)
-_KNOWN_LOCALES = ['en',
-                  'de',
-                  'fr',
-                  'it',
-                  'es',
-                  'pl',
-                  'ru',
-                  'nl',
-                  'sv', # Swedish
-                  'no',
-#                  'el', # Greek
-                  'cs_CZ',
-                  'hu',
-                  'pt_BR',
-                  'fi', 
-                  'bg',
-                  'ca',
-                  'sq', 
-                  ]
-
-def get_available_locales():
-    return map(Locale.parse, _KNOWN_LOCALES)
-
-def get_default_locale():
-    from pylons import config
-    return Locale.parse(config.get('ckan.locale')) or \
-            Locale.parse('en')
-
-def set_session_locale(locale):
-    if locale not in _KNOWN_LOCALES:
-        raise ValueError
-    from pylons import session
-    session['locale'] = locale
-    session.save()
-
-def handle_request(request, tmpl_context):
-    from pylons import session
-
-    tmpl_context.language = locale = None
-    if 'locale' in session:
-        locale = Locale.parse(session.get('locale'))
-    else:
-        requested = [l.replace('-', '_') for l in request.languages]
-        locale = Locale.parse(Locale.negotiate(_KNOWN_LOCALES, requested))
-
-    if locale is None:
-        locale = get_default_locale()
-
-    options = [str(locale), locale.language, str(get_default_locale()),
-        get_default_locale().language]
-    for language in options:
-        try:
-            set_lang(language) 
-            tmpl_context.language = language
-        except: pass
-
-

ckan/i18n/ckan.pot

@@ -214,6 +214,7 @@ msgstr ""
 msgid "Invalid language specified"
 msgstr ""
 
+#. NOTE: Substitute 'English' for the language being translated into.
 #: ckan/controllers/home.py:89
 msgid "Language has been set to: English"
 msgstr ""

ckan/lib/base.py

@@ -20,7 +20,7 @@
 
 import ckan
 from ckan import authz
-from ckan import i18n
+from ckan.lib import i18n
 import ckan.lib.helpers as h
 from ckan.plugins import PluginImplementations, IGenshiStreamFilter
 from ckan.lib.helpers import json

ckan/lib/helpers.py

@@ -19,7 +19,7 @@
 from routes import url_for, redirect_to
 from alphabet_paginate import AlphaPage
 from lxml.html import fromstring
-from ckan.i18n import get_available_locales
+from i18n import get_available_locales
 
 
 

ckan/lib/i18n.py

@@ -0,0 +1,201 @@
+import os
+
+import pylons
+from pylons.i18n import _, add_fallback, set_lang, gettext, LanguageError
+from pylons.i18n.translation import _get_translator
+from babel import Locale, localedata
+from babel.core import LOCALE_ALIASES
+
+import ckan.i18n
+
+def singleton(cls):
+    instances = {}
+    def getinstance():
+        if cls not in instances:
+            instances[cls] = cls()
+        return instances[cls]
+    return getinstance
+
+i18n_path = os.path.dirname(ckan.i18n.__file__)
+
+@singleton
+class Locales(object):
+    def __init__(self):
+        from pylons import config
+
+        # Get names of the locales
+        # (must be a better way than scanning for i18n directory?)
+        known_locales = ['en'] + [locale_name for locale_name in os.listdir(i18n_path) \
+                                  if localedata.exists(locale_name)]
+        self._locale_names, self._default_locale_name = self._work_out_locales(
+            known_locales, config)
+        self._locale_objects = map(Locale.parse, self._locale_names)
+        self._default_locale_object = Locale.parse(self._default_locale_name)
+
+        self._aliases = LOCALE_ALIASES
+        self._aliases['pt'] = 'pt_BR' # Default Portuguese language to
+                                     # Brazilian territory, since
+                                     # we don't have a Portuguese territory
+                                     # translation currently.
+
+    def _work_out_locales(self, known_locales, config_dict):
+        '''Work out the locale_names to offer to the user and the default locale.
+        All locales in this method are strings, not Locale objects.'''
+        # pass in a config_dict rather than ckan.config to make this testable
+
+        # Get default locale
+        assert not config_dict.get('lang'), \
+               '"lang" config option not supported - please use ckan.locale_default instead.'
+        default_locale = config_dict.get('ckan.locale_default') or \
+                         config_dict.get('ckan.locale') or \
+                         None # in this case, set it later on
+        if default_locale:
+            assert default_locale in known_locales
+
+        # Filter and reorder locales by config options
+        def get_locales_in_config_option(config_option):
+            locales_ = config_dict.get(config_option, '').split()
+            if locales_:
+                unknown_locales = set(locales_) - set(known_locales)
+                assert not unknown_locales, \
+                       'Bad config option %r - locales not found: %s' % \
+                       (config_option, unknown_locales)
+            return locales_
+        only_locales_offered = get_locales_in_config_option('ckan.locales_offered')
+        if only_locales_offered:
+            locales = only_locales_offered
+        else:
+            locales = known_locales
+
+        def move_locale_to_start_of_list(locale_):
+            if locale_ not in locales:
+                raise ValueError('Cannot find locale "%s" in locales offered.' % locale_)
+            locales.pop(locales.index(locale_))
+            locales.insert(0, locale_)
+
+        locales_filtered_out = get_locales_in_config_option('ckan.locales_filtered_out')
+        for locale in locales_filtered_out:
+            try:
+                locales.pop(locales.index(locale))
+            except ValueError, e:
+                raise ValueError('Could not filter out locale "%s" from offered locale list "%s": %s') % \
+                      (locale, locales, e)
+
+        locale_order = get_locales_in_config_option('ckan.locale_order')
+        if locale_order:
+            for locale in locale_order[::-1]:
+                # bring locale_name to the front
+                try:
+                    move_locale_to_start_of_list(locale)
+                except ValueError, e:
+                    raise ValueError('Could not process ckan.locale_order options "%s" for offered locale list "%s": %s' % \
+                                     (locale_order, locales, e))
+        elif default_locale:
+            if default_locale not in locales:
+                raise ValueError('Default locale "%s" is not amongst locales offered: %s' % \
+                                 (default_locale, locales))
+            # move the default locale to the start of the list
+            try:
+                move_locale_to_start_of_list(default_locale)
+            except ValueError, e:
+                raise ValueError('Could not move default locale "%s" to the start ofthe list of offered locales "%s": %s' % \
+                                 (default_locale, locales, e))
+
+        assert locales
+
+        if not default_locale:
+            default_locale = locales[0]
+        assert default_locale in locales
+
+        return locales, default_locale
+
+    def get_available_locales(self):
+        '''Returns a list of the locale objects for which translations are
+        available.'''
+        return self._locale_objects
+
+    def get_available_locale_names(self):
+        '''Returns a list of the locale strings for which translations are
+        available.'''
+        return self._locale_names
+
+    def get_default_locale(self):
+        '''Returns the default locale/language as specified in the CKAN
+        config. It is a locale object.'''
+        return self._default_locale_object
+
+    def get_aliases(self):
+        '''Returns a mapping of language aliases, like the Babel LOCALE_ALIASES
+        but with hacks for specific CKAN issues.'''
+        return self._aliases
+
+    def negotiate_known_locale(self, preferred_locales):
+        '''Given a list of preferred locales, this method returns the best
+        match locale object from the known ones.'''
+        assert isinstance(preferred_locales, (tuple, list))
+        preferred_locales = [str(l).replace('-', '_') for l in preferred_locales]
+        return Locale.parse(Locale.negotiate(preferred_locales,
+                                             self.get_available_locale_names(),
+                                             aliases=self.get_aliases()
+                                             ))
+
+def get_available_locales():
+    return Locales().get_available_locales()
+
+def set_session_locale(locale):
+    if locale not in get_available_locales():
+        raise ValueError
+    from pylons import session
+    session['locale'] = locale
+    session.save()
+
+def handle_request(request, tmpl_context):
+    from pylons import session
+
+    # Work out what language to show the page in.
+    locales = [] # Locale objects. Ordered highest preference first.
+    tmpl_context.language = None
+    if session.get('locale'):
+        # First look for locale saved in the session (by home controller)
+        locales.append(Locale.parse(session.get('locale')))
+    else:
+        # Next try languages in the HTTP_ACCEPT_LANGUAGE header
+        locales.append(Locales().negotiate_known_locale(request.languages))
+
+    # Next try the default locale in the CKAN config file
+    locales.append(Locales().get_default_locale())
+
+    locale = set_lang_list(locales)
+    tmpl_context.language = locale.language
+    return locale
+
+def set_lang_list(locales):
+    '''Takes a list of locales (ordered by reducing preference) and tries
+    to set them in order. If one fails then it puts up a flash message and
+    tries the next.'''
+    import ckan.lib.helpers as h
+    failed_locales = set()
+    for locale in locales:
+        # try locales in order of preference until one works
+        try:
+            if str(locale) == 'en':
+                # There is no language file for English, so if we set_lang
+                # we would get an error. Just don't set_lang and finish.
+                break
+            set_lang(str(locale))
+            break
+        except LanguageError, e:
+            if str(locale) not in failed_locales:
+                h.flash_error('Could not change language to %r: %s' % \
+                              (str(locale), e))
+                failed_locales.add(str(locale))
+    return locale
+
+def get_lang():
+    '''Returns the current language. Based on babel.i18n.get_lang but works
+    when set_lang has not been run (i.e. still in English).'''
+    langs = pylons.i18n.get_lang()
+    if langs:
+        return langs[0]
+    else:
+        return 'en'