Merge branch 'master' of ssh://github.com/okfn/ckan

bin/running_stats.py

@@ -1,3 +1,37 @@
+'''Tool for a script to keep track changes performed on a large number
+of objects.
+
+StatsCount - when you are counting incidences of a small set of outcomes
+StatsList - when you also want to remember an ID associated with each incidence
+
+Examples:
+
+from running_stats import StatsCount
+package_stats = StatsCount()
+for package in packages:
+    if package.enabled:
+        package.delete()
+        package_stats.increment('deleted')
+    else:
+        package_stats.increment('not deleted')    
+print package_stats.report()
+> deleted: 30
+> not deleted: 70
+    
+from running_stats import StatsList
+package_stats = StatsList()
+for package in packages:
+    if package.enabled:
+        package.delete()
+        package_stats.add('deleted', package.name)
+    else:
+        package_stats.add('not deleted' package.name)
+print package_stats.report()
+> deleted: 30 pollution-uk, flood-regions, river-quality, ...
+> not deleted: 70 spending-bristol, ... 
+
+'''
+
 import copy
 
 class StatsCount(dict):

ckan/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '1.6.1b'
+__version__ = '1.8a'
 __description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
 __long_description__ = \
 '''CKAN software provides a hub for datasets. The flagship site running CKAN 

ckan/ckan_nose_plugin.py

@@ -30,7 +30,6 @@ def startContext(self, ctx):
             # init_db is run at the start of every class because
             # when you use an in-memory sqlite db, it appears that
             # the db is destroyed after every test when you Session.Remove().
-            model.repo.init_db()
 
             ## This is to make sure the configuration is run again.
             ## Plugins use configure to make their own tables and they
@@ -40,6 +39,7 @@ def startContext(self, ctx):
             for plugin in PluginImplementations(IConfigurable):
                 plugin.configure(config)
 
+            model.repo.init_db()
 
     def options(self, parser, env):
         parser.add_option(

ckan/config/deployment.ini_tmpl

@@ -99,7 +99,7 @@ ckan.build_search_index_synchronously = true
 ckan.site_title = CKAN
 
 ## Logo image to use on the home page
-ckan.site_logo = /img/logo.png
+ckan.site_logo = /img/logo_64px_wide.png
 
 ## Site tagline / description (used on front page)
 ckan.site_description = 
@@ -165,6 +165,39 @@ ckan.locale_default = en
 ckan.locale_order = en de fr it es pl ru nl sv no cs_CZ hu pt_BR fi bg ca sq sr sr_Latn
 ckan.locales_filtered_out = el ro lt sl
 
+## Atom Feeds
+#
+# Settings for customising the metadata provided in
+# atom feeds.
+#
+# These settings are used to generate the <id> tags for both feeds
+# and entries. The unique <id>s are created following the method
+# outlined in http://www.taguri.org/  ie - they generate tagURIs, as specified
+# in http://tools.ietf.org/html/rfc4151#section-2.1 :
+#
+# <id>tag:thedatahub.org,2012:/feeds/group/933f3857-79fd-4beb-a835-c0349e31ce76</id>
+#
+# Each component has the corresponding settings:
+#
+#   "thedatahub.org" is ckan.feeds.authority_name
+#   "2012"           is ckan.feeds.date
+#
+
+# Leave blank to use the ckan.site_url config value, otherwise set to a
+# domain or email address that you own.  e.g. thedatahub.org or
+# [email protected]
+ckan.feeds.authority_name =
+
+# Pick a date of the form "yyyy[-mm[-dd]]" during which the above domain was
+# owned by you.
+ckan.feeds.date = 2012
+
+# If not set, then the value in `ckan.site_id` is used.
+ckan.feeds.author_name =
+
+# If not set, then the value in `ckan.site_url` is used.
+ckan.feeds.author_link =
+
 ## Webstore
 ## Uncommment to enable datastore
 # ckan.datastore.enabled = 1

ckan/config/environment.py

@@ -1,30 +1,85 @@
 """Pylons environment configuration"""
 import os
-from urlparse import urlparse
 import logging
 import warnings
-
-from paste.deploy.converters import asbool
-
-# Suppress benign warning 'Unbuilt egg for setuptools'
-warnings.simplefilter('ignore', UserWarning)
-
+from urlparse import urlparse
 
 import pylons
+from paste.deploy.converters import asbool
 import sqlalchemy
-
 from pylons import config
-from pylons.i18n.translation import ugettext
 from genshi.template import TemplateLoader
 from genshi.filters.i18n import Translator
 
+import ckan.config.routing as routing
+import ckan.model as model
+import ckan.plugins as p
+import ckan.lib.helpers as h
+import ckan.lib.search as search
 import ckan.lib.app_globals as app_globals
-import ckan.lib.helpers
-from ckan.config.routing import make_map
-from ckan import model
-from ckan import plugins
 
 
+# Suppress benign warning 'Unbuilt egg for setuptools'
+warnings.simplefilter('ignore', UserWarning)
+
+class _Helpers(object):
+    ''' Helper object giving access to template helpers stopping
+    missing functions from causing template exceptions. Useful if
+    templates have helper functions provided by extensions that have
+    not been enabled. '''
+    def __init__(self, helpers, restrict=True):
+        functions = {}
+        allowed = helpers.__allowed_functions__
+        # list of functions due to be depreciated
+        self.depreciated = []
+
+        for helper in dir(helpers):
+            if helper not in allowed:
+                self.depreciated.append(helper)
+                if restrict:
+                    continue
+            functions[helper] = getattr(helpers, helper)
+        self.functions = functions
+
+        # extend helper functions with ones supplied by plugins
+        extra_helpers = []
+        for plugin in p.PluginImplementations(p.ITemplateHelpers):
+            helpers = plugin.get_helpers()
+            for helper in helpers:
+                if helper in extra_helpers:
+                    raise Exception('overwritting extra helper %s' % helper)
+                extra_helpers.append(helper)
+                functions[helper] = helpers[helper]
+        # logging
+        self.log = logging.getLogger('ckan.helpers')
+
+    @classmethod
+    def null_function(cls, *args, **kw):
+        ''' This function is returned if no helper is found. The idea is
+        to try to allow templates to be rendered even if helpers are
+        missing.  Returning the empty string seems to work well.'''
+        return ''
+
+    def __getattr__(self, name):
+        ''' return the function/object requested '''
+        if name in self.functions:
+            if name in self.depreciated:
+                msg = 'Template helper function `%s` is depriciated' % name
+                self.log.warn(msg)
+            return self.functions[name]
+        else:
+            if name in self.depreciated:
+                msg = 'Template helper function `%s` is not available ' \
+                      'as it has been depriciated.\nYou can enable it ' \
+                      'by setting ckan.restrict_template_vars = true ' \
+                      'in your .ini file.' % name
+                self.log.critical(msg)
+            else:
+                msg = 'Helper function `%s` could not be found\n ' \
+                      '(are you missing an extension?)' % name
+                self.log.critical(msg)
+            return self.null_function
+
 
 def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
@@ -64,12 +119,9 @@ def find_controller(self, controller):
     config.init_app(global_conf, app_conf, package='ckan', paths=paths)
 
     # load all CKAN plugins
-    plugins.load_all(config)
-
-    from ckan.plugins import PluginImplementations
-    from ckan.plugins.interfaces import IConfigurer
+    p.load_all(config)
 
-    for plugin in PluginImplementations(IConfigurer):
+    for plugin in p.PluginImplementations(p.IConfigurer):
         # must do update in place as this does not work:
         # config = plugin.update_config(config)
         plugin.update_config(config)
@@ -89,15 +141,19 @@ def find_controller(self, controller):
         config['ckan.site_id'] = ckan_host
 
     # Init SOLR settings and check if the schema is compatible
-    from ckan.lib.search import SolrSettings, check_solr_schema_version
-    SolrSettings.init(config.get('solr_url'),
-                      config.get('solr_user'),
-                      config.get('solr_password'))
-    check_solr_schema_version()
+    #from ckan.lib.search import SolrSettings, check_solr_schema_version
+    search.SolrSettings.init(config.get('solr_url'),
+                             config.get('solr_user'),
+                             config.get('solr_password'))
+    search.check_solr_schema_version()
 
-    config['routes.map'] = make_map()
+    config['routes.map'] = routing.make_map()
     config['pylons.app_globals'] = app_globals.Globals()
-    config['pylons.h'] = ckan.lib.helpers
+
+    # add helper functions
+    restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'false'))
+    helpers = _Helpers(h, restrict_helpers)
+    config['pylons.h'] = helpers
 
     ## redo template setup to use genshi.search_path (so remove std template setup)
     template_paths = [paths['templates'][0]]
@@ -132,16 +188,12 @@ def template_loaded(template):
     ckan_db = os.environ.get('CKAN_DB') 
 
     if ckan_db:
-        engine = sqlalchemy.create_engine(ckan_db)
-    else:
-        engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
+        config['sqlalchemy.url'] = ckan_db
+    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
 
     if not model.meta.engine:
         model.init_model(engine)
 
-    from ckan.plugins import PluginImplementations
-    from ckan.plugins.interfaces import IConfigurable
-
-    for plugin in PluginImplementations(IConfigurable):
+    for plugin in p.PluginImplementations(p.IConfigurable):
         plugin.configure(config)
 

ckan/config/middleware.py

@@ -1,8 +1,11 @@
 """Pylons middleware initialization"""
 import urllib
-import logging 
+import urllib2
+import logging
 import json
+import hashlib
 
+import sqlalchemy as sa
 from beaker.middleware import CacheMiddleware, SessionMiddleware
 from paste.cascade import Cascade
 from paste.registry import RegistryManager
@@ -14,6 +17,7 @@
 from routes.middleware import RoutesMiddleware
 from repoze.who.config import WhoConfig
 from repoze.who.middleware import PluggableAuthenticationMiddleware
+
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 from ckan.lib.i18n import get_locales
@@ -130,6 +134,9 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     if asbool(config.get('ckan.page_cache_enabled')):
         app = PageCacheMiddleware(app, config)
 
+    # Tracking
+    if asbool(config.get('ckan.tracking_enabled', 'false')):
+        app = TrackingMiddleware(app, config)
     return app
 
 class I18nMiddleware(object):
@@ -277,3 +284,40 @@ def _start_response(status, response_headers, exc_info=None):
             pipe.rpush(key, page_string)
             pipe.execute()
         return page
+
+
+class TrackingMiddleware(object):
+
+    def __init__(self, app, config):
+        self.app = app
+        self.engine = sa.create_engine(config.get('sqlalchemy.url'))
+
+
+    def __call__(self, environ, start_response):
+        path = environ['PATH_INFO']
+        if path == '/_tracking':
+            # do the tracking
+            # get the post data
+            payload = environ['wsgi.input'].read()
+            parts = payload.split('&')
+            data = {}
+            for part in parts:
+                k, v = part.split('=')
+                data[k] = urllib2.unquote(v).decode("utf8")
+            start_response('200 OK', [('Content-Type', 'text/html')])
+            # we want a unique anonomized key for each user so that we do
+            # not count multiple clicks from the same user.
+            key = ''.join([
+                environ['HTTP_USER_AGENT'],
+                environ['REMOTE_ADDR'],
+                environ['HTTP_ACCEPT_LANGUAGE'],
+                environ['HTTP_ACCEPT_ENCODING'],
+            ])
+            key = hashlib.md5(key).hexdigest()
+            # store key/data here
+            sql = '''INSERT INTO tracking_raw
+                     (user_key, url, tracking_type)
+                     VALUES (%s, %s, %s)'''
+            self.engine.execute(sql, key, data.get('url'), data.get('type'))
+            return []
+        return self.app(environ, start_response)