Add requirements.txt for python 3

Also:
* Install-by-source instructions is for python 3
* Adds initial python2->3 upgrade instructions
* Initial adjustments to ckan/common.py for import in python3

I'm keen to do 'trunk based development' for python 3 work now. So python 3 is the default in the docs and requirements.txt. I realize that the devs who use master for other work might be a bit surprised/disrupted by these changes. However I think it is worth it because the focus now needs to be python3, and it's better to switch master's default now rather than just before release.

.circleci/config.yml

@@ -42,7 +42,7 @@ jobs:
 
           # Python Dependencies
           pip install -r requirement-setuptools.txt
-          pip install -r requirements.txt
+          pip install -r requirements-py2.txt
           pip install -r dev-requirements.txt
           python setup.py develop
 

Dockerfile

@@ -50,7 +50,7 @@ RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \
 ADD . $CKAN_VENV/src/ckan/
 RUN ckan-pip install -U pip && \
     ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \
-    ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt && \
+    ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements-py2.txt && \
     ckan-pip install -e $CKAN_VENV/src/ckan/ && \
     ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
     cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \

bin/travis-install-dependencies

@@ -30,7 +30,7 @@ sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh
 
 export PIP_USE_MIRRORS=true
 pip install -r requirement-setuptools.txt --allow-all-external
-pip install -r requirements.txt --allow-all-external
+pip install -r requirements-py2.txt --allow-all-external
 pip install -r dev-requirements.txt --allow-all-external
 
 python setup.py develop

ckan/common.py

@@ -11,20 +11,21 @@
 from collections import MutableMapping
 
 import flask
-import pylons
 import six
 
 from werkzeug.local import Local, LocalProxy
 
 from flask_babel import (gettext as flask_ugettext,
                          ngettext as flask_ungettext)
-from pylons.i18n import (ugettext as pylons_ugettext,
-                         ungettext as pylons_ungettext)
-
-from pylons import response
 
 import simplejson as json
 
+if six.PY2:
+    import pylons
+    from pylons.i18n import (ugettext as pylons_ugettext,
+                             ungettext as pylons_ungettext)
+    from pylons import response
+
 current_app = flask.current_app
 
 try:
@@ -38,6 +39,8 @@ def is_flask_request():
     A centralized way to determine whether we are in the context of a
     request being served by Flask or Pylons
     '''
+    if six.PY3:
+        return True
     try:
         pylons.request.environ
         pylons_request_available = True

doc/maintaining/installing/install-from-source.rst

@@ -22,7 +22,14 @@ work on CKAN.
 If you're using a Debian-based operating system (such as Ubuntu) install the
 required packages with this command::
 
-    sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server
+    sudo apt-get install python3-dev postgresql libpq-dev python3-pip python3-venv git-core solr-jetty openjdk-8-jdk redis-server
+
+.. note::
+
+    For Python 2 (deprecated, but compatible with CKAN 2.9 and earlier), do
+    this instead:
+
+        sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server
 
 If you're not using a Debian-based operating system, find the best way to
 install the following packages on your operating system (see
@@ -32,16 +39,16 @@ wiki page for help):
 =====================  ===============================================
 Package                Description
 =====================  ===============================================
-Python                 `The Python programming language, v2.7 <http://www.python.org/getit/>`_
-|postgres|             `The PostgreSQL database system, v9.3 or newer <http://www.postgresql.org/download/>`_
+Python                 `The Python programming language, v3.6 or newer (or v2.7) <https://www.python.org/getit/>`_
+|postgres|             `The PostgreSQL database system, v9.3 or newer <https://www.postgresql.org/docs/9.5/libpq.html>`_
 libpq                  `The C programmer's interface to PostgreSQL <http://www.postgresql.org/docs/8.1/static/libpq.html>`_
-pip                    `A tool for installing and managing Python packages <http://www.pip-installer.org>`_
-virtualenv             `The virtual Python environment builder <http://www.virtualenv.org>`_
-Git                    `A distributed version control system <http://book.git-scm.com/2_installing_git.html>`_
-Apache Solr            `A search platform <http://lucene.apache.org/solr>`_
-Jetty                  `An HTTP server <http://www.eclipse.org/jetty/>`_ (used for Solr).
-OpenJDK JDK            `The Java Development Kit <http://openjdk.java.net/install/>`_ (used by Jetty)
-Redis                  `An in-memory data structure store <http://redis.io/>`_
+pip                    `A tool for installing and managing Python packages <https://pip.pypa.io/en/stable/>`_
+python3-venv           `The Python3 virtual environment builder (or for Python 2 use 'virtualenv' instead) <https://virtualenv.pypa.io/en/latest/>`_
+Git                    `A distributed version control system <https://git-scm.com/book/en/v2/Getting-Started-Installing-Git>`_
+Apache Solr            `A search platform <https://lucene.apache.org/solr/>`_
+Jetty                  `An HTTP server <https://www.eclipse.org/jetty/>`_ (used for Solr).
+OpenJDK JDK            `The Java Development Kit <https://openjdk.java.net/install/>`_ (used by Jetty)
+Redis                  `An in-memory data structure store <https://redis.io/>`_
 =====================  ===============================================
 
 
@@ -66,28 +73,16 @@ Redis                  `An in-memory data structure store <http://redis.io/>`_
      mkdir -p ~/ckan/etc
      sudo ln -s ~/ckan/etc |config_parent_dir|
 
-a. Create a Python `virtual environment <http://www.virtualenv.org>`_
+a. Create a Python `virtual environment <https://virtualenv.pypa.io/en/latest/>`_
    (virtualenv) to install CKAN into, and activate it:
 
    .. parsed-literal::
 
        sudo mkdir -p |virtualenv|
        sudo chown \`whoami\` |virtualenv|
-       virtualenv --no-site-packages |virtualenv|
+       python3 -m venv |virtualenv|
        |activate|
 
-.. note::
-
-    If your system uses Python3 by default (e.g. Ubuntu 18.04) make sure to create
-    the virtualenv using the Python2.7 executable with the ``--python`` option:
-
-    .. parsed-literal::
-
-        sudo mkdir -p |virtualenv|
-        sudo chown \`whoami\` |virtualenv|
-        virtualenv --python=/usr/bin/python2.7 --no-site-packages |virtualenv|
-        |activate|
-
 .. important::
 
    The final command above activates your virtualenv. The virtualenv has to
@@ -105,11 +100,21 @@ a. Create a Python `virtual environment <http://www.virtualenv.org>`_
 
        |activate|
 
-b. Install the recommended ``setuptools`` version:
+.. note::
+
+    For Python 2 then replace the `python3 -m venv` command with:
+
+    .. parsed-literal::
+
+        virtualenv --python=/usr/bin/python2.7 --no-site-packages |virtualenv|
+        |activate|
+
+b. Install the recommended ``setuptools`` version and up-to-date pip:
 
    .. parsed-literal::
 
        pip install setuptools==\ |min_setuptools_version|
+       pip install --upgrade pip
 
 c. Install the CKAN source code into your virtualenv.
 
@@ -140,6 +145,10 @@ d. Install the Python modules that CKAN requires into your virtualenv:
 
        pip install -r |virtualenv|/src/ckan/requirements.txt
 
+.. note::
+
+    For Python 2 adjust the filename to: `requirements-py2.txt`
+
 e. Deactivate and reactivate your virtualenv, to make sure you're using the
    virtualenv's copies of commands like ``paster`` rather than any system-wide
    installed copies:

doc/maintaining/upgrading/index.rst

@@ -128,3 +128,4 @@ appropriate one of these documents:
     upgrade-package-to-minor-release
     upgrade-source
     upgrade-postgres
+    upgrade-to-python3

doc/maintaining/upgrading/upgrade-to-python3.rst

@@ -0,0 +1,35 @@
+==================================================
+Upgrading a CKAN install from Python 2 to Python 3
+==================================================
+
+These instructions describe how to upgrade a source install of CKAN 2.9 from
+Python 2 to Python 3, which is necessary because Python 2 is end of life, as of
+January 1st, 2020.
+
+Preparation
+-----------
+
+* Backup your CKAN source, virtualenv and databases, just in case.
+* Upgrade to CKAN 2.9, if you've not done already.
+
+Upgrade
+-------
+
+You'll probably need to deactivate your existing virtual environment::
+
+    deactivate
+
+The existing setup has the virtual environment here: |virtualenv|
+and the CKAN source code underneath in `/usr/lib/ckan/default/src`. We'll move
+that aside in case we need to roll-back:
+
+   .. parsed-literal::
+
+    sudo mv |virtualenv| /usr/lib/ckan/py2
+
+From this doc: :doc:`/maintaining/installing/install-from-source` you need to
+do these sections:
+
+* 1. Install the required packages
+* 2. Install CKAN into a Python virtual environment
+* 6. Link to who.ini

requirements-py2.in

@@ -0,0 +1,40 @@
+# The file contains the direct ckan requirements.
+# Use pip-compile to create a requirements.txt file from this
+alembic==1.0.0
+Babel==2.3.4
+bleach==3.0.2
+click==6.7
+fanstatic==0.12
+Flask==1.1.1
+Flask-Babel==0.11.2
+Jinja2==2.10.1
+Markdown==2.6.7
+passlib==1.6.5
+paste==1.7.5.1
+PasteScript==2.0.2
+polib==1.0.7
+psycopg2==2.8.2
+python-magic==0.4.15
+pysolr==3.6.0
+Pylons==0.9.7
+python-dateutil>=1.5.0
+pytz==2016.7
+PyUtilib==5.7.1
+pyyaml  # needed by webassets. latest should be fine.
+repoze.who-friendlyform==1.0.8
+repoze.who==2.3
+requests==2.22.0
+Routes==1.13
+rq==1.0
+simplejson==3.10.0
+sqlalchemy-migrate==0.12.0
+SQLAlchemy==1.3.5
+sqlparse==0.2.2
+tzlocal==1.3
+unicodecsv>=0.9
+webassets==0.12.1
+WebHelpers==1.3
+WebOb==1.0.8
+WebTest==1.4.3  # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8
+werkzeug==0.15.5
+zope.interface==4.3.2

requirements-py2.txt

@@ -0,0 +1,66 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements=py2.txt requirements-py2.in
+#
+alembic==1.0.0
+babel==2.3.4
+beaker==1.10.1            # via pylons
+bleach==3.0.2
+certifi==2019.3.9         # via requests
+chardet==3.0.4            # via requests
+click==6.7
+decorator==4.4.0          # via pylons, sqlalchemy-migrate
+fanstatic==0.12
+flask-babel==0.11.2
+Flask==1.1.1
+formencode==1.3.1         # via pylons
+funcsigs==1.0.2           # via beaker
+idna==2.8                 # via requests
+itsdangerous==1.1.0       # via flask
+jinja2==2.10.1
+mako==1.0.9               # via alembic, pylons
+markdown==2.6.7
+markupsafe==1.1.1         # via jinja2, mako, webhelpers
+nose==1.3.7               # via pylons
+passlib==1.6.5
+paste==1.7.5.1
+pastedeploy==2.0.1        # via pastescript, pylons
+pastescript==2.0.2
+pbr==5.2.0                # via sqlalchemy-migrate
+polib==1.0.7
+psycopg2==2.8.2
+pygments==2.3.1           # via weberror
+pylons==0.9.7
+pysolr==3.6.0
+python-dateutil==2.8.0
+python-editor==1.0.4      # via alembic
+python-magic==0.4.15
+pytz==2016.7
+PyUtilib==5.7.1
+pyyaml==5.1
+redis==3.2.1              # via rq
+repoze.lru==0.7           # via routes
+repoze.who-friendlyform==1.0.8
+repoze.who==2.3
+requests==2.22.0
+routes==1.13
+rq==1.0
+simplejson==3.10.0
+six==1.12.0               # via bleach, pastescript, python-dateutil, pyutilib.component.core, sqlalchemy-migrate
+sqlalchemy-migrate==0.12.0
+sqlalchemy==1.3.5
+sqlparse==0.2.2
+tempita==0.5.2            # via pylons, sqlalchemy-migrate, weberror
+tzlocal==1.3
+unicodecsv==0.14.1
+urllib3==1.25.2           # via requests
+webassets==0.12.1
+webencodings==0.5.1       # via bleach
+weberror==0.13.1          # via pylons
+webhelpers==1.3
+webob==1.0.8
+webtest==1.4.3
+werkzeug==0.15.5
+zope.interface==4.3.2

requirements.in

@@ -1,4 +1,4 @@
-# The file contains the direct ckan requirements.
+# The file contains the direct ckan requirements (python3).
 # Use pip-compile to create a requirements.txt file from this
 alembic==1.0.0
 Babel==2.3.4
@@ -16,12 +16,12 @@ polib==1.0.7
 psycopg2==2.8.2
 python-magic==0.4.15
 pysolr==3.6.0
-Pylons==0.9.7
+# Pylons==0.9.7                    - not python3 compatible
 python-dateutil>=1.5.0
 pytz==2016.7
 PyUtilib==5.7.1
 pyyaml  # needed by webassets. latest should be fine.
-repoze.who-friendlyform==1.0.8
+# repoze.who-friendlyform==1.0.8   - not python3 compatible
 repoze.who==2.3
 requests==2.22.0
 Routes==1.13
@@ -33,8 +33,8 @@ sqlparse==0.2.2
 tzlocal==1.3
 unicodecsv>=0.9
 webassets==0.12.1
-WebHelpers==1.3
-WebOb==1.0.8
-WebTest==1.4.3  # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8
+# WebHelpers==1.3                  - not python3 compatible
+# WebOb==1.0.8                     - pylons dependency
+WebTest==1.4.3
 werkzeug==0.15.5
 zope.interface==4.3.2