diff --git a/.circleci/config.yml b/.circleci/config.yml index 713dc46d8b3..38edecf2a83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ jobs: # Python Dependencies pip install -r requirement-setuptools.txt - pip install -r requirements.txt + pip install -r requirements-py2.txt pip install -r dev-requirements.txt python setup.py develop diff --git a/Dockerfile b/Dockerfile index d19856e20df..6ca2704dd21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ ADD . $CKAN_VENV/src/ckan/ RUN ckan-pip install -U pip && \ ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \ - ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt && \ + ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements-py2.txt && \ ckan-pip install -e $CKAN_VENV/src/ckan/ && \ ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \ diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies index 77514786c2e..adcae8386f2 100755 --- a/bin/travis-install-dependencies +++ b/bin/travis-install-dependencies @@ -30,7 +30,7 @@ sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh export PIP_USE_MIRRORS=true pip install -r requirement-setuptools.txt --allow-all-external -pip install -r requirements.txt --allow-all-external +pip install -r requirements-py2.txt --allow-all-external pip install -r dev-requirements.txt --allow-all-external python setup.py develop diff --git a/ckan/common.py b/ckan/common.py index c1c82bb4398..5c9f61aa8da 100644 --- a/ckan/common.py +++ b/ckan/common.py @@ -11,20 +11,21 @@ from collections import MutableMapping import flask -import pylons import six from werkzeug.local import Local, LocalProxy from flask_babel import (gettext as flask_ugettext, ngettext as flask_ungettext) -from pylons.i18n import (ugettext as pylons_ugettext, - ungettext as pylons_ungettext) - -from pylons import response import simplejson as json +if six.PY2: + import pylons + from pylons.i18n import (ugettext as pylons_ugettext, + ungettext as pylons_ungettext) + from pylons import response + current_app = flask.current_app try: @@ -38,6 +39,8 @@ def is_flask_request(): A centralized way to determine whether we are in the context of a request being served by Flask or Pylons ''' + if six.PY3: + return True try: pylons.request.environ pylons_request_available = True diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index e09b34b885d..cd1fa98accb 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -22,7 +22,14 @@ work on CKAN. If you're using a Debian-based operating system (such as Ubuntu) install the required packages with this command:: - sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server + sudo apt-get install python3-dev postgresql libpq-dev python3-pip python3-venv git-core solr-jetty openjdk-8-jdk redis-server + +.. note:: + + For Python 2 (deprecated, but compatible with CKAN 2.9 and earlier), do + this instead: + + sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server If you're not using a Debian-based operating system, find the best way to install the following packages on your operating system (see @@ -32,16 +39,16 @@ wiki page for help): ===================== =============================================== Package Description ===================== =============================================== -Python `The Python programming language, v2.7 `_ -|postgres| `The PostgreSQL database system, v9.3 or newer `_ +Python `The Python programming language, v3.6 or newer (or v2.7) `_ +|postgres| `The PostgreSQL database system, v9.3 or newer `_ libpq `The C programmer's interface to PostgreSQL `_ -pip `A tool for installing and managing Python packages `_ -virtualenv `The virtual Python environment builder `_ -Git `A distributed version control system `_ -Apache Solr `A search platform `_ -Jetty `An HTTP server `_ (used for Solr). -OpenJDK JDK `The Java Development Kit `_ (used by Jetty) -Redis `An in-memory data structure store `_ +pip `A tool for installing and managing Python packages `_ +python3-venv `The Python3 virtual environment builder (or for Python 2 use 'virtualenv' instead) `_ +Git `A distributed version control system `_ +Apache Solr `A search platform `_ +Jetty `An HTTP server `_ (used for Solr). +OpenJDK JDK `The Java Development Kit `_ (used by Jetty) +Redis `An in-memory data structure store `_ ===================== =============================================== @@ -66,28 +73,16 @@ Redis `An in-memory data structure store `_ mkdir -p ~/ckan/etc sudo ln -s ~/ckan/etc |config_parent_dir| -a. Create a Python `virtual environment `_ +a. Create a Python `virtual environment `_ (virtualenv) to install CKAN into, and activate it: .. parsed-literal:: sudo mkdir -p |virtualenv| sudo chown \`whoami\` |virtualenv| - virtualenv --no-site-packages |virtualenv| + python3 -m venv |virtualenv| |activate| -.. note:: - - If your system uses Python3 by default (e.g. Ubuntu 18.04) make sure to create - the virtualenv using the Python2.7 executable with the ``--python`` option: - - .. parsed-literal:: - - sudo mkdir -p |virtualenv| - sudo chown \`whoami\` |virtualenv| - virtualenv --python=/usr/bin/python2.7 --no-site-packages |virtualenv| - |activate| - .. important:: The final command above activates your virtualenv. The virtualenv has to @@ -105,11 +100,21 @@ a. Create a Python `virtual environment `_ |activate| -b. Install the recommended ``setuptools`` version: +.. note:: + + For Python 2 then replace the `python3 -m venv` command with: + + .. parsed-literal:: + + virtualenv --python=/usr/bin/python2.7 --no-site-packages |virtualenv| + |activate| + +b. Install the recommended ``setuptools`` version and up-to-date pip: .. parsed-literal:: pip install setuptools==\ |min_setuptools_version| + pip install --upgrade pip c. Install the CKAN source code into your virtualenv. @@ -140,6 +145,10 @@ d. Install the Python modules that CKAN requires into your virtualenv: pip install -r |virtualenv|/src/ckan/requirements.txt +.. note:: + + For Python 2 adjust the filename to: `requirements-py2.txt` + e. Deactivate and reactivate your virtualenv, to make sure you're using the virtualenv's copies of commands like ``paster`` rather than any system-wide installed copies: diff --git a/doc/maintaining/upgrading/index.rst b/doc/maintaining/upgrading/index.rst index dfcb327b378..c0d6edde221 100644 --- a/doc/maintaining/upgrading/index.rst +++ b/doc/maintaining/upgrading/index.rst @@ -128,3 +128,4 @@ appropriate one of these documents: upgrade-package-to-minor-release upgrade-source upgrade-postgres + upgrade-to-python3 diff --git a/doc/maintaining/upgrading/upgrade-to-python3.rst b/doc/maintaining/upgrading/upgrade-to-python3.rst new file mode 100644 index 00000000000..d6fef452e7b --- /dev/null +++ b/doc/maintaining/upgrading/upgrade-to-python3.rst @@ -0,0 +1,35 @@ +================================================== +Upgrading a CKAN install from Python 2 to Python 3 +================================================== + +These instructions describe how to upgrade a source install of CKAN 2.9 from +Python 2 to Python 3, which is necessary because Python 2 is end of life, as of +January 1st, 2020. + +Preparation +----------- + +* Backup your CKAN source, virtualenv and databases, just in case. +* Upgrade to CKAN 2.9, if you've not done already. + +Upgrade +------- + +You'll probably need to deactivate your existing virtual environment:: + + deactivate + +The existing setup has the virtual environment here: |virtualenv| +and the CKAN source code underneath in `/usr/lib/ckan/default/src`. We'll move +that aside in case we need to roll-back: + + .. parsed-literal:: + + sudo mv |virtualenv| /usr/lib/ckan/py2 + +From this doc: :doc:`/maintaining/installing/install-from-source` you need to +do these sections: + +* 1. Install the required packages +* 2. Install CKAN into a Python virtual environment +* 6. Link to who.ini diff --git a/requirements-py2.in b/requirements-py2.in new file mode 100644 index 00000000000..2fc6e717dda --- /dev/null +++ b/requirements-py2.in @@ -0,0 +1,40 @@ +# The file contains the direct ckan requirements. +# Use pip-compile to create a requirements.txt file from this +alembic==1.0.0 +Babel==2.3.4 +bleach==3.0.2 +click==6.7 +fanstatic==0.12 +Flask==1.1.1 +Flask-Babel==0.11.2 +Jinja2==2.10.1 +Markdown==2.6.7 +passlib==1.6.5 +paste==1.7.5.1 +PasteScript==2.0.2 +polib==1.0.7 +psycopg2==2.8.2 +python-magic==0.4.15 +pysolr==3.6.0 +Pylons==0.9.7 +python-dateutil>=1.5.0 +pytz==2016.7 +PyUtilib==5.7.1 +pyyaml # needed by webassets. latest should be fine. +repoze.who-friendlyform==1.0.8 +repoze.who==2.3 +requests==2.22.0 +Routes==1.13 +rq==1.0 +simplejson==3.10.0 +sqlalchemy-migrate==0.12.0 +SQLAlchemy==1.3.5 +sqlparse==0.2.2 +tzlocal==1.3 +unicodecsv>=0.9 +webassets==0.12.1 +WebHelpers==1.3 +WebOb==1.0.8 +WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 +werkzeug==0.15.5 +zope.interface==4.3.2 diff --git a/requirements-py2.txt b/requirements-py2.txt new file mode 100644 index 00000000000..c90903da987 --- /dev/null +++ b/requirements-py2.txt @@ -0,0 +1,66 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements=py2.txt requirements-py2.in +# +alembic==1.0.0 +babel==2.3.4 +beaker==1.10.1 # via pylons +bleach==3.0.2 +certifi==2019.3.9 # via requests +chardet==3.0.4 # via requests +click==6.7 +decorator==4.4.0 # via pylons, sqlalchemy-migrate +fanstatic==0.12 +flask-babel==0.11.2 +Flask==1.1.1 +formencode==1.3.1 # via pylons +funcsigs==1.0.2 # via beaker +idna==2.8 # via requests +itsdangerous==1.1.0 # via flask +jinja2==2.10.1 +mako==1.0.9 # via alembic, pylons +markdown==2.6.7 +markupsafe==1.1.1 # via jinja2, mako, webhelpers +nose==1.3.7 # via pylons +passlib==1.6.5 +paste==1.7.5.1 +pastedeploy==2.0.1 # via pastescript, pylons +pastescript==2.0.2 +pbr==5.2.0 # via sqlalchemy-migrate +polib==1.0.7 +psycopg2==2.8.2 +pygments==2.3.1 # via weberror +pylons==0.9.7 +pysolr==3.6.0 +python-dateutil==2.8.0 +python-editor==1.0.4 # via alembic +python-magic==0.4.15 +pytz==2016.7 +PyUtilib==5.7.1 +pyyaml==5.1 +redis==3.2.1 # via rq +repoze.lru==0.7 # via routes +repoze.who-friendlyform==1.0.8 +repoze.who==2.3 +requests==2.22.0 +routes==1.13 +rq==1.0 +simplejson==3.10.0 +six==1.12.0 # via bleach, pastescript, python-dateutil, pyutilib.component.core, sqlalchemy-migrate +sqlalchemy-migrate==0.12.0 +sqlalchemy==1.3.5 +sqlparse==0.2.2 +tempita==0.5.2 # via pylons, sqlalchemy-migrate, weberror +tzlocal==1.3 +unicodecsv==0.14.1 +urllib3==1.25.2 # via requests +webassets==0.12.1 +webencodings==0.5.1 # via bleach +weberror==0.13.1 # via pylons +webhelpers==1.3 +webob==1.0.8 +webtest==1.4.3 +werkzeug==0.15.5 +zope.interface==4.3.2 diff --git a/requirements.in b/requirements.in index 2fc6e717dda..b86f1d422b5 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -# The file contains the direct ckan requirements. +# The file contains the direct ckan requirements (python3). # Use pip-compile to create a requirements.txt file from this alembic==1.0.0 Babel==2.3.4 @@ -16,12 +16,12 @@ polib==1.0.7 psycopg2==2.8.2 python-magic==0.4.15 pysolr==3.6.0 -Pylons==0.9.7 +# Pylons==0.9.7 - not python3 compatible python-dateutil>=1.5.0 pytz==2016.7 PyUtilib==5.7.1 pyyaml # needed by webassets. latest should be fine. -repoze.who-friendlyform==1.0.8 +# repoze.who-friendlyform==1.0.8 - not python3 compatible repoze.who==2.3 requests==2.22.0 Routes==1.13 @@ -33,8 +33,8 @@ sqlparse==0.2.2 tzlocal==1.3 unicodecsv>=0.9 webassets==0.12.1 -WebHelpers==1.3 -WebOb==1.0.8 -WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 +# WebHelpers==1.3 - not python3 compatible +# WebOb==1.0.8 - pylons dependency +WebTest==1.4.3 werkzeug==0.15.5 zope.interface==4.3.2 diff --git a/requirements.txt b/requirements.txt index 8fa02fbf298..7ce0cd91d4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,61 +6,53 @@ # alembic==1.0.0 babel==2.3.4 -beaker==1.10.1 # via pylons bleach==3.0.2 -certifi==2019.3.9 # via requests +certifi==2019.9.11 # via requests chardet==3.0.4 # via requests click==6.7 -decorator==4.4.0 # via pylons, sqlalchemy-migrate +decorator==4.4.1 # via sqlalchemy-migrate fanstatic==0.12 flask-babel==0.11.2 -Flask==1.1.1 -formencode==1.3.1 # via pylons -funcsigs==1.0.2 # via beaker +flask==1.1.1 idna==2.8 # via requests itsdangerous==1.1.0 # via flask jinja2==2.10.1 -mako==1.0.9 # via alembic, pylons +mako==1.1.0 # via alembic markdown==2.6.7 -markupsafe==1.1.1 # via jinja2, mako, webhelpers -nose==1.3.7 # via pylons +markupsafe==1.1.1 # via jinja2, mako +nose==1.3.7 # via pyutilib passlib==1.6.5 paste==1.7.5.1 -pastedeploy==2.0.1 # via pastescript, pylons +pastedeploy==2.0.1 # via pastescript pastescript==2.0.2 -pbr==5.2.0 # via sqlalchemy-migrate +pbr==5.4.3 # via sqlalchemy-migrate polib==1.0.7 psycopg2==2.8.2 -pygments==2.3.1 # via weberror -pylons==0.9.7 pysolr==3.6.0 -python-dateutil==2.8.0 +python-dateutil==2.8.1 python-editor==1.0.4 # via alembic python-magic==0.4.15 pytz==2016.7 -PyUtilib==5.7.1 -pyyaml==5.1 -redis==3.2.1 # via rq +pyutilib==5.7.1 +pyyaml==5.1.2 +redis==3.3.11 # via rq repoze.lru==0.7 # via routes -repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.22.0 routes==1.13 rq==1.0 simplejson==3.10.0 -six==1.12.0 # via bleach, pastescript, python-dateutil, pyutilib.component.core, sqlalchemy-migrate +six==1.13.0 # via bleach, pastescript, python-dateutil, pyutilib, sqlalchemy-migrate sqlalchemy-migrate==0.12.0 sqlalchemy==1.3.5 sqlparse==0.2.2 -tempita==0.5.2 # via pylons, sqlalchemy-migrate, weberror +tempita==0.5.2 # via sqlalchemy-migrate tzlocal==1.3 unicodecsv==0.14.1 -urllib3==1.25.2 # via requests +urllib3==1.25.6 # via requests webassets==0.12.1 webencodings==0.5.1 # via bleach -weberror==0.13.1 # via pylons -webhelpers==1.3 -webob==1.0.8 +webob==1.0.8 # via fanstatic, repoze.who, webtest webtest==1.4.3 werkzeug==0.15.5 zope.interface==4.3.2