From 6f1845b71e0ace318cb7601788de49534d42350d Mon Sep 17 00:00:00 2001 From: "james.gardner@okfn.org" Date: Sun, 23 Oct 2011 17:11:01 +0100 Subject: [PATCH] [Packaging] Adding Solr support and pyenvs for 1.5b and CKAN workshop --- LICENSE.txt | 117 ++++++++++++++++++++++++++ MANIFEST.in | 3 +- ckan/config/deployment.ini_tmpl | 2 + ckan_deb/DEBIAN/control.template | 2 +- ckan_deb/usr/bin/ckan-create-instance | 15 +++- ckan_deb/usr/lib/ckan/common.sh | 20 ++++- requires/lucid_missing.txt | 10 +-- setup.py | 1 - 8 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000000..0e58e4e3401 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,117 @@ +License ++++++++ + +CKAN - Data Catalogue Software +Copyright (C) 2007 Open Knowledge Foundation + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + + +Note +==== + +CKAN is sometimes packaged directly with other software (listed in +requires/lucid_conflict.txt). In these cases, we are required to list the +licenses of the packaged softare too. They are all AGPL compatible and read as +follows in the next sections. + +WebHelpers +---------- + +Portions of WebHelpers covered by the following license:: + +Copyright (c) 2005, the Lawrence Journal-World +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Django nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Genshi +------ + +Copyright © 2006-2007 Edgewall Software +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Migrate +------- + +Open Source Initiative OSI - The MIT License:Licensing +[OSI Approved License] + +The MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +SQLAlchemy +---------- + +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright (c) 2005-2011 Michael Bayer and contributors. SQLAlchemy is a trademark of Michael Bayer. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + diff --git a/MANIFEST.in b/MANIFEST.in index bf6eb17214e..ddab6b52e36 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,9 @@ include ckan/config/deployment.ini_tmpl recursive-include ckan/public * recursive-include ckan/config *.ini +recursive-include ckan/config *.xml recursive-include ckan/templates * recursive-include ckan *.ini prune .hg include CHANGELOG.txt -include ckan/migration/migrate.cfg \ No newline at end of file +include ckan/migration/migrate.cfg diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 551e2f69fa0..fda9e2f9c21 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -143,6 +143,8 @@ ckan.site_url = ## Favicon (default is the CKAN software favicon) ckan.favicon = http://assets.okfn.org/p/ckan/img/ckan.ico +## Solr support +#solr_url = http://127.0.0.1:8983/solr ## An 'id' for the site (using, for example, when creating entries in a common search index) ## If not specified derived from the site_url diff --git a/ckan_deb/DEBIAN/control.template b/ckan_deb/DEBIAN/control.template index 4683988306d..9895b797719 100644 --- a/ckan_deb/DEBIAN/control.template +++ b/ckan_deb/DEBIAN/control.template @@ -5,7 +5,7 @@ Architecture: all Maintainer: James Gardner Section: main/web Priority: extra -Depends: python-ckan, postgresql-8.4, apache2, libapache2-mod-wsgi, python-apachemiddleware, python-virtualenv, python-pip +Depends: python-ckan, postgresql-8.4, apache2, libapache2-mod-wsgi, python-virtualenv, python-pip, solr-jetty Homepage: http://ckan.org Description: ckan Common files useful for managing CKAN installations diff --git a/ckan_deb/usr/bin/ckan-create-instance b/ckan_deb/usr/bin/ckan-create-instance index d8821cae238..6108aafa250 100755 --- a/ckan_deb/usr/bin/ckan-create-instance +++ b/ckan_deb/usr/bin/ckan-create-instance @@ -43,6 +43,8 @@ echo "Making sure PostgreSQL is running ..." echo "Ensuring users and groups are set up correctly ..." ckan_ensure_users_and_groups ${INSTANCE} +chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/install_settings.sh + echo "Ensuring directories exist for ${INSTANCE} CKAN INSTANCE ..." ckan_make_ckan_directories ${INSTANCE} @@ -75,6 +77,17 @@ ckan_create_wsgi_handler ${INSTANCE} echo "Setting the password of the ${INSTANCE} user in PostgreSQL" ckan_add_or_replace_database_user ${INSTANCE} ${CKAN_DB_PASSWORD} +# Solr support +echo "Setting up Solr ..." +sed \ + -e "s,NO_START=1,NO_START=0," \ + -e "s,#JETTY_HOST=\$(uname -n),JETTY_HOST=127.0.0.1," \ + -e "s,#JETTY_PORT=8080,JETTY_PORT=8983," \ + -i /etc/default/jetty +mv /usr/share/solr/conf/schema.xml /usr/share/solr/conf/schema.xml.`date --utc "+%Y-%m-%d_%T"`.bak +ln -s /usr/lib/pymodules/python2.6/ckan/config/schema.xml /usr/share/solr/conf/schema.xml +service jetty start + # Create the config file echo "Creating/overwriting the config for CKAN ... " # We use the actual password in PostgreSQL in case any non-sense has gone on @@ -121,7 +134,7 @@ echo "Restarting apache ..." # Install the new crontab echo "Enabling crontab for the ckan${INSTANCE} user ..." PACKAGED_CRONJOB="/tmp/${INSTANCE}-cronjob" -cat < ${PACKAGED_CRONJOB} +cat << EOF > ${PACKAGED_CRONJOB} # WARNING: Do not edit these cron tabs, they will be overwritten any time # the ckan INSTANCE package is upgraded # QUESTION: Should email reports be sent to root? diff --git a/ckan_deb/usr/lib/ckan/common.sh b/ckan_deb/usr/lib/ckan/common.sh index 97626b30006..f89fd40ad42 100644 --- a/ckan_deb/usr/lib/ckan/common.sh +++ b/ckan_deb/usr/lib/ckan/common.sh @@ -45,12 +45,12 @@ ckan_ensure_users_and_groups () { INSTANCE=$1 COMMAND_OUTPUT=`cat /etc/group | grep "ckan${INSTANCE}:"` if ! [[ "$COMMAND_OUTPUT" =~ "ckan${INSTANCE}:" ]] ; then - echo "Crating the 'ckan${INSTANCE}' group ..." + echo "Creating the 'ckan${INSTANCE}' group ..." sudo groupadd --system "ckan${INSTANCE}" fi COMMAND_OUTPUT=`cat /etc/passwd | grep "ckan${INSTANCE}:"` if ! [[ "$COMMAND_OUTPUT" =~ "ckan${INSTANCE}:" ]] ; then - echo "Crating the 'ckan${INSTANCE}' user ..." + echo "Creating the 'ckan${INSTANCE}' user ..." sudo useradd --system --gid "ckan${INSTANCE}" --home /var/lib/ckan/${INSTANCE} -M --shell /usr/sbin/nologin ckan${INSTANCE} fi } @@ -84,6 +84,7 @@ ckan_create_who_ini () { cp -n /usr/share/pyshared/ckan/config/who.ini /etc/ckan/${INSTANCE}/who.ini sed -e "s,%(here)s,/var/lib/ckan/${INSTANCE}," \ -i /etc/ckan/${INSTANCE}/who.ini + chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/who.ini fi fi } @@ -109,7 +110,11 @@ ckan_create_config_file () { -e "s,^\(sqlalchemy.url\)[ =].*,\1 = postgresql://${INSTANCE}:${password}@localhost/${INSTANCE}," \ -e "s,ckan\.site_logo,\#ckan.site_logo," \ -e "s,ckan\.log,/var/log/ckan/${INSTANCE}/${INSTANCE}.log," \ + -e "s,ckan\.site_id,${INSTANCE}," \ + -e "s,ckan\.site_description,${INSTANCE}," \ + -e "s,#solr_url = http://127.0.0.1:8983/solr,solr_url = http://127.0.0.1:8983/solr," \ -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini + sudo chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/${INSTANCE}.ini fi } @@ -147,6 +152,7 @@ ckan_ensure_db_exists () { if ! [[ "$COMMAND_OUTPUT" =~ ${INSTANCE} ]] ; then echo "Creating the database ..." sudo -u postgres createdb -O ${INSTANCE} ${INSTANCE} + paster --plugin=ckan db init --config=/etc/ckan/${INSTANCE}/${INSTANCE}.ini fi fi } @@ -161,7 +167,11 @@ ckan_create_wsgi_handler () { INSTANCE=$1 if [ ! -f "/var/lib/ckan/${INSTANCE}/wsgi.py" ] then - sudo virtualenv --setuptools /var/lib/ckan/${INSTANCE}/pyenv + sudo mkdir /var/lib/ckan/${INSTANCE}/pyenv + sudo chown -R ckan${INSTANCE}:ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv + sudo -u ckan${INSTANCE} virtualenv --setuptools /var/lib/ckan/${INSTANCE}/pyenv + echo "Attempting to install Pip 1.0 from pypi.python.org into pyenv to be used for extensions ..." + sudo -u ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv/bin/easy_install pip -U "pip>=1.0" "pip<=1.0.99" cat <<- EOF > /var/lib/ckan/${INSTANCE}/wsgi.py import os instance_dir = '/var/lib/ckan/${INSTANCE}' @@ -235,6 +245,10 @@ ckan_overwrite_apache_config () { # pass authorization info on (needed for rest api) WSGIPassAuthorization On + # Deploy as a daemon (avoids conflicts between CKAN instances) + # WSGIDaemonProcess ${INSTANCE} display-name=${INSTANCE} processes=4 threads=15 maximum-requests=10000 + # WSGIProcessGroup ${INSTANCE} + ErrorLog /var/log/apache2/${INSTANCE}.error.log CustomLog /var/log/apache2/${INSTANCE}.custom.log combined EOF diff --git a/requires/lucid_missing.txt b/requires/lucid_missing.txt index 63f394a7636..a1504094a6d 100644 --- a/requires/lucid_missing.txt +++ b/requires/lucid_missing.txt @@ -8,17 +8,17 @@ # vdm>=0.9,<0.9.99 -e hg+https://bitbucket.org/okfn/vdm@vdm-0.9#egg=vdm # markupsafe==0.9.2 required by webhelpers==1.2 required by formalchemy with SQLAlchemy 0.6 --e git+https://github.com/mitsuhiko/markupsafe.git@0.9.2#egg=markupsafe +markupsafe==0.9.2 # autoneg>=0.5 -e git+https://github.com/wwaites/autoneg.git@b4c727b164f411cc9d60#egg=autoneg # flup>=0.5 -e hg+http://hg.saddi.com/flup@301a58656bfb#egg=flup # solrpy == 0.9.4 -solrpy==0.9.4 -# All the conflicting dependencies from the lucid_conflict.txt file --e hg+https://bitbucket.org/okfn/ckan-deps@6287665a1965#egg=ckan-deps +#solrpy==0.9.4 # FormAlchemy --e git+https://github.com/FormAlchemy/formalchemy.git@1.3.9#egg=formalchemy +formalchemy==1.3.9 +# Apachemiddleware +-e hg+https://hg.3aims.com/public/ApacheMiddleware@tip#egg=apachemiddleware # NOTE: Developers, our build script for the Debian packages relies on the # requirements above being specified as editable resources with their diff --git a/setup.py b/setup.py index d190e918a78..57c66e00d44 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,6 @@ install_requires=[ ], extras_require = { - 'solr': ['solrpy==0.9.4'], }, packages=find_packages(exclude=['ez_setup']), include_package_data=True,