[Packaging] Adding Solr support and pyenvs for 1.5b and CKAN workshop

LICENSE.txt

@@ -0,0 +1,117 @@
+License
++++++++
+
+CKAN - Data Catalogue Software
+Copyright (C) 2007 Open Knowledge Foundation
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+Note
+====
+
+CKAN is sometimes packaged directly with other software (listed in
+requires/lucid_conflict.txt). In these cases, we are required to list the
+licenses of the packaged softare too. They are all AGPL compatible and read as
+follows in the next sections.
+
+WebHelpers
+----------
+
+Portions of WebHelpers covered by the following license::
+
+Copyright (c) 2005, the Lawrence Journal-World
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice, 
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright 
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of Django nor the names of its contributors may be used
+       to endorse or promote products derived from this software without
+       specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Genshi
+------
+
+Copyright © 2006-2007 Edgewall Software
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+   2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+   3. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. 
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Migrate
+-------
+
+Open Source Initiative OSI - The MIT License:Licensing
+[OSI Approved License]
+
+The MIT License
+
+Copyright (c) <year> <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+SQLAlchemy
+----------
+
+This is the MIT license: http://www.opensource.org/licenses/mit-license.php
+
+Copyright (c) 2005-2011 Michael Bayer and contributors. SQLAlchemy is a trademark of Michael Bayer.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

MANIFEST.in

@@ -1,8 +1,9 @@
 include ckan/config/deployment.ini_tmpl
 recursive-include ckan/public *
 recursive-include ckan/config *.ini
+recursive-include ckan/config *.xml
 recursive-include ckan/templates *
 recursive-include ckan *.ini
 prune .hg
 include CHANGELOG.txt
-include ckan/migration/migrate.cfg
+include ckan/migration/migrate.cfg

ckan/config/deployment.ini_tmpl

@@ -143,6 +143,8 @@ ckan.site_url =
 ## Favicon (default is the CKAN software favicon)
 ckan.favicon = http://assets.okfn.org/p/ckan/img/ckan.ico
 
+## Solr support
+#solr_url = http://127.0.0.1:8983/solr
 
 ## An 'id' for the site (using, for example, when creating entries in a common search index) 
 ## If not specified derived from the site_url

ckan_deb/DEBIAN/control.template

@@ -5,7 +5,7 @@ Architecture: all
 Maintainer: James Gardner <[email protected]>
 Section: main/web
 Priority: extra
-Depends: python-ckan, postgresql-8.4, apache2, libapache2-mod-wsgi, python-apachemiddleware, python-virtualenv, python-pip
+Depends: python-ckan, postgresql-8.4, apache2, libapache2-mod-wsgi, python-virtualenv, python-pip, solr-jetty
 Homepage: http://ckan.org
 Description: ckan
  Common files useful for managing CKAN installations

ckan_deb/usr/bin/ckan-create-instance

@@ -43,6 +43,8 @@ echo "Making sure PostgreSQL is running ..."
 echo "Ensuring users and groups are set up correctly ..."
 ckan_ensure_users_and_groups ${INSTANCE}
 
+chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/install_settings.sh
+
 echo "Ensuring directories exist for ${INSTANCE} CKAN INSTANCE ..."
 ckan_make_ckan_directories ${INSTANCE}
 
@@ -75,6 +77,17 @@ ckan_create_wsgi_handler ${INSTANCE}
 echo "Setting the password of the ${INSTANCE} user in PostgreSQL"
 ckan_add_or_replace_database_user ${INSTANCE} ${CKAN_DB_PASSWORD}
 
+# Solr support
+echo "Setting up Solr ..."
+sed \
+    -e "s,NO_START=1,NO_START=0," \
+    -e "s,#JETTY_HOST=\$(uname -n),JETTY_HOST=127.0.0.1," \
+    -e "s,#JETTY_PORT=8080,JETTY_PORT=8983," \
+    -i /etc/default/jetty
+mv /usr/share/solr/conf/schema.xml /usr/share/solr/conf/schema.xml.`date --utc "+%Y-%m-%d_%T"`.bak 
+ln -s /usr/lib/pymodules/python2.6/ckan/config/schema.xml /usr/share/solr/conf/schema.xml
+service jetty start
+
 # Create the config file
 echo "Creating/overwriting the config for CKAN ... "
 # We use the actual password in PostgreSQL in case any non-sense has gone on
@@ -121,7 +134,7 @@ echo "Restarting apache ..."
 # Install the new crontab
 echo "Enabling crontab for the ckan${INSTANCE} user ..."
 PACKAGED_CRONJOB="/tmp/${INSTANCE}-cronjob"
-cat <<EOF > ${PACKAGED_CRONJOB}
+cat << EOF > ${PACKAGED_CRONJOB}
 # WARNING:  Do not edit these cron tabs, they will be overwritten any time 
 #           the ckan INSTANCE package is upgraded
 # QUESTION: Should email reports be sent to root?

ckan_deb/usr/lib/ckan/common.sh

@@ -45,12 +45,12 @@ ckan_ensure_users_and_groups () {
     INSTANCE=$1
     COMMAND_OUTPUT=`cat /etc/group | grep "ckan${INSTANCE}:"`
     if ! [[ "$COMMAND_OUTPUT" =~ "ckan${INSTANCE}:" ]] ; then
-        echo "Crating the 'ckan${INSTANCE}' group ..." 
+        echo "Creating the 'ckan${INSTANCE}' group ..." 
         sudo groupadd --system "ckan${INSTANCE}"
     fi
     COMMAND_OUTPUT=`cat /etc/passwd | grep "ckan${INSTANCE}:"`
     if ! [[ "$COMMAND_OUTPUT" =~ "ckan${INSTANCE}:" ]] ; then
-        echo "Crating the 'ckan${INSTANCE}' user ..." 
+        echo "Creating the 'ckan${INSTANCE}' user ..." 
         sudo useradd  --system  --gid "ckan${INSTANCE}" --home /var/lib/ckan/${INSTANCE} -M  --shell /usr/sbin/nologin ckan${INSTANCE}
     fi
 }
@@ -84,6 +84,7 @@ ckan_create_who_ini () {
             cp -n /usr/share/pyshared/ckan/config/who.ini /etc/ckan/${INSTANCE}/who.ini
             sed -e "s,%(here)s,/var/lib/ckan/${INSTANCE}," \
                 -i /etc/ckan/${INSTANCE}/who.ini
+            chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/who.ini
         fi
     fi
 }
@@ -109,7 +110,11 @@ ckan_create_config_file () {
             -e "s,^\(sqlalchemy.url\)[ =].*,\1 = postgresql://${INSTANCE}:${password}@localhost/${INSTANCE}," \
             -e "s,ckan\.site_logo,\#ckan.site_logo," \
             -e "s,ckan\.log,/var/log/ckan/${INSTANCE}/${INSTANCE}.log," \
+            -e "s,ckan\.site_id,${INSTANCE}," \
+            -e "s,ckan\.site_description,${INSTANCE}," \
+            -e "s,#solr_url = http://127.0.0.1:8983/solr,solr_url = http://127.0.0.1:8983/solr," \
             -i /etc/ckan/${INSTANCE}/${INSTANCE}.ini
+        sudo chown ckan${INSTANCE}:ckan${INSTANCE} /etc/ckan/${INSTANCE}/${INSTANCE}.ini
     fi
 }
 
@@ -147,6 +152,7 @@ ckan_ensure_db_exists () {
         if ! [[ "$COMMAND_OUTPUT" =~ ${INSTANCE} ]] ; then
             echo "Creating the database ..."
             sudo -u postgres createdb -O ${INSTANCE} ${INSTANCE}
+            paster --plugin=ckan db init --config=/etc/ckan/${INSTANCE}/${INSTANCE}.ini
         fi
     fi
 }
@@ -161,7 +167,11 @@ ckan_create_wsgi_handler () {
         INSTANCE=$1
         if [ ! -f "/var/lib/ckan/${INSTANCE}/wsgi.py" ]
         then
-            sudo virtualenv --setuptools /var/lib/ckan/${INSTANCE}/pyenv
+            sudo mkdir /var/lib/ckan/${INSTANCE}/pyenv
+            sudo chown -R ckan${INSTANCE}:ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv
+            sudo -u ckan${INSTANCE} virtualenv --setuptools /var/lib/ckan/${INSTANCE}/pyenv
+            echo "Attempting to install Pip 1.0 from pypi.python.org into pyenv to be used for extensions ..."
+            sudo -u ckan${INSTANCE} /var/lib/ckan/${INSTANCE}/pyenv/bin/easy_install pip -U "pip>=1.0" "pip<=1.0.99"
             cat <<- EOF > /var/lib/ckan/${INSTANCE}/wsgi.py
 	import os
 	instance_dir = '/var/lib/ckan/${INSTANCE}'
@@ -235,6 +245,10 @@ ckan_overwrite_apache_config () {
     # pass authorization info on (needed for rest api)
     WSGIPassAuthorization On
 
+    # Deploy as a daemon (avoids conflicts between CKAN instances)
+    # WSGIDaemonProcess ${INSTANCE} display-name=${INSTANCE} processes=4 threads=15 maximum-requests=10000
+    # WSGIProcessGroup ${INSTANCE}
+
     ErrorLog /var/log/apache2/${INSTANCE}.error.log
     CustomLog /var/log/apache2/${INSTANCE}.custom.log combined
 EOF

requires/lucid_missing.txt

@@ -8,17 +8,17 @@
 # vdm>=0.9,<0.9.99
 -e hg+https://bitbucket.org/okfn/[email protected]#egg=vdm
 # markupsafe==0.9.2 required by webhelpers==1.2 required by formalchemy with SQLAlchemy 0.6
--e git+https://github.com/mitsuhiko/markupsafe.git@0.9.2#egg=markupsafe
+markupsafe==0.9.2
 # autoneg>=0.5
 -e git+https://github.com/wwaites/autoneg.git@b4c727b164f411cc9d60#egg=autoneg
 # flup>=0.5
 -e hg+http://hg.saddi.com/flup@301a58656bfb#egg=flup
 # solrpy == 0.9.4
-solrpy==0.9.4
-# All the conflicting dependencies from the lucid_conflict.txt file
--e hg+https://bitbucket.org/okfn/ckan-deps@6287665a1965#egg=ckan-deps
+#solrpy==0.9.4
 # FormAlchemy
--e git+https://github.com/FormAlchemy/[email protected]#egg=formalchemy
+formalchemy==1.3.9
+# Apachemiddleware
+-e hg+https://hg.3aims.com/public/ApacheMiddleware@tip#egg=apachemiddleware
 
 # NOTE: Developers, our build script for the Debian packages relies on the 
 #       requirements above being specified as editable resources with their

setup.py

@@ -20,7 +20,6 @@
     install_requires=[
     ],
     extras_require = {
-        'solr': ['solrpy==0.9.4'],
     },
     packages=find_packages(exclude=['ez_setup']),
     include_package_data=True,