Merge pull request IQSS#5701 from IQSS/develop

v4.12

conf/docker-aio/1prep.sh

@@ -10,7 +10,6 @@ cp ../jhove/jhove.conf testdata/
 cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../
 cp -r scripts conf/docker-aio/testdata/
-cp doc/sphinx-guides/source/_static/util/pg8-createsequence-prep.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
 cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
 
 # not using dvinstall.zip for setupIT.bash; but still used in install.bash for normal ops

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/testscripts/post

@@ -3,7 +3,6 @@ cd scripts/api
 ./setup-all.sh --insecure -p=admin1 | tee /tmp/setup-all.sh.out
 cd ../..
 psql -U dvnapp dvndb -f scripts/database/reference_data.sql
-psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/pg8-createsequence-prep.sql
 psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/createsequence.sql
 scripts/search/tests/publish-dataverse-root
 #git checkout scripts/api/data/dv-root.json

conf/docker-dcm/0prep.sh

@@ -1,3 +1,11 @@
 #!/bin/sh
+DCM_VERSION=0.5
+RSAL_VERSION=0.1
 
-wget https://github.com/sbgrid/data-capture-module/releases/download/0.5/dcm-0.5-0.noarch.rpm
+if [ ! -e dcm-${DCM_VERSION}-0.noarch.rpm ]; then
+	wget https://github.com/sbgrid/data-capture-module/releases/download/${DCM_VERSION}/dcm-${DCM_VERSION}-0.noarch.rpm
+fi
+
+if [ ! -e rsal-${RSAL_VERSION}-0.noarch.rpm ] ;then
+	wget https://github.com/sbgrid/rsal/releases/download/${RSAL_VERSION}/rsal-${RSAL_VERSION}-0.noarch.rpm
+fi

conf/docker-dcm/cfg/rsal/entrypoint-rsal.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+#/usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf
+/usr/bin/rsync --daemon --config /etc/rsyncd.conf
+lighttpd -D -f /etc/lighttpd/lighttpd.conf

conf/docker-dcm/cfg/rsal/lighttpd-modules.conf

@@ -0,0 +1,174 @@
+#######################################################################
+##
+## ansible managed
+#
+##  Modules to load
+## -----------------
+##
+## at least mod_access and mod_accesslog should be loaded
+## all other module should only be loaded if really neccesary
+##
+## - saves some time
+## - saves memory
+##
+## the default module set contains:
+##
+## "mod_indexfile", "mod_dirlisting", "mod_staticfile"
+##
+## you dont have to include those modules in your list
+##
+## Modules, which are pulled in via conf.d/*.conf
+##
+## NOTE: the order of modules is important.
+##
+## - mod_accesslog     -> conf.d/access_log.conf
+## - mod_compress      -> conf.d/compress.conf
+## - mod_status        -> conf.d/status.conf
+## - mod_webdav        -> conf.d/webdav.conf
+## - mod_cml           -> conf.d/cml.conf
+## - mod_evhost        -> conf.d/evhost.conf
+## - mod_simple_vhost  -> conf.d/simple_vhost.conf
+## - mod_mysql_vhost   -> conf.d/mysql_vhost.conf
+## - mod_trigger_b4_dl -> conf.d/trigger_b4_dl.conf
+## - mod_userdir       -> conf.d/userdir.conf
+## - mod_rrdtool       -> conf.d/rrdtool.conf
+## - mod_ssi           -> conf.d/ssi.conf
+## - mod_cgi           -> conf.d/cgi.conf
+## - mod_scgi          -> conf.d/scgi.conf
+## - mod_fastcgi       -> conf.d/fastcgi.conf
+## - mod_proxy         -> conf.d/proxy.conf
+## - mod_secdownload   -> conf.d/secdownload.conf
+## - mod_expire        -> conf.d/expire.conf
+##
+
+server.modules = (
+  "mod_access",
+#  "mod_alias",
+#  "mod_auth",
+#  "mod_evasive",
+#  "mod_redirect",
+#  "mod_rewrite",
+#  "mod_setenv",
+#  "mod_usertrack",
+)
+
+##
+#######################################################################
+
+#######################################################################
+##
+##  Config for various Modules
+##
+
+##
+## mod_ssi
+##
+#include "conf.d/ssi.conf"
+
+##
+## mod_status
+##
+#include "conf.d/status.conf"
+
+##
+## mod_webdav
+##
+#include "conf.d/webdav.conf"
+
+##
+## mod_compress
+##
+#include "conf.d/compress.conf"
+
+##
+## mod_userdir
+##
+#include "conf.d/userdir.conf"
+
+##
+## mod_magnet
+##
+#include "conf.d/magnet.conf"
+
+##
+## mod_cml
+##
+#include "conf.d/cml.conf"
+
+##
+## mod_rrdtool
+##
+#include "conf.d/rrdtool.conf"
+
+##
+## mod_proxy
+##
+#include "conf.d/proxy.conf"
+
+##
+## mod_expire
+##
+#include "conf.d/expire.conf"
+
+##
+## mod_secdownload
+##
+#include "conf.d/secdownload.conf"
+
+##
+#######################################################################
+
+#######################################################################
+##
+## CGI modules
+##
+
+##
+## SCGI (mod_scgi)
+##
+#include "conf.d/scgi.conf"
+
+##
+## FastCGI (mod_fastcgi)
+##
+#include "conf.d/fastcgi.conf"
+
+##
+## plain old CGI (mod_cgi)
+##
+include "conf.d/cgi.conf"
+
+##
+#######################################################################
+
+#######################################################################
+##
+## VHost Modules
+##
+##  Only load ONE of them!
+## ========================
+##
+
+##
+## You can use conditionals for vhosts aswell.
+## 
+## see http://www.lighttpd.net/documentation/configuration.html
+##
+
+##
+## mod_evhost
+##
+#include "conf.d/evhost.conf"
+
+##
+## mod_simple_vhost
+##
+#include "conf.d/simple_vhost.conf"
+
+##
+## mod_mysql_vhost
+##
+#include "conf.d/mysql_vhost.conf"
+
+##
+#######################################################################

conf/docker-dcm/cfg/rsal/lighttpd.conf

@@ -0,0 +1,43 @@
+## lighttpd configuration customized for RSAL; centos7
+
+# refuse connections not from frontend or localhost
+# DO NOT HAVE THIS OPEN TO THE WORLD!!!
+#$HTTP["remoteip"] !~ "192.168.2.2|127.0.0.1" {
+#url.access-deny = ("")
+#}
+server.breakagelog = "/var/log/lighttpd/breakage.log"
+
+#######################################################################
+##
+## Some Variable definition which will make chrooting easier.
+##
+## if you add a variable here. Add the corresponding variable in the
+## chroot example aswell.
+##
+var.log_root    = "/var/log/lighttpd"
+var.server_root = "/opt/rsal/api"
+var.state_dir   = "/var/run"
+var.home_dir    = "/var/lib/lighttpd"
+var.conf_dir    = "/etc/lighttpd"
+
+var.cache_dir   = "/var/cache/lighttpd"
+var.socket_dir  = home_dir + "/sockets"
+include "modules.conf"
+server.port = 80
+server.use-ipv6 = "disable"
+server.username  = "lighttpd"
+server.groupname = "lighttpd"
+server.document-root = server_root 
+server.pid-file = state_dir + "/lighttpd.pid"
+server.errorlog             = log_root + "/error.log"
+include "conf.d/access_log.conf"
+include "conf.d/debug.conf"
+server.event-handler = "linux-sysepoll"
+server.network-backend = "linux-sendfile"
+server.stat-cache-engine = "simple"
+server.max-connections = 1024
+static-file.exclude-extensions = ( ".php", ".pl", ".fcgi", ".scgi" )
+include "conf.d/mime.conf"
+include "conf.d/dirlisting.conf"
+server.follow-symlink = "enable"
+server.upload-dirs = ( "/var/tmp" )

conf/docker-dcm/cfg/rsal/rsyncd.conf

@@ -0,0 +1,8 @@
+lock file=/var/run/rsync.lock
+log file=/var/log/rsyncd.log
+pid file=/var/log/rsyncd.pid
+
+[10.5072]
+ path=/public/
+ read only=yes
+

conf/docker-dcm/configure_rsal.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+fn=rsal-workflow2.json
+# needs an actual IP (vs a hostname) for whitelist
+rsalip=`dig +short rsalsrv`
+
+# create workflow
+curl -s -X POST -H "Content-type: application/json" -d @${fn} "http://localhost:8080/api/admin/workflows" 
+
+# put rsal on the whitelist
+curl -X PUT -d "127.0.0.1;${rsalip}" "http://localhost:8080/api/admin/workflows/ip-whitelist"
+
+# set workflow as default
+curl -X PUT -d "1" "http://localhost:8080/api/admin/workflows/default/PrePublishDataset"
+
+# local access path
+curl -X PUT -d "/hpc/storage" "http://localhost:8080/api/admin/settings/:LocalDataAccessPath"
+
+# storage sites
+curl -X POST -H "Content-type: application/json" --upload-file site-primary.json "http://localhost:8080/api/admin/storageSites"
+curl -X POST -H "Content-type: application/json" --upload-file site-remote.json "http://localhost:8080/api/admin/storageSites"

conf/docker-dcm/dcmsrv.dockerfile

@@ -3,17 +3,17 @@ FROM centos:6
 RUN yum install -y epel-release
 ARG RPMFILE=dcm-0.5-0.noarch.rpm
 COPY ${RPMFILE} /tmp/
-COPY bashrc /root/.bashrc
-COPY test_install.sh /root/
+COPY cfg/dcm/bashrc /root/.bashrc
+COPY cfg/dcm/test_install.sh /root/
 RUN yum localinstall -y /tmp/${RPMFILE}
 RUN pip install -r /opt/dcm/requirements.txt
 RUN pip install awscli==1.15.75
 run export PATH=~/.local/bin:$PATH
 RUN /root/test_install.sh
-COPY rq-init-d /etc/init.d/rq
+COPY cfg/dcm/rq-init-d /etc/init.d/rq
 RUN useradd glassfish
-COPY entrypoint-dcm.sh /
-COPY healthcheck-dcm.sh /
+COPY cfg/dcm/entrypoint-dcm.sh /
+COPY cfg/dcm/healthcheck-dcm.sh /
 EXPOSE 80
 EXPOSE 22
 VOLUME /hold

conf/docker-dcm/docker-compose.yml

@@ -10,13 +10,29 @@ services:
     container_name: dcmsrv
     volumes:
       - hold:/hold
+  rsalsrv:
+    build:
+      context: .
+      dockerfile: rsalsrv.dockerfile
+    container_name: rsalsrv
+#    image: rsalrepo_rsal
+    volumes:
+      - hold:/hold
+      - ./:/mnt
+    environment:
+      DV_HOST: http://dvsrv:8080
+      DV_APIKEY: burrito
+    ports:
+      - "8889:80"
+      - "873:873"
   dvsrv:
     build:
       context: .
       dockerfile: dv0dcm.dockerfile
     container_name: dvsrv
     volumes:
       - hold:/hold
+      - ./:/mnt
     ports:
       - "8083:8080"
       - "8084:80"

conf/docker-dcm/dv0dcm.dockerfile

@@ -1,4 +1,7 @@
 # dv0 assumed to be image name for docker-aio
 FROM dv0
+RUN yum install -y bind-utils
 COPY configure_dcm.sh /opt/dv/
+COPY configure_rsal.sh /opt/dv/
+COPY rsal-workflow2.json site-primary.json site-remote.json /opt/dv/
 VOLUME /hold

conf/docker-dcm/publish_major.bash

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# publish dataset based on database id
+
+dset_id=$1
+if [ -z "$dset_id" ]; then
+	echo "no dataset id specified, bailing out"
+	exit 1
+fi
+
+k_d=burrito
+
+h=http://dvsrv
+
+curl -X POST -H "X-Dataverse-key: ${k_d}" "${h}/api/datasets/${dset_id}/actions/:publish?type=major"
+
+

conf/docker-dcm/readme.txt → conf/docker-dcm/readme.md

@@ -2,18 +2,21 @@ This docker-compose setup is intended for use in development, small scale evalua
 
 Setup:
 
-- build docker-aio image with name dv0 as described in ../docker-aio` (don't start up the docker image or run setupIT.bash)
+- build docker-aio image with name dv0 as described in `../docker-aio` (don't start up the docker image or run setupIT.bash)
 - work in the `conf/docker-dcm` directory for below commands
 - download/prepare dependencies: `./0prep.sh`
 - build dcm/dv0dcm images with docker-compose: `docker-compose -f docker-compose.yml build`
 - start containers: `docker-compose -f docker-compose.yml up -d`
-- wait for container to show "healthy" (aka - `docker ps`), then wait another 4-5 minutes (even though it shows healthy, glassfish is still standing itself up), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash`
+- wait for container to show "healthy" (aka - `docker ps`), then run dataverse app installation: `docker exec -it dvsrv /opt/dv/install.bash` (until https://github.com/IQSS/dataverse/issues/5374 is sorted, it may be preferable to connect to the container and run configuration scripts inside - e.g. `docker exec -it dvsrv bash ; ./install.bash`)
+- for development, you probably want to use the `FAKE` DOI provider: `docker exec -it dvsrv /opt/dv/configure_doi.bash`
 - configure dataverse application to use DCM: `docker exec -it dvsrv /opt/dv/configure_dcm.sh`
+- configure dataverse application to use RSAL (if desired): `docker exec -it dvsrv /opt/dv/configure_rsal.sh`
 
 Operation:
 The dataverse installation is accessible at `http://localhost:8084`.
 The `dcm_client` container is intended to be used for executing transfer scripts, and `conf/docker-dcm` is available at `/mnt` inside the container; this container can be accessed with `docker exec -it dcm_client bash`.
 The DCM cron job is NOT configured here; for development purposes the DCM checks can be run manually with `docker exec -it dcmsrv /opt/dcm/scn/post_upload.bash`.
+The RSAL cron job is similarly NOT configured; for development purposes `docker exec -it rsalsrv /opt/rsal/scn/pub.py` can be run manually.
 
 
 Cleanup: