Merge pull request IQSS#7210 from IQSS/develop

Version 5.0

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,7 @@ Closes #
 
 **Suggestions on how to test this**:
 
-**Does this PR introduce a user interface change?**:
+**Does this PR introduce a user interface change? If mockups are available, please link/include them here**:
 
 **Is there a release notes update needed for this change?**:
 

.gitignore

@@ -60,3 +60,8 @@ scripts/installer/default.config
 tests/node_modules
 tests/package-lock.json
 venv
+
+# from thumbnail tests in SearchIT
+scripts/search/data/binary/trees.png.thumb140
+src/main/webapp/resources/images/cc0.png.thumb140
+src/main/webapp/resources/images/dataverseproject.png.thumb140

Dockerfile

@@ -1 +1 @@
-# See `conf/docker` for Docker images
+# See http://guides.dataverse.org/en/latest/developers/containers.html

README.md

@@ -18,7 +18,7 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
 [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org)
 
 [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/)
-[![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/)
+[![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html)
 [![Unit Test Status](https://img.shields.io/travis/IQSS/dataverse?label=Unit%20Test%20Status)](https://travis-ci.org/IQSS/dataverse)
 [![Unit Test Coverage](https://img.shields.io/coveralls/github/IQSS/dataverse?label=Unit%20Test%20Coverage)](https://coveralls.io/github/IQSS/dataverse?branch=develop)
 

Vagrantfile

@@ -1,77 +1,27 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  config.vm.box = "bento/centos-8.2"
 
-  config.vm.define "standalone", primary: true do |standalone|
-    config.vm.hostname = "standalone"
-    # Uncomment this temporarily to get `vagrant destroy` to work
-    #standalone.vm.box = "puppetlabs/centos-7.2-64-puppet"
-
-    operating_system = "centos"
-    if ENV['OPERATING_SYSTEM'].nil?
-      config.vm.box = "puppetlabs/centos-7.2-64-puppet"
-      config.vm.box_version = '1.0.1'
-    elsif ENV['OPERATING_SYSTEM'] == 'debian'
-      puts "WARNING: Debian specified. Here be dragons! https://github.com/IQSS/dataverse/issues/1059"
-      config.vm.box_url = "http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-puppet.box"
-      config.vm.box = "puppet-vagrant-boxes.puppetlabs.com-debian-73-x64-virtualbox-puppet.box"
-    else
-      operating_system = ENV['OPERATING_SYSTEM']
-      puts "Not sure what do to with operating system: #{operating_system}"
-      exit 1
-    end
-
-    mailserver = "localhost"
-    if ENV['MAIL_SERVER'].nil?
-      puts "MAIL_SERVER environment variable not specified. Using #{mailserver} by default.\nTo specify it in bash: export MAIL_SERVER=localhost"
-    else
-      mailserver = ENV['MAIL_SERVER']
-      puts "MAIL_SERVER environment variable found, using #{mailserver}"
-    end
-
-    config.vm.provider "virtualbox" do |v|
-      v.memory = 2048
-      v.cpus = 1
-    end
-    config.vm.provision "shell", path: "scripts/vagrant/setup.sh"
-    config.vm.provision "shell", path: "scripts/vagrant/setup-solr.sh"
-    config.vm.provision "shell", path: "scripts/vagrant/install-dataverse.sh", args: mailserver
-    # FIXME: get tests working and re-enable them!
-    #config.vm.provision "shell", path: "scripts/vagrant/test.sh"
-
-    config.vm.network "private_network", type: "dhcp"
-    config.vm.network "forwarded_port", guest: 80, host: 8888
-    config.vm.network "forwarded_port", guest: 443, host: 9999
-    config.vm.network "forwarded_port", guest: 8983, host: 8993
-    config.vm.network "forwarded_port", guest: 8080, host: 8088
-    config.vm.network "forwarded_port", guest: 8181, host: 8188
-
-    # FIXME: use /dataverse/downloads instead
-    config.vm.synced_folder "downloads", "/downloads"
-    # FIXME: use /dataverse/conf instead
-    config.vm.synced_folder "conf", "/conf"
-    # FIXME: use /dataverse/scripts instead
-    config.vm.synced_folder "scripts", "/scripts"
-    config.vm.synced_folder ".", "/dataverse"
+  config.vm.provider "virtualbox" do |vbox|
+    vbox.cpus = 4
+    vbox.memory = 4096
   end
 
-  config.vm.define "solr", autostart: false do |solr|
-    config.vm.hostname = "solr"
-    solr.vm.box = "puppet-vagrant-boxes.puppetlabs.com-centos-65-x64-virtualbox-puppet.box"
-    config.vm.synced_folder ".", "/dataverse"
-    config.vm.network "private_network", type: "dhcp"
-    config.vm.network "forwarded_port", guest: 8983, host: 9001
-  end
+  config.vm.provision "shell", path: "scripts/vagrant/setup.sh"
+  config.vm.provision "shell", path: "scripts/vagrant/setup-solr.sh"
+  config.vm.provision "shell", path: "scripts/vagrant/install-dataverse.sh"
 
-  config.vm.define "test", autostart: false do |test|
-    config.vm.hostname = "test"
-    test.vm.box = "puppet-vagrant-boxes.puppetlabs.com-centos-65-x64-virtualbox-puppet.box"
-    config.vm.synced_folder ".", "/dataverse"
-    config.vm.network "private_network", type: "dhcp"
-  end
+  config.vm.network "private_network", type: "dhcp"
+  config.vm.network "forwarded_port", guest: 80, host: 8888
+  config.vm.network "forwarded_port", guest: 443, host: 9999
+  config.vm.network "forwarded_port", guest: 8983, host: 8993
+  config.vm.network "forwarded_port", guest: 8080, host: 8088
+  config.vm.network "forwarded_port", guest: 8181, host: 8188
+
+  config.vm.synced_folder ".", "/dataverse"
 
 end

conf/docker-aio/0prep_deps.sh

@@ -3,18 +3,11 @@ if [ ! -d dv/deps ]; then
 	mkdir -p dv/deps
 fi
 wdir=`pwd`
-if [ ! -e dv/deps/glassfish4dv.tgz ]; then
-	echo "glassfish dependency prep"
-	mkdir -p /tmp/dv-prep/gf
-	cd /tmp/dv-prep/gf
-	wget http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip
-	wget https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar -O weld-osgi-bundle-2.2.10.Final-glassfish4.jar
-	unzip glassfish-4.1.zip
-	rm glassfish4/glassfish/modules/weld-osgi-bundle.jar
-	mv weld-osgi-bundle-2.2.10.Final-glassfish4.jar glassfish4/glassfish/modules
-	tar zcf $wdir/dv/deps/glassfish4dv.tgz glassfish4
-	cd $wdir
-	# assuming that folks usually have /tmp auto-clean as needed
+
+if [ ! -e dv/deps/payara-5.2020.2.zip ]; then
+	echo "payara dependency prep"
+	# no more fiddly patching :)
+	wget https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip  -O dv/deps/payara-5.2020.2.zip
 fi
 
 if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then

conf/docker-aio/c7.dockerfile

@@ -1,11 +1,10 @@
 FROM centos:7
 # OS dependencies
-RUN yum install -y https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm
-#RUN yum install -y java-1.8.0-openjdk-headless postgresql-server sudo epel-release unzip perl curl httpd
+RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
 RUN yum install -y java-1.8.0-openjdk-devel postgresql96-server sudo epel-release unzip perl curl httpd
 RUN yum install -y jq lsof awscli
 
-# copy and unpack dependencies (solr, glassfish)
+# copy and unpack dependencies (solr, payara)
 COPY dv /tmp/dv
 COPY testdata/schema*.xml /tmp/dv/
 COPY testdata/solrconfig.xml /tmp/dv
@@ -18,10 +17,10 @@ COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
 RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz 
-RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz
+RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.2.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
-COPY domain-restmonitor.xml /opt/glassfish4/glassfish/domains/domain1/config/domain.xml
+COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml
 
 #RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
 RUN sudo -u postgres /usr/pgsql-9.6/bin/initdb -D /var/lib/pgsql/data
@@ -32,15 +31,15 @@ RUN cp -r /opt/solr-7.7.2/server/solr/configsets/_default /opt/solr-7.7.2/server
 RUN cp /tmp/dv/schema*.xml /opt/solr-7.7.2/server/solr/collection1/conf/
 RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml
 
-# skipping glassfish user and solr user (run both as root)
+# skipping payara user and solr user (run both as root)
 
 #solr port
 EXPOSE 8983
 
 # postgres port
 EXPOSE 5432
 
-# glassfish port
+# payara port
 EXPOSE 8080
 
 # apache port, http
@@ -52,8 +51,8 @@ EXPOSE 9009
 
 RUN mkdir /opt/dv
 
-# yeah - still not happy if glassfish isn't in /usr/local :<
-RUN ln -s /opt/glassfish4 /usr/local/glassfish4
+# keeping the symlink on the off chance that something else is still assuming /usr/local/glassfish4
+RUN ln -s /opt/payara5 /usr/local/glassfish4
 COPY dv/install/ /opt/dv/
 COPY install.bash /opt/dv/
 COPY entrypoint.bash /opt/dv/
@@ -76,7 +75,7 @@ ENV doi_username=${doi_username}
 ENV doi_password=${doi_password}
 COPY configure_doi.bash /opt/dv
 
-# healthcheck for glassfish only (assumes modified domain.xml); 
+# healthcheck for payara only (assumes modified domain.xml);
 #  does not check dataverse application status.
 HEALTHCHECK CMD curl --fail http://localhost:4848/monitoring/domain/server.json || exit 1
 CMD ["/opt/dv/entrypoint.bash"]

conf/docker-aio/configure_doi.bash

@@ -17,8 +17,8 @@ if [ ! -z "${doi_baseurl}" ]; then
         doi_baseurl_esc=`echo ${doi_baseurl} | sed -e 's/:/\\:/'`
         bin/asadmin create-jvm-options "\"-Ddoi.baseurlstring=${doi_baseurl_esc}\""
 fi
-if [ ! -z "${doi_mdcbaseurl}" ]; then
-        bin/asadmin delete-jvm-options "-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"
-        doi_mdcbaseurl_esc=`echo ${doi_mdcbaseurl} | sed -e 's/:/\\:/'`
-        bin/asadmin create-jvm-options "\"-Ddoi.mdcbaseurlstring=${doi_mdcbaseurl_esc}\""
+if [ ! -z "${doi_dataciterestapiurl}" ]; then
+        bin/asadmin delete-jvm-options "-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"
+        doi_dataciterestapiurl_esc=`echo ${doi_dataciterestapiurl} | sed -e 's/:/\\:/'`
+        bin/asadmin create-jvm-options "\"-Ddoi.dataciterestapiurlstring=${doi_dataciterestapiurl_esc}\""
 fi

conf/docker-aio/readme.md

@@ -18,7 +18,7 @@ Also usable for normal development and system evaluation; not intended for produ
 
 
 ### Initial setup (aka - do once):
-- `cd conf/docker-aio` and run `./0prep_deps.sh` to created Glassfish and Solr tarballs in `conf/docker-aio/dv/deps`.
+- `cd conf/docker-aio` and run `./0prep_deps.sh` to created Payara and Solr tarballs in `conf/docker-aio/dv/deps`.
 
 ### Per-build:
 
@@ -29,7 +29,7 @@ Also usable for normal development and system evaluation; not intended for produ
 - `cd conf/docker-aio`, and run `./1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file
 - build the docker image: `docker build -t dv0 -f c7.dockerfile .`
 
-- Run image: `docker run -d -p 8083:8080 -p 8084:80 --name dv dv0` (aka - forward port 8083 locally to 8080 in the container for glassfish, and 8084 to 80 for apache); if you'd like to connect a java debugger to glassfish, use `docker run -d -p 8083:8080 -p 8084:80 -p 9010:9009 --name dv dv0`
+- Run image: `docker run -d -p 8083:8080 -p 8084:80 --name dv dv0` (aka - forward port 8083 locally to 8080 in the container for payara, and 8084 to 80 for apache); if you'd like to connect a java debugger to payara, use `docker run -d -p 8083:8080 -p 8084:80 -p 9010:9009 --name dv dv0`
 
 - Installation (integration test): `docker exec dv /opt/dv/setupIT.bash` 
   (Note that it's possible to customize the installation by editing `conf/docker-aio/default.config` and running `docker exec dv /opt/dv/install.bash` but for the purposes of integration testing, the `setupIT.bash` script above works fine.)
@@ -55,6 +55,6 @@ There isn't any strict requirement on the local port (8083, 8084 in this doc), t
  ``` 
     run something like `docker ps -a | grep dv` to see the container left over from the last run and something like `docker rm 5f72a45b68c8` to remove it. Then try the `docker run` command above again.
 
-* `empty reply from server` or `Failed to connect to ::1: Cannot assign requested address` tend to indicate either that you haven't given glassfish enough time to start, or your docker setup is in an inconsistent state and should probably be restarted.
+* `empty reply from server` or `Failed to connect to ::1: Cannot assign requested address` tend to indicate either that you haven't given payara enough time to start, or your docker setup is in an inconsistent state and should probably be restarted.
 
 * For manually fiddling around with the created dataverse, use user `dataverseAdmin` with password `admin1`.

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/testscripts/install

@@ -15,8 +15,8 @@ export SMTP_SERVER=localhost
 export MEM_HEAP_SIZE=2048
 export GLASSFISH_DOMAIN=domain1
 cd scripts/installer
-cp pgdriver/postgresql-42.2.9.jar $GLASSFISH_ROOT/glassfish/lib
+cp pgdriver/postgresql-42.2.12.jar $GLASSFISH_ROOT/glassfish/lib
 #cp ../../conf/jhove/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 cp /opt/dv/testdata/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 cp /opt/dv/testdata/jhoveConfig.xsd $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhoveConfig.xsd
-./glassfish-setup.sh
+./as-setup.sh dvndb