Merge pull request IQSS#4098 from solhm/3.6to4.xmigration

3.6 to 4.x migration

scripts/migration/files_source_

@@ -189,6 +189,8 @@ while ( <> )
 	    # datafile object:
 	    $fsname = $dbh->quote($fsname);
 	    print qq {INSERT INTO datafile (id, contenttype, filesystemname, filesize, md5, restricted) VALUES ($newdatafileid, '$type', $fsname, $fsize, '$md5', $restricted);} . "\n";
+	    # Use the below line instead of the above if you are using 4.6 or above
+            # print qq {INSERT INTO datafile (id, contenttype, filesystemname, filesize, checksumtype, restricted,checksumvalue,rootdatafileid) VALUES ($newdatafileid, '$type', $fsname, $fsize, 'MD5', $restricted,'',-1);} . "\n";
 	}
 	else 
 	{
@@ -198,7 +200,7 @@ while ( <> )
 
 	# file metadata object:
 	print qq {INSERT INTO filemetadata (id, description, label, restricted, version, datasetversion_id, datafile_id) VALUES ($fmid, $description, $label, $restricted, 1, $dsvid, $newdatafileid);} . "\n";
-
+	
 	# and the category, if exists:
 
 	if ($category && $category ne "") 

scripts/migration/migrate_users.sql

@@ -10,6 +10,13 @@ insert into authenticateduser(	id, affiliation, email, firstname, lastname, posi
 	select			id, institution, email, firstname, lastname, position, username, false
 	from _dvn3_vdcuser;
 
+----------------------
+--use the below instead of the above query for migrating to 4.7.1 and above
+---------------------
+--insert into authenticateduser(id, affiliation, email, firstname, lastname, position, useridentifier, superuser,createdtime)
+--	select			id, institution, email, firstname, lastname, position, username, false, '01-01-2000 00:00:00'
+--	from _dvn3_vdcuser;
+
 insert into authenticateduserlookup(	authenticationproviderid, persistentuserid, authenticateduser_id)
 	select				'builtin',  username, id
 	from _dvn3_vdcuser;
@@ -34,4 +41,4 @@ insert into explicitgroup_authenticateduser(  explicitgroup_id, containedauthent
 
 SELECT setval('builtinuser_id_seq', (SELECT MAX(id) FROM builtinuser));
 SELECT setval('authenticateduser_id_seq', (SELECT MAX(id) FROM authenticateduser));
-SELECT setval('explicitgroup_id_seq', (SELECT MAX(id) FROM explicitgroup));
+SELECT setval('explicitgroup_id_seq', (SELECT MAX(id) FROM explicitgroup));

scripts/migration/migration_instructions.txt

@@ -22,6 +22,7 @@ Pre steps (contained in the migration_presteps document):
 Migration:
 
 1. run migrate_users.sql script
+   If you are migrating to 4.7.1 or above check the comment around line 14 of the migrate_users.sql script 
 2. run migrate_dataverses.sql script
 2a. migrate preloaded customizations
 3. run custom_field_map.sql script (this must be updated to contain the custom field mappings specific to 
@@ -68,6 +69,7 @@ cat migrated_datasets.txt | ./files_source_ <OFFSET> > files_import.sql
 
 where <OFFSET> is the "last ID ..." from step a. 
 
+If you are migrating to 4.6 or above check the comment around line 192 of files_source_ script 
 This script may produce a lot of stderr output, that you may want to save. 
 You can do that by running it as 
 

scripts/migration/scrub_duplicate_emails.sql

@@ -3,17 +3,17 @@
 --------------------
 /*
 -- Query to list all user acocunts with duplicate e-mails
-select id, username, email from vdcuser
-where email in (
-select email from vdcuser
-group by email
+select id, username, lower(email) from vdcuser 
+where lower(email) in (
+select lower(email) from vdcuser 
+group by lower(email)
 having count(*) > 1
 )
 order by email
 
 -- Query to list all e-mails that have are duplicated (total = # of actual users, without duplicates)
-select email, count(*) from vdcuser
-group by email
+select lower(email), count(*) from vdcuser
+group by lower(email)
 having count(*) > 1
 order by count(*) desc
 
@@ -22,34 +22,34 @@ select u1.id, u1.username, u1.active,u1.email, u2.id, u2.username, u2.active
 from vdcuser u1, vdcuser u2
 where 1=1
 and u1.id != u2.id
-and u1.email = u2.email
-and u1.email in (
-select email from vdcuser
-group by email
+and lower(u1.email) = lower(u2.email)
+and lower(u1.email) in (
+select lower(email) from vdcuser
+group by lower(email)
 having count(*) > 1
 )
 and u2.id in (
 select min(id) from vdcuser
-group by email
+group by lower(email)
 having count(*) > 1
 )
-order by u1.email
+order by lower(u1.email)
 
 -- Delete query, to be run after all the updates
 delete from vdcuser where id in (
 select u1.id
 from vdcuser u1, vdcuser u2
 where 1=1
 and u1.id != u2.id
-and u1.email = u2.email
-and u1.email in (
-select email from vdcuser
-group by email
+and lower(u1.email) = lower(u2.email)
+and lower(u1.email) in (
+select lower(email) from vdcuser
+group by lower(email)
 having count(*) > 1
 )
 and u2.id in (
 select min(id) from vdcuser
-group by email
+group by lower(email)
 having count(*) > 1
 )
 )