From 867c19e7b084d833b674bf52889da4030108b94f Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Wed, 19 Jun 2024 08:37:10 +0200 Subject: [PATCH 01/12] use SQL's NOW() fct to log creation of a new user account (validated or not). --- qiita_db/support_files/populate_test_db.sql | 2 +- qiita_db/support_files/qiita-db-unpatched.sql | 10 ++++- qiita_db/test/test_user.py | 38 +++++++++++++++---- qiita_db/user.py | 6 +++ qiita_pet/handlers/user_handlers.py | 8 +++- qiita_pet/templates/user_profile.html | 3 ++ 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index 46c0aaed7..0a71959ac 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -50,7 +50,7 @@ INSERT INTO qiita.user_level VALUES (7, 'wet-lab admin', 'Can access the private -- Data for Name: qiita_user; Type: TABLE DATA; Schema: qiita; Owner: antoniog -- -INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false, '0000-0002-0975-9019', 'Rob-Knight', '_e3QL94AAAAJ'); +INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false, '0000-0002-0975-9019', 'Rob-Knight', '_e3QL94AAAAJ', '2015-12-03 13:52:42.751331-07'); INSERT INTO qiita.qiita_user VALUES ('shared@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false); INSERT INTO qiita.qiita_user VALUES ('admin@foo.bar', 1, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Admin', 'Owner University', '312 noname st, Apt K, Nonexistantown, CO 80302', '222-444-6789', NULL, NULL, NULL, false); INSERT INTO qiita.qiita_user VALUES ('demo@microbio.me', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Demo', 'Qiita Dev', '1345 Colorado Avenue', '303-492-1984', NULL, NULL, NULL, false); diff --git a/qiita_db/support_files/qiita-db-unpatched.sql b/qiita_db/support_files/qiita-db-unpatched.sql index a61b4645d..137e72963 100644 --- a/qiita_db/support_files/qiita-db-unpatched.sql +++ b/qiita_db/support_files/qiita-db-unpatched.sql @@ -1891,7 +1891,8 @@ CREATE TABLE qiita.qiita_user ( receive_processing_job_emails boolean DEFAULT false, social_orcid character varying DEFAULT NULL, social_researchgate character varying DEFAULT NULL, - social_googlescholar character varying DEFAULT NULL + social_googlescholar character varying DEFAULT NULL, + creation_timestamp timestamp without time zone DEFAULT NULL ); @@ -1931,6 +1932,13 @@ COMMENT ON COLUMN qiita.qiita_user.pass_reset_code IS 'Randomly generated code f COMMENT ON COLUMN qiita.qiita_user.pass_reset_timestamp IS 'Time the reset code was generated'; +-- +-- Name: COLUMN qiita_user.creation_timestamp; Type: COMMENT; Schema: qiita +-- + +COMMENT ON COLUMN qiita.qiita_user.creation_timestamp IS 'The date the user account was created'; + + -- -- Name: reference; Type: TABLE; Schema: qiita -- diff --git a/qiita_db/test/test_user.py b/qiita_db/test/test_user.py index 666746c36..f5d54ab3d 100644 --- a/qiita_db/test/test_user.py +++ b/qiita_db/test/test_user.py @@ -75,7 +75,8 @@ def setUp(self): 'receive_processing_job_emails': True, 'social_orcid': None, 'social_researchgate': None, - 'social_googlescholar': None + 'social_googlescholar': None, + 'creation_timestamp': None } def tearDown(self): @@ -88,7 +89,22 @@ def test_instantiate_unknown_user(self): with self.assertRaises(qdb.exceptions.QiitaDBUnknownIDError): qdb.user.User('FAIL@OMG.bar') - def _check_correct_info(self, obs, exp): + def _check_correct_info(self, obs, exp, ts_before=None): + """Compares info dict of user with special handling of specific keys. + + Parameters + ---------- + obs : dict + Observed user info dictionary. + exp : dict + Expected user info dictionary. + ts_before : datetime.datetime or None + User.create records the creation timestamp through SQL's NOW(). + Since it is set by the database to the microsecond, we can't + predict it a priori and therefore simply record timestamp before + execution of user.create() and compare the relation. + The DB creation_timestamp column is optional, i.e. can be None. + """ self.assertEqual(set(exp.keys()), set(obs.keys())) for key in exp: # user_verify_code and password seed randomly generated so just @@ -97,10 +113,14 @@ def _check_correct_info(self, obs, exp): self.assertEqual(len(obs[key]), 20) elif key == "password": self.assertEqual(len(obs[key]), 60) + elif key == "creation_timestamp": + self.assertTrue(((exp[key] is None) and (obs[key] is None)) + or (ts_before <= exp[key])) else: self.assertEqual(obs[key], exp[key]) def test_create_user(self): + before = datetime.now() user = qdb.user.User.create('testcreateuser@test.bar', 'password') # adding a couple of messages @@ -131,8 +151,9 @@ def test_create_user(self): 'email': 'testcreateuser@test.bar', 'social_orcid': None, 'social_researchgate': None, - 'social_googlescholar': None} - self._check_correct_info(obs, exp) + 'social_googlescholar': None, + 'creation_timestamp': datetime.now()} + self._check_correct_info(obs, exp, before) # Make sure new system messages are linked to user sql = """SELECT message_id FROM qiita.message_user @@ -146,6 +167,7 @@ def test_create_user(self): qdb.util.clear_system_messages() def test_create_user_info(self): + before = datetime.now() user = qdb.user.User.create('testcreateuserinfo@test.bar', 'password', self.userinfo) self.assertEqual(user.id, 'testcreateuserinfo@test.bar') @@ -171,8 +193,9 @@ def test_create_user_info(self): 'email': 'testcreateuserinfo@test.bar', 'social_orcid': None, 'social_researchgate': None, - 'social_googlescholar': None} - self._check_correct_info(obs, exp) + 'social_googlescholar': None, + 'creation_timestamp': datetime.now()} + self._check_correct_info(obs, exp, before) def test_create_user_column_not_allowed(self): self.userinfo["email"] = "FAIL" @@ -241,7 +264,8 @@ def test_get_info(self): 'phone': '222-444-6789', 'social_orcid': None, 'social_researchgate': None, - 'social_googlescholar': None + 'social_googlescholar': None, + 'creation_timestamp': None } self.assertEqual(self.user.info, expinfo) diff --git a/qiita_db/user.py b/qiita_db/user.py index b4beac184..094d48d57 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -234,6 +234,12 @@ def create(cls, email, password, info=None): cls._table, ','.join(columns), ','.join(['%s'] * len(values))) qdb.sql_connection.TRN.add(sql, values) + # log timestamp of user creation + sql = """UPDATE qiita.{0} + SET creation_timestamp = NOW() + WHERE email = %s""".format(cls._table) + qdb.sql_connection.perform_as_transaction(sql, [email]) + # Add system messages to user sql = """INSERT INTO qiita.message_user (email, message_id) SELECT %s, message_id FROM qiita.message diff --git a/qiita_pet/handlers/user_handlers.py b/qiita_pet/handlers/user_handlers.py index d75316a80..cd65f18c7 100644 --- a/qiita_pet/handlers/user_handlers.py +++ b/qiita_pet/handlers/user_handlers.py @@ -196,7 +196,9 @@ class UserProfileHandler(BaseHandler): def get(self): profile = UserProfile() profile.process(data=self.current_user.info) - self.render("user_profile.html", profile=profile, msg="", passmsg="") + self.render("user_profile.html", profile=profile, msg="", passmsg="", + creation_timestamp=self.current_user.info[ + 'creation_timestamp']) @authenticated @execute_as_transaction @@ -248,7 +250,9 @@ def post(self): else: passmsg = "Incorrect old password" self.render("user_profile.html", user=user.id, profile=form_data, - msg=msg, passmsg=passmsg) + msg=msg, passmsg=passmsg, + creation_timestamp=self.current_user.info[ + 'creation_timestamp']) class ForgotPasswordHandler(BaseHandler): diff --git a/qiita_pet/templates/user_profile.html b/qiita_pet/templates/user_profile.html index 66da290d9..cb8519f75 100644 --- a/qiita_pet/templates/user_profile.html +++ b/qiita_pet/templates/user_profile.html @@ -27,6 +27,9 @@

User Information

{% end %} {% end %} + {%if creation_timestamp is not None %} +
account created at {{creation_timestamp}}
+ {% end %}
{{msg}}
From c94ccb92ca814594165b631c782303c2c8ffcead Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Wed, 19 Jun 2024 08:43:53 +0200 Subject: [PATCH 02/12] at -> on --- qiita_pet/templates/user_profile.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_pet/templates/user_profile.html b/qiita_pet/templates/user_profile.html index cb8519f75..5f5336d95 100644 --- a/qiita_pet/templates/user_profile.html +++ b/qiita_pet/templates/user_profile.html @@ -28,7 +28,7 @@

User Information

{% end %} {%if creation_timestamp is not None %} -
account created at {{creation_timestamp}}
+
account created on {{creation_timestamp}}
{% end %}
{{msg}}
From ca55a2fc99f9d9d1282646b3be96295d57cad3fe Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Wed, 19 Jun 2024 09:07:37 +0200 Subject: [PATCH 03/12] add new col --- qiita_db/test/test_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 26fff3004..57afd6b4d 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -93,7 +93,8 @@ def test_get_table_cols(self): exp = {"email", "user_level_id", "password", "name", "affiliation", "address", "phone", "user_verify_code", "pass_reset_code", "pass_reset_timestamp", "receive_processing_job_emails", - "social_orcid", "social_researchgate", "social_googlescholar"} + "social_orcid", "social_researchgate", "social_googlescholar", + "creation_timestamp"} self.assertEqual(set(obs), exp) def test_exists_table(self): From b8f959a99aa5c0a3ed14016da2bfa9a4649951c6 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Wed, 19 Jun 2024 16:36:09 +0200 Subject: [PATCH 04/12] adding a new page for admins to easily list all users not yet validated and offer means to delete those --- qiita_db/support_files/populate_test_db.sql | 3 + qiita_pet/handlers/user_handlers.py | 73 +++++++++++++++++++++ qiita_pet/templates/sitebase.html | 1 + qiita_pet/test/test_user_handlers.py | 33 +++++++++- qiita_pet/webserver.py | 4 +- 5 files changed, 112 insertions(+), 2 deletions(-) diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index 0a71959ac..94f803ce3 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -54,6 +54,9 @@ INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v INSERT INTO qiita.qiita_user VALUES ('shared@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false); INSERT INTO qiita.qiita_user VALUES ('admin@foo.bar', 1, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Admin', 'Owner University', '312 noname st, Apt K, Nonexistantown, CO 80302', '222-444-6789', NULL, NULL, NULL, false); INSERT INTO qiita.qiita_user VALUES ('demo@microbio.me', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Demo', 'Qiita Dev', '1345 Colorado Avenue', '303-492-1984', NULL, NULL, NULL, false); +INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW()); +INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '1 YEAR'); +INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '30 DAY'); -- diff --git a/qiita_pet/handlers/user_handlers.py b/qiita_pet/handlers/user_handlers.py index cd65f18c7..faa0d925d 100644 --- a/qiita_pet/handlers/user_handlers.py +++ b/qiita_pet/handlers/user_handlers.py @@ -7,6 +7,8 @@ # ----------------------------------------------------------------------------- import re +from json import dumps +import warnings from tornado.web import authenticated, HTTPError from wtforms import Form, StringField, BooleanField, validators @@ -14,6 +16,8 @@ from qiita_pet.handlers.base_handlers import BaseHandler from qiita_pet.handlers.api_proxy import user_jobs_get_req +from qiita_pet.handlers.portal import PortalEditBase +import qiita_db as qdb from qiita_db.util import send_email from qiita_db.user import User from qiita_db.logger import LogEntry @@ -375,3 +379,72 @@ class UserJobs(BaseHandler): def get(self): response = user_jobs_get_req(self.current_user) self.write(response) + + +class PurgeUsersAJAXHandler(PortalEditBase): + # define columns besides email that will be displayed on website + FIELDS = ['name', 'affiliation', 'address', 'phone', + 'creation_timestamp'] + @authenticated + @execute_as_transaction + def get(self): + # retrieving users not yet verified + self.check_admin() + with qdb.sql_connection.TRN: + sql = """SELECT email,{0} + FROM qiita.qiita_user + WHERE (user_level_id=5) AND + (creation_timestamp < (NOW() - INTERVAL '30 DAY')) + """.format(','.join(self.FIELDS)) + qdb.sql_connection.TRN.add(sql) + users = qdb.sql_connection.TRN.execute()[1:] + + # fetching information for each user + result = [] + for list in users: + for user in list: + usermail = user[0] + user_unit = {'email': usermail} + user_infos = User(usermail).info + for col in self.FIELDS: + user_unit[col] = str(user_infos[col]) + result.append(user_unit) + # returning information as JSON + self.write(dumps(result, separators=(',', ':'))) + + +class PurgeUsersHandler(PortalEditBase): + @authenticated + @execute_as_transaction + def get(self): + # render page and transfer headers to be included for the table + self.check_admin() + self.render('admin_purge_users.html', + headers=['email'] + PurgeUsersAJAXHandler.FIELDS, + submit_url="/admin/purge_users/") + + def post(self): + # check if logged in user is admin and fetch all checked boxes as well + # as the action + self.check_admin() + users = map(str, self.get_arguments('selected')) + action = self.get_argument('action') + + # depending on the action delete user from db (remove) + num_deleted_user = 0 + for user in users: + try: + with warnings.catch_warnings(record=True) as warns: + if action == "Remove": + user_to_delete = User(user) + user_to_delete.delete(user) + num_deleted_user += 1 + else: + raise HTTPError( + 400, reason="Unknown action: %s" % action) + except QiitaDBError as e: + self.write(action.upper() + " ERROR:
" + str(e)) + return + msg = '; '.join([str(w.message) for w in warns]) + self.write(("%i non-validated user(s) successfully removed from " + "database
%s") % (num_deleted_user, msg)) diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 5fcc0db12..7165f33a2 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -383,6 +383,7 @@
  • View Errors
  • View Studies awaiting approval
  • Edit study portal connections
    • +
  • Purge non-validated users
    • {% end %}
  • Sample Validation
  • Processing Jobs
    • diff --git a/qiita_pet/test/test_user_handlers.py b/qiita_pet/test/test_user_handlers.py index 42fd46d8a..bf59e53a7 100644 --- a/qiita_pet/test/test_user_handlers.py +++ b/qiita_pet/test/test_user_handlers.py @@ -9,10 +9,14 @@ from unittest import main from wtforms.validators import ValidationError from wtforms import StringField +from mock import Mock +from json import loads from qiita_pet.test.tornado_test_base import TestHandlerBase from qiita_pet.handlers.user_handlers import UserProfile - +from qiita_pet.handlers.base_handlers import BaseHandler +import qiita_db as qdb +from qiita_db.user import User class TestUserProfile(TestHandlerBase): # TODO: add proper test for this once figure out how. Issue 567 @@ -124,5 +128,32 @@ def test_get(self): self.assertEqual(response.code, 200) +class TestPurgeUsersAJAXHandler(TestHandlerBase): + def setUp(self): + super().setUp() + BaseHandler.get_current_user = Mock(return_value=User("admin@foo.bar")) + + def test_get(self): + response = self.get('/admin/purge_usersAjax/?_=1718805487494') + obs_users_table = loads(response.body.decode('ascii')) + obs_users = {user['email'] for user in obs_users_table} + self.assertIn('ayearago@nonvalidat.ed', obs_users) + self.assertIn('3Xdays@nonvalidat.ed', obs_users) + self.assertNotIn('justnow@nonvalidat.ed', obs_users) + + def test_post_removeBoth(self): + # remove both users + response = self.post('/admin/purge_users/', + {'action': 'Remove', + 'selected': ['ayearago@nonvalidat.ed', + '3Xdays@nonvalidat.ed']}) + self.assertEqual(response.code, 200) + + # test that zero users are listed now + response = self.get('/admin/purge_usersAjax/?_=1718805487495') + obs_users_table = loads(response.body.decode('ascii')) + self.assertEqual(obs_users_table, []) + + if __name__ == "__main__": main() diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 84d71431a..50b34fd2b 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -23,7 +23,7 @@ AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler) from qiita_pet.handlers.user_handlers import ( ChangeForgotPasswordHandler, ForgotPasswordHandler, UserProfileHandler, - UserMessagesHander, UserJobs) + UserMessagesHander, UserJobs, PurgeUsersAJAXHandler, PurgeUsersHandler) from qiita_pet.handlers.admin_processing_job import ( AdminProcessingJob, AJAXAdminProcessingJobListing, SampleValidation) from qiita_pet.handlers.analysis_handlers import ( @@ -134,6 +134,8 @@ def __init__(self): (r"/admin/processing_jobs/", AdminProcessingJob), (r"/admin/processing_jobs/list", AJAXAdminProcessingJobListing), (r"/admin/sample_validation/", SampleValidation), + (r"/admin/purge_users/", PurgeUsersHandler), + (r"/admin/purge_usersAjax/", PurgeUsersAJAXHandler), (r"/ebi_submission/(.*)", EBISubmitHandler), # Study handlers (r"/study/create/", StudyEditHandler), From 09e1a509e6658f7c13cc5b7cfc6960e0c82f7782 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Wed, 19 Jun 2024 16:41:41 +0200 Subject: [PATCH 05/12] flake8 --- qiita_pet/handlers/user_handlers.py | 3 ++- qiita_pet/test/test_user_handlers.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qiita_pet/handlers/user_handlers.py b/qiita_pet/handlers/user_handlers.py index faa0d925d..c85677cb8 100644 --- a/qiita_pet/handlers/user_handlers.py +++ b/qiita_pet/handlers/user_handlers.py @@ -385,6 +385,7 @@ class PurgeUsersAJAXHandler(PortalEditBase): # define columns besides email that will be displayed on website FIELDS = ['name', 'affiliation', 'address', 'phone', 'creation_timestamp'] + @authenticated @execute_as_transaction def get(self): @@ -397,7 +398,7 @@ def get(self): (creation_timestamp < (NOW() - INTERVAL '30 DAY')) """.format(','.join(self.FIELDS)) qdb.sql_connection.TRN.add(sql) - users = qdb.sql_connection.TRN.execute()[1:] + users = qdb.sql_connection.TRN.execute()[1:] # fetching information for each user result = [] diff --git a/qiita_pet/test/test_user_handlers.py b/qiita_pet/test/test_user_handlers.py index bf59e53a7..a47729ad7 100644 --- a/qiita_pet/test/test_user_handlers.py +++ b/qiita_pet/test/test_user_handlers.py @@ -15,9 +15,9 @@ from qiita_pet.test.tornado_test_base import TestHandlerBase from qiita_pet.handlers.user_handlers import UserProfile from qiita_pet.handlers.base_handlers import BaseHandler -import qiita_db as qdb from qiita_db.user import User + class TestUserProfile(TestHandlerBase): # TODO: add proper test for this once figure out how. Issue 567 pass From cf92fa479685251c7e0d579c804cdd435eec6384 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Wed, 19 Jun 2024 17:04:34 +0200 Subject: [PATCH 06/12] fix tests --- qiita_db/handlers/tests/test_user.py | 5 ++++- qiita_db/test/test_meta_util.py | 2 +- qiita_db/test/test_setup.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/qiita_db/handlers/tests/test_user.py b/qiita_db/handlers/tests/test_user.py index 38831471b..3f1102ec5 100644 --- a/qiita_db/handlers/tests/test_user.py +++ b/qiita_db/handlers/tests/test_user.py @@ -54,7 +54,10 @@ def test_get(self): {'email': 'test@foo.bar', 'name': 'Dude'}, {'email': 'shared@foo.bar', 'name': 'Shared'}, {'email': 'admin@foo.bar', 'name': 'Admin'}, - {'email': 'demo@microbio.me', 'name': 'Demo'}]} + {'email': 'demo@microbio.me', 'name': 'Demo'}, + {'email': 'justnow@nonvalidat.ed', 'name': 'JustNow'}, + {'email': 'ayearago@nonvalidat.ed', 'name': 'Oldie'}, + {'email': '3Xdays@nonvalidat.ed', 'name': 'TooLate'}]} self.assertEqual(obs, exp) diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index 8bfe05d9a..e8fe4c0de 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -281,7 +281,7 @@ def _get_daily_stats(): self.assertDictEqual(f(redis_key), exp) # then the unique values vals = [ - ('num_users', b'4', r_client.get), + ('num_users', b'7', r_client.get), ('lat_longs', b'[]', r_client.get), ('num_studies_ebi', b'1', r_client.get), ('num_samples_ebi', b'27', r_client.get), diff --git a/qiita_db/test/test_setup.py b/qiita_db/test/test_setup.py index 3cdabe255..70825fe59 100644 --- a/qiita_db/test/test_setup.py +++ b/qiita_db/test/test_setup.py @@ -15,7 +15,7 @@ class SetupTest(TestCase): """Tests that the test database have been successfully populated""" def test_qiita_user(self): - self.assertEqual(get_count("qiita.qiita_user"), 4) + self.assertEqual(get_count("qiita.qiita_user"), 7) def test_study_person(self): self.assertEqual(get_count("qiita.study_person"), 3) From cf462aad194c382f871d72e84f833f6d4b889f90 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Thu, 20 Jun 2024 13:04:06 +0200 Subject: [PATCH 07/12] move DB changes to patch --- qiita_db/support_files/patches/92.sql | 7 +++++++ qiita_db/support_files/populate_test_db.sql | 7 ++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql index 2162533d4..c37472dde 100644 --- a/qiita_db/support_files/patches/92.sql +++ b/qiita_db/support_files/patches/92.sql @@ -43,3 +43,10 @@ COMMENT ON COLUMN qiita.qiita_user.creation_timestamp IS 'The date the user acco -- for testing: provide creation date for one of the existing users UPDATE qiita.qiita_user SET creation_timestamp = '2015-12-03 13:52:42.751331-07' WHERE email = 'test@foo.bar'; + +-- Jun 20, 2024 +-- Add some non-verified users to the test DB to test new admin page: /admin/purge_users/ + +INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW()); +INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '1 YEAR'); +INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '30 DAY'); diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index 9477b10c0..cb569eac8 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -50,13 +50,10 @@ INSERT INTO qiita.user_level VALUES (7, 'wet-lab admin', 'Can access the private -- Data for Name: qiita_user; Type: TABLE DATA; Schema: qiita; Owner: antoniog -- -INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false, '0000-0002-0975-9019', 'Rob-Knight', '_e3QL94AAAAJ', '2015-12-03 13:52:42.751331-07'); +INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false, '0000-0002-0975-9019', 'Rob-Knight', '_e3QL94AAAAJ'); INSERT INTO qiita.qiita_user VALUES ('shared@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false); INSERT INTO qiita.qiita_user VALUES ('admin@foo.bar', 1, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Admin', 'Owner University', '312 noname st, Apt K, Nonexistantown, CO 80302', '222-444-6789', NULL, NULL, NULL, false); INSERT INTO qiita.qiita_user VALUES ('demo@microbio.me', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Demo', 'Qiita Dev', '1345 Colorado Avenue', '303-492-1984', NULL, NULL, NULL, false); -INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW()); -INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '1 YEAR'); -INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '30 DAY'); -- @@ -332,7 +329,7 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin -- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog -- -INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757); +INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757); INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546); INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377); INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986); From 73b66bebfed4cd4ed37b561769d422cf8d1b9825 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Thu, 20 Jun 2024 13:04:55 +0200 Subject: [PATCH 08/12] revert file --- qiita_db/support_files/populate_test_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index cb569eac8..1ca2edb66 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -329,7 +329,7 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin -- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog -- -INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757); +INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757); INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546); INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377); INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986); From 2de7acc9e73138ef02c5729fc7b24abc326d166b Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Thu, 20 Jun 2024 13:06:00 +0200 Subject: [PATCH 09/12] no change here --- qiita_db/support_files/qiita-db-unpatched.sql | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/qiita_db/support_files/qiita-db-unpatched.sql b/qiita_db/support_files/qiita-db-unpatched.sql index 137e72963..a61b4645d 100644 --- a/qiita_db/support_files/qiita-db-unpatched.sql +++ b/qiita_db/support_files/qiita-db-unpatched.sql @@ -1891,8 +1891,7 @@ CREATE TABLE qiita.qiita_user ( receive_processing_job_emails boolean DEFAULT false, social_orcid character varying DEFAULT NULL, social_researchgate character varying DEFAULT NULL, - social_googlescholar character varying DEFAULT NULL, - creation_timestamp timestamp without time zone DEFAULT NULL + social_googlescholar character varying DEFAULT NULL ); @@ -1932,13 +1931,6 @@ COMMENT ON COLUMN qiita.qiita_user.pass_reset_code IS 'Randomly generated code f COMMENT ON COLUMN qiita.qiita_user.pass_reset_timestamp IS 'Time the reset code was generated'; --- --- Name: COLUMN qiita_user.creation_timestamp; Type: COMMENT; Schema: qiita --- - -COMMENT ON COLUMN qiita.qiita_user.creation_timestamp IS 'The date the user account was created'; - - -- -- Name: reference; Type: TABLE; Schema: qiita -- From 1e7ad502b1bcf82d3832daf05dec19b78da017b1 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Thu, 20 Jun 2024 13:06:57 +0200 Subject: [PATCH 10/12] execute merge manually --- qiita_db/user.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qiita_db/user.py b/qiita_db/user.py index 094d48d57..b4beac184 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -234,12 +234,6 @@ def create(cls, email, password, info=None): cls._table, ','.join(columns), ','.join(['%s'] * len(values))) qdb.sql_connection.TRN.add(sql, values) - # log timestamp of user creation - sql = """UPDATE qiita.{0} - SET creation_timestamp = NOW() - WHERE email = %s""".format(cls._table) - qdb.sql_connection.perform_as_transaction(sql, [email]) - # Add system messages to user sql = """INSERT INTO qiita.message_user (email, message_id) SELECT %s, message_id FROM qiita.message From ec9000339ae1e215d02184a31788efb63e3a8861 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 20 Jun 2024 14:24:24 +0200 Subject: [PATCH 11/12] moving data insertion to test_db_sql patch --- qiita_db/support_files/patches/92.sql | 11 ----------- qiita_db/support_files/patches/test_db_sql/92.sql | 13 ++++++++++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql index c37472dde..19d284b36 100644 --- a/qiita_db/support_files/patches/92.sql +++ b/qiita_db/support_files/patches/92.sql @@ -39,14 +39,3 @@ ALTER TABLE qiita.qiita_user ADD creation_timestamp timestamp without time zone DEFAULT NOW(); COMMENT ON COLUMN qiita.qiita_user.creation_timestamp IS 'The date the user account was created'; - --- for testing: provide creation date for one of the existing users - -UPDATE qiita.qiita_user SET creation_timestamp = '2015-12-03 13:52:42.751331-07' WHERE email = 'test@foo.bar'; - --- Jun 20, 2024 --- Add some non-verified users to the test DB to test new admin page: /admin/purge_users/ - -INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW()); -INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '1 YEAR'); -INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '30 DAY'); diff --git a/qiita_db/support_files/patches/test_db_sql/92.sql b/qiita_db/support_files/patches/test_db_sql/92.sql index 0e9709f95..65266e9f0 100644 --- a/qiita_db/support_files/patches/test_db_sql/92.sql +++ b/qiita_db/support_files/patches/test_db_sql/92.sql @@ -925,4 +925,15 @@ INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, columns ('61da73ff-b4ff-49a1-b775-c6215cfbd291', 231, 107, 2, 'nan', 333544000, 200, '2023-02-17T15:05:17', NULL, NULL), ('6c84dcf1-c5ea-4e69-b17f-d2d5b8d48bdf', 123, 50, 2, 'nan', 327520000, 82, '2023-02-18T15:13:15', NULL, NULL), ('dcb12603-4142-44d1-9a52-3ca3511e380e', 320, 44, 2, 'nan', 329448000, 475, '2023-02-19T06:29:32', NULL, NULL), -('a0dd0a4d-b73f-4e9d-87dd-d29efba25336', 41, 50, 2, 'nan', 301108000, 144, '2023-02-19T09:14:27', NULL, NULL); \ No newline at end of file +('a0dd0a4d-b73f-4e9d-87dd-d29efba25336', 41, 50, 2, 'nan', 301108000, 144, '2023-02-19T09:14:27', NULL, NULL); + +-- for testing: provide creation date for one of the existing users + +UPDATE qiita.qiita_user SET creation_timestamp = '2015-12-03 13:52:42.751331-07' WHERE email = 'test@foo.bar'; + +-- Jun 20, 2024 +-- Add some non-verified users to the test DB to test new admin page: /admin/purge_users/ + +INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW()); +INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '1 YEAR'); +INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '30 DAY'); From 71b97f3e729b34bb07e013a48c0e775a265dafb6 Mon Sep 17 00:00:00 2001 From: sjanssen2 Date: Thu, 20 Jun 2024 16:35:16 +0200 Subject: [PATCH 12/12] account for additional test users --- qiita_db/test/test_portal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiita_db/test/test_portal.py b/qiita_db/test/test_portal.py index 2a5bb5413..59b277a42 100644 --- a/qiita_db/test/test_portal.py +++ b/qiita_db/test/test_portal.py @@ -47,7 +47,8 @@ def test_add_portal(self): qdb.sql_connection.TRN.add("SELECT * FROM qiita.analysis_portal") obs = qdb.sql_connection.TRN.execute_fetchindex() exp = [[1, 1], [2, 1], [3, 1], [4, 1], [5, 1], [6, 1], [7, 2], [8, 2], - [9, 2], [10, 2], [11, 4], [12, 4], [13, 4], [14, 4]] + [9, 2], [10, 2], [11, 4], [12, 4], [13, 4], [14, 4], + [15, 4], [16, 4], [17, 4]] self.assertCountEqual(obs, exp) with self.assertRaises(qdb.exceptions.QiitaDBDuplicateError):