Merge pull request #718 from topcoder-platform/feature/tsjr-313-gener…

…ate-tcUserId-for-existing-job_candidates Enhance scripts for data processing
topcoder-platform · Jan 31, 2024 · 4f4f849 · 4f4f849
2 parents db19927 + 5f4d15d
commit 4f4f849
Show file tree

Hide file tree

Showing 2 changed files with 87 additions and 36 deletions.
diff --git a/scripts/alter-bookings-db-tcUserId.sql b/scripts/alter-bookings-db-tcUserId.sql
@@ -28,10 +28,6 @@ ALTER COLUMN member_id TYPE TEXT,
 ALTER COLUMN created_by TYPE TEXT,
 ALTER COLUMN updated_by TYPE TEXT;
 
-ALTER TABLE bookings.roles
-ALTER COLUMN created_by TYPE TEXT,
-ALTER COLUMN updated_by TYPE TEXT;
-
 ALTER TABLE bookings.user_meeting_settings
 ALTER COLUMN created_by TYPE TEXT,
 ALTER COLUMN updated_by TYPE TEXT;

diff --git a/scripts/process-remaining-ubahn-uuids.js b/scripts/process-remaining-ubahn-uuids.js
@@ -3,66 +3,121 @@ const tcUserId = require('./common/tcUserId')
 const Sequelize = require('sequelize')
 const _ = require('lodash')
 const helper = require('../src/common/helper')
+const request = require('superagent')
+const config = require('config')
 
-// read cli arguments, pass the table name
-const tableName = process.argv[2]
-// read cli arguments, pass the column names to update
-const columnNames = process.argv[3]
+/**
+ *
+ * @param {string} handle - The member handle
+ * @returns {Promise<object>} The member details from the member API
+ */
+async function getMemberDetailsByHandle (handle) {
+  const token = await helper.getM2MToken()
+  let res
+  try {
+    res = await request
+      .get(`${config.TOPCODER_MEMBERS_API}/${handle}`)
+      .query({
+        fields: 'userId,handle,handleLower'
+      })
+      .set('Authorization', `Bearer ${token}`)
+      .set('Accept', 'application/json')
+    return res.body
+  } catch (error) {
+    console.log(`Unable to find member with handle ${handle}`)
+    return {}
+  }
+}
 
 const processRemainingUUIDs = async (tableName, columnNames) => {
   const dbUrl = process.env.UBAHN_DB_URL
   const MODE = process.env.MODE || 'test'
+  const handleToIDMap = {}
 
   if (_.isUndefined(dbUrl) || _.isNull(dbUrl)) {
     console.log('Ubahn DB URL not set, exiting!')
     process.exit(0)
   }
 
+  // only for readability
+  console.log()
+  console.log('---------------------------------------------------------------------------------------------------------------')
+  console.log()
+
   for (const columnName of _.split(columnNames, ',')) {
-    const query = `SELECT DISTINCT ${columnName} FROM bookings.${tableName} WHERE LENGTH(${columnName}) > 9 AND ${columnName} <> '00000000-0000-0000-0000-000000000000';`
+    const transaction = await models.sequelize.transaction()
 
-    console.log(`Executing query in table ${tableName} against column ${columnName}`)
-    console.log(`SQL query: ${query}`)
+    try {
+      // check each column to find distinct existing uuids which have not yet been converted to TC legacy user id
+      const query = `SELECT DISTINCT ${columnName} FROM bookings.${tableName} WHERE LENGTH(${columnName}) > 9 AND ${columnName} <> '00000000-0000-0000-0000-000000000000';`
+      console.log(`Executing query in table ${tableName} against column ${columnName}`)
+      console.log(`SQL query: ${query}`)
+      let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT })
 
-    let results = await models.sequelize.query(query, { type: Sequelize.QueryTypes.SELECT })
+      if (results.length > 0) {
+        results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`]))
+        console.log(`SQL query result: ${JSON.stringify(results)}`)
 
-    if (results.length > 0) {
-      results = _.uniq(_.map(_.filter(results, val => toString(val[`${columnName}`]).length > 9), val => val[`${columnName}`]))
-      console.log(`SQL query result: ${JSON.stringify(results)}`)
+        // get the ubahn uuid to handle map
+        const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl)
+        const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results)
 
-      const ubahnConn = await tcUserId.getUbahnDatabaseConnection(dbUrl)
-      const uuidToHandleMap = await tcUserId.getUserUbahnUUIDToHandleMap(ubahnConn, results)
+        // get the handle to legacy topcoder id map
+        for (const handle of Object.values(uuidToHandleMap)) {
+          console.log(`handle to search for ${handle}`)
+          if (_.isUndefined(handleToIDMap[handle])) {
+            const member = await getMemberDetailsByHandle(handle)
+            handleToIDMap[member.handleLower] = member.userId
+          }
+        }
 
-      const handleToIDMap = {}
-      const batches = _.chunk(Object.values(uuidToHandleMap), 30)
-      for (const batch of batches) {
-        console.log(`Batch of handles: ${JSON.stringify(batch)}`)
-        const memberAPIRes = await helper.getMemberDetailsByHandles(batch)
-        _.forEach(memberAPIRes, member => {
-          handleToIDMap[member.handleLower] = member.userId
-        })
-      }
+        // build the update queries
+        let sql = ''
+        for (const [key, value] of Object.entries(uuidToHandleMap)) {
+          if (!_.isUndefined(handleToIDMap[value.toLowerCase()])) {
+            sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';`
+          }
+        }
 
-      let sql = ''
-      for (const [key, value] of Object.entries(uuidToHandleMap)) {
-        if (!_.isUndefined(handleToIDMap[value.toLowerCase()])) {
-          sql += `UPDATE bookings.${tableName} SET ${columnName} = '${handleToIDMap[value.toLowerCase()]}' WHERE ${columnName} = '${key}';`
+        // execute update queries if and only if it's not in test mode
+        if (sql !== '') {
+          console.log(`SQL UPDATE statements: ${sql}`)
+          if (MODE !== 'test') {
+            console.log('Executing UPDATE statements')
+            await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE, transaction: transaction })
+          }
+        } else {
+          console.log(`No UPDATE statements to execute against column ${columnName}`)
         }
+      } else {
+        console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`)
       }
-      console.log(`SQL UPDATE statements: ${sql}`)
-      if (MODE !== test && sql.length > 0) {
-        await models.sequelize.query(sql, { type: Sequelize.QueryTypes.UPDATE })
-      }
-    } else {
-      console.log(`No data eligible to be updated for table: ${tableName} against column ${columnName}`)
+
+      // only for readability
+      console.log('---------------------------------------------------------------------------------------------------------------')
+      console.log()
+
+      await transaction.commit()
+    } catch (error) {
+      console.log('Error encountered')
+      console.error(JSON.stringify(error))
+      await transaction.rollback()
     }
   }
+
+  console.log(`DONE processing table ${tableName}`)
 }
 
+// read cli arguments, pass the table name
+const tableName = process.argv[2]
+// read cli arguments, pass the column names to update
+const columnNames = process.argv[3]
+
 processRemainingUUIDs(tableName, columnNames).then(res => {
   console.log(`Processed remaining records for model '${tableName}' against columns: ${columnNames}`)
   process.exit(0)
 }).catch(err => {
+  console.log('Error encountered!')
   console.error(`${JSON.stringify(err)}`)
   process.exit(1)
 })