Bug 1639284 - Store records, timestamp and metadata in one IndexedDB …

…transaction r=Gijs,mixedpuppy Differential Revision: https://phabricator.services.mozilla.com/D78246
dothq · Jun 18, 2020 · 986bc37 · 986bc37
1 parent e6c3203
commit 986bc37
Show file tree

Hide file tree

Showing 29 changed files with 332 additions and 315 deletions.
diff --git a/browser/components/aboutlogins/tests/browser/head.js b/browser/components/aboutlogins/tests/browser/head.js
@@ -88,7 +88,7 @@ add_task(async function setup_head() {
       useRecordId: true,
     });
   }
-  await db.saveLastModified(42);
+  await db.importChanges({}, 42);
   if (EXPECTED_BREACH) {
     await RemoteSettings(LoginBreaches.REMOTE_SETTINGS_COLLECTION).emit(
       "sync",

diff --git a/browser/components/aboutlogins/tests/unit/test_getPotentialBreachesByLoginGUID.js b/browser/components/aboutlogins/tests/unit/test_getPotentialBreachesByLoginGUID.js
@@ -282,8 +282,7 @@ add_task(async function test_setBreachesFromRemoteSettingsSync() {
   );
   gBrowserGlue.observe(null, "browser-glue-test", "add-breaches-sync-handler");
   const db = await RemoteSettings(LoginBreaches.REMOTE_SETTINGS_COLLECTION).db;
-  await db.create(nowExampleIsInBreachedRecords[0]);
-  await db.saveLastModified(42);
+  await db.importChanges({}, 42, [nowExampleIsInBreachedRecords[0]]);
   await emitSync();
 
   const breachesByLoginGUID = await LoginBreaches.getPotentialBreachesByLoginGUID(

diff --git a/browser/components/fxmonitor/test/browser/browser_fxmonitor_doorhanger.js b/browser/components/fxmonitor/test/browser/browser_fxmonitor_doorhanger.js
@@ -66,7 +66,7 @@ add_task(async function test_main_flow() {
       1}-${AddedDate.getDate()}`,
     PwnCount: 1000000,
   });
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
 
   // Trigger a sync.
   await RemoteSettings(kRemoteSettingsKey).emit("sync", {
@@ -103,7 +103,7 @@ add_task(async function test_main_flow() {
 
   // Reset state.
   await db.clear();
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
   await clearWarnedHosts();
   await SpecialPowers.pushPrefEnv({
     clear: [["extensions.fxmonitor.firstAlertShown"]],
@@ -133,7 +133,7 @@ add_task(async function test_main_flow() {
 
   // Reset state (but not firstAlertShown).
   await db.clear();
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
   await clearWarnedHosts();
 
   info(
@@ -151,7 +151,7 @@ add_task(async function test_main_flow() {
       1}-${AddedDate.getDate()}`,
     PwnCount: 1000000,
   });
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
 
   // Trigger a sync.
   await RemoteSettings(kRemoteSettingsKey).emit("sync", {
@@ -169,7 +169,7 @@ add_task(async function test_main_flow() {
   // Reset state (but not firstAlertShown).
   AddedDate.setMonth(AddedDate.getMonth() + 3);
   await db.clear();
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
   await clearWarnedHosts();
 
   info("Test that we do show the second alert for a recent breach.");
@@ -184,7 +184,7 @@ add_task(async function test_main_flow() {
       1}-${AddedDate.getDate()}`,
     PwnCount: 1000000,
   });
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
 
   // Trigger a sync.
   await RemoteSettings(kRemoteSettingsKey).emit("sync", {
@@ -201,7 +201,7 @@ add_task(async function test_main_flow() {
 
   // Reset state (including firstAlertShown)
   await db.clear();
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
   await clearWarnedHosts();
   await SpecialPowers.pushPrefEnv({
     clear: [["extensions.fxmonitor.firstAlertShown"]],
@@ -222,7 +222,7 @@ add_task(async function test_main_flow() {
       1}-${AddedDate.getDate()}`,
     PwnCount: 1000000,
   });
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
 
   // Trigger a sync.
   await RemoteSettings(kRemoteSettingsKey).emit("sync", {
@@ -240,7 +240,7 @@ add_task(async function test_main_flow() {
   // Clean up.
   BrowserTestUtils.removeTab(tab);
   await db.clear();
-  await db.saveLastModified(1234567);
+  await db.importChanges({}, 1234567);
   // Trigger a sync to clear.
   await RemoteSettings(kRemoteSettingsKey).emit("sync", {
     data: {

diff --git a/browser/components/newtab/test/browser/browser_asrouter_experimentsAPILoader.js b/browser/components/newtab/test/browser/browser_asrouter_experimentsAPILoader.js
@@ -135,13 +135,15 @@ add_task(async function test_loading_experimentsAPI() {
     ],
   });
   const client = RemoteSettings("messaging-experiments");
-  await client.db.clear();
-  await client.db.create(
-    // Modify targeting to ensure the messages always show up
-    { ...EXPERIMENT_PAYLOAD }
+  await client.db.importChanges(
+    {},
+    42,
+    [
+      // Modify targeting to ensure the messages always show up
+      { ...EXPERIMENT_PAYLOAD },
+    ],
+    { clear: true }
   );
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
-
   // Fetch the new recipe from RS
   await RemoteSettingsExperimentLoader.updateRecipes();
   await BrowserTestUtils.waitForCondition(

diff --git a/browser/components/newtab/test/browser/browser_asrouter_group_frequency.js b/browser/components/newtab/test/browser/browser_asrouter_group_frequency.js
@@ -35,9 +35,9 @@ add_task(async function setup() {
     id: `HEARTBEAT_MESSAGE_${Date.now()}`,
   };
   const client = RemoteSettings("cfr");
-  await client.db.clear();
-  await client.db.create(testMessage);
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
+  await client.db.importChanges({}, 42, [testMessage], {
+    clear: true,
+  });
 
   // Reload the providers
   await BrowserTestUtils.waitForCondition(async () => {
@@ -93,9 +93,9 @@ add_task(async function test_heartbeat_tactic_2() {
     frequency: { lifetime: 2 },
   };
   const client = RemoteSettings("message-groups");
-  await client.db.clear();
-  await client.db.create(groupConfiguration);
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
+  await client.db.importChanges({}, 42, [groupConfiguration], {
+    clear: true,
+  });
 
   // Reload the providers
   await ASRouter._updateMessageProviders();

diff --git a/browser/components/newtab/test/browser/browser_asrouter_group_userprefs.js b/browser/components/newtab/test/browser/browser_asrouter_group_userprefs.js
@@ -35,9 +35,7 @@ add_task(async function setup() {
     id: `HEARTBEAT_MESSAGE_${Date.now()}`,
   };
   const client = RemoteSettings("cfr");
-  await client.db.clear();
-  await client.db.create(testMessage);
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
+  await client.db.importChanges({}, 42, [testMessage], { clear: true });
 
   // Reload the providers
   await BrowserTestUtils.waitForCondition(async () => {
@@ -89,9 +87,7 @@ add_task(async function test_heartbeat_tactic_2() {
     userPreferences: ["browser.userPreference.messaging-experiments"],
   };
   const client = RemoteSettings("message-groups");
-  await client.db.clear();
-  await client.db.create(groupConfiguration);
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
+  await client.db.importChanges({}, 42, [groupConfiguration], { clear: true });
 
   // Reload the providers
   await ASRouter._updateMessageProviders();

diff --git a/browser/components/newtab/test/browser/browser_asrouter_momentspagehub.js b/browser/components/newtab/test/browser/browser_asrouter_momentspagehub.js
@@ -28,16 +28,20 @@ add_task(async function test_with_rs_messages() {
   );
   const initialMessageCount = ASRouter.state.messages.length;
   const client = RemoteSettings("cfr");
-  await client.db.clear();
-  await client.db.create({
-    // Modify targeting and randomize message name to work around the message
-    // getting blocked (for --verify)
-    ...msg,
-    id: `MOMENTS_MOCHITEST_${Date.now()}`,
-    targeting: "true",
-  });
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
-
+  await client.db.importChanges(
+    {},
+    42,
+    [
+      {
+        // Modify targeting and randomize message name to work around the message
+        // getting blocked (for --verify)
+        ...msg,
+        id: `MOMENTS_MOCHITEST_${Date.now()}`,
+        targeting: "true",
+      },
+    ],
+    { clear: true }
+  );
   // Reload the provider
   await ASRouter._updateMessageProviders();
   // Wait to load the WNPanel messages
@@ -67,8 +71,7 @@ add_task(async function test_with_rs_messages() {
       id: `MOMENTS_MOCHITEST_${Date.now()}`,
       priority: 2,
       targeting: "true",
-    },
-    { useRecordId: true }
+    }
   );
 
   // Reset so we can `await` for the pref value to be set again

diff --git a/browser/components/newtab/test/browser/browser_asrouter_whatsnewpanel.js b/browser/components/newtab/test/browser/browser_asrouter_whatsnewpanel.js
@@ -26,14 +26,15 @@ add_task(async function test_with_rs_messages() {
   );
   const initialMessageCount = ASRouter.state.messages.length;
   const client = RemoteSettings("whats-new-panel");
-  await client.db.clear();
-  for (const record of msgs) {
-    await client.db.create(
+  await client.db.importChanges(
+    {},
+    42,
+    [
       // Modify targeting to ensure the messages always show up
-      { ...record, targeting: "true" }
-    );
-  }
-  await client.db.saveLastModified(42); // Prevent from loading JSON dump.
+      { ...record, targeting: "true" },
+    ],
+    { clear: true }
+  );
 
   const whatsNewBtn = document.getElementById("appMenu-whatsnew-button");
   Assert.equal(whatsNewBtn.hidden, true, "What's New btn doesn't exist");

diff --git a/security/manager/ssl/RemoteSecuritySettings.jsm b/security/manager/ssl/RemoteSecuritySettings.jsm
@@ -428,7 +428,9 @@ class IntermediatePreloads {
       }
       const toReset = current.filter(record => record.cert_import_complete);
       try {
-        await this.client.db.importBulk(
+        await this.client.db.importChanges(
+          undefined, // do not touch metadata.
+          undefined, // do not touch collection timestamp.
           toReset.map(r => ({ ...r, cert_import_complete: false }))
         );
       } catch (err) {
@@ -495,7 +497,9 @@ class IntermediatePreloads {
       return;
     }
     try {
-      await this.client.db.importBulk(
+      await this.client.db.importChanges(
+        undefined, // do not touch metadata.
+        undefined, // do not touch collection timestamp.
         recordsToUpdate.map(r => ({ ...r, cert_import_complete: true }))
       );
     } catch (err) {
@@ -521,6 +525,7 @@ class IntermediatePreloads {
     const finalWaiting = finalCurrent.filter(
       record => !record.cert_import_complete
     );
+
     const countPreloaded = finalCurrent.length - finalWaiting.length;
 
     TelemetryStopwatch.finish(INTERMEDIATES_UPDATE_MS_TELEMETRY);

diff --git a/services/common/docs/RemoteSettings.rst b/services/common/docs/RemoteSettings.rst
@@ -435,7 +435,7 @@ And records can be created manually (as if they were synchronized from the serve
       domain: "website.com",
       usernameSelector: "#login-account",
       passwordSelector: "#pass-signin",
-    }, { synced: true });
+    });
 
 If no timestamp is set, any call to ``.get()`` will trigger the load of initial data (JSON dump) if any, or a synchronization will be triggered. To avoid that, store a fake timestamp: