assert_on_prepare_conflict_with_hole.js

/**
 * Constructs the following cycle that can lead to stalling a sharded cluster:
 * | Preparer                              | Retryable findAndModify   | OplogVisibility Ts |
 * |---------------------------------------+---------------------------+--------------------|
 * | BeginTxn                              |                           |                    |
 * | Write A                               |                           |                    |
 * |                                       | BeginTxn                  |                    |
 * |                                       | Preallocates TS(9),TS(10) |                  8 |
 * | (side txn commits prepare oplog @ 11) |                           |                    |
 * | Prepare 11                            |                           |                    |
 * |                                       | Write A (PrepareConflict) |                    |
 *
 * In this scenario, the prepared transaction blocks waiting for its prepare oplog entry at
 * timestamp 11 to become majority committed. However, the prepare oplog entry cannot replicate to
 * secondaries until the oplog visibility timestamp advances to 11. The oplog visibility timestamp
 * advancing is blocked on the findAndModify that allocated timestamps 9 and 10. The findAndModify
 * cannot make progress because it has hit a prepare conflict. The prepare conflict this test
 * specifically exercises is for duplicate key detection on a non-_id unique index.
 *
 * @tags: [uses_transactions, uses_prepare_transaction]
 */
import {Thread} from "jstests/libs/parallelTester.js";
import {ReplSetTest} from "jstests/libs/replsettest.js";

// Use a single node replica set for simplicity. Note that an oplog hole on a single node replica
// will block new writes from becoming majority committed.
const rst = new ReplSetTest({
    nodes: 1,
    nodeOptions: {
        setParameter: {logComponentVerbosity: tojson({storage: 1})},
    }
});
rst.startSet();
rst.initiate();

const primary = rst.getPrimary();
assert.commandWorked(primary.adminCommand(
    {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}}));
const db = primary.getDB("test");

const collName = "mycoll";
assert.commandWorked(db.runCommand({create: collName, writeConcern: {w: "majority"}}));
// A secondary unique index requires cursor positioning in WT which can result in hitting a prepare
// conflict.
assert.commandWorked(db[collName].createIndex({a: 1}, {unique: true}));

// Having changeStreamPreAndPostImages is required to cause the findAndModify to reserve optimes.
assert.commandWorked(
    db.runCommand({collMod: collName, changeStreamPreAndPostImages: {enabled: true}}));

// Insert the document to be updated by findAndModify.
assert.commandWorked(db[collName].insert({a: 3}));

// Start a multi-document transaction that inserts an `a: 2` update.
const lsid = ({id: UUID()});
assert.commandWorked(db.runCommand({
    insert: collName,
    documents: [{a: 2}],
    lsid,
    txnNumber: NumberLong(1),
    autocommit: false,
    startTransaction: true,
}));

// Prepare the `a: 2` update.
let prepTs = assert.commandWorked(db.adminCommand({
    prepareTransaction: 1,
    lsid,
    txnNumber: NumberLong(1),
    autocommit: false
}))["prepareTimestamp"];

// In another thread, perform an retryable find and modify that also attempts to touch the `a: 2`
// document. This findAndModify will block until the above transaction commits or aborts. If the
// above transaction commits, this findAndModify will fail with a duplicate key. If the above
// transaction is aborted, this findAndModify will succeed.
//
// This operation will open up a hole in the oplog preventing writes from becoming majority
// committed. In a properly behaving system, we will notice this resource being held while
// entering a blocking call (prepare conflict resolution) and retry the transaction (which
// releases the resource that prevents writes from becoming majority committed).
const triggerPrepareConflictThread = new Thread(function(host, dbName, collName) {
    const conn = new Mongo(host);
    const session = conn.startSession({retryWrites: true});
    const collection = session.getDatabase(dbName).getCollection(collName);
    jsTestLog("Inserting a conflicting operation while keeping a hole open.");
    assert.throwsWithCode(() => {
        collection.findAndModify(
            {query: {a: 3}, update: {a: 2, fromFindAndModify: true}, upsert: true});
    }, ErrorCodes.DuplicateKey);
}, primary.host, db.getName(), collName);

triggerPrepareConflictThread.start();

// Wait for the findAndModify to be in the system before attempting the majority write. Technically,
// this is insufficient to prove we're properly exercising the code that detects a possible deadlock
// and releases resources. In these cases, the test succeeds because the (yet to happen) majority
// write occurs before the above thread creates a hole.
assert.soon(() => {
    const ops = primary.getDB("admin")
                    .aggregate([
                        {$currentOp: {allUsers: true}},
                        {
                            $match: {
                                type: "op",
                                ns: db[collName].getFullName(),
                                "command.findandmodify": {$exists: true},
                            }
                        }
                    ])
                    .toArray();

    if (ops.length === 0) {
        return false;
    }

    assert.eq(ops.length, 1, ops);
    return true;
});

// If the system is misbehaving, this write will fail to "majority replicate". As noted above, in a
// single node replica set, an operation must be visible in the oplog before it can be considered
// majority replicated.
jsTestLog("Doing the majority write.");
assert.soon(() => {
    assert.commandWorked(db.bla.insert({}, {writeConcern: {w: "majority"}}));
    return true;
});

// We could stop the test here, but by committing the transaction we can also assert that the
// `triggerPrepareConflictThread` sees a `DuplicateKey` error.
jsTestLog({"Committing. CommitTs": prepTs});
assert.commandWorked(db.adminCommand({
    commitTransaction: 1,
    lsid,
    txnNumber: NumberLong(1),
    autocommit: false,
    commitTimestamp: prepTs
}));

triggerPrepareConflictThread.join();

rst.stopSet();