diff --git a/packages/toolkit/src/query/core/buildInitiate.ts b/packages/toolkit/src/query/core/buildInitiate.ts
index 06eeafa3b1..8641c3c373 100644
--- a/packages/toolkit/src/query/core/buildInitiate.ts
+++ b/packages/toolkit/src/query/core/buildInitiate.ts
@@ -291,15 +291,23 @@ export function buildInitiate({
       Object.values(runningMutations.get(dispatch) || {}).filter(isNotNullish)
   }
 
-  function middlewareWarning(getState: () => RootState<{}, string, string>) {
+  function middlewareWarning(dispatch: Dispatch) {
     if (process.env.NODE_ENV !== 'production') {
       if ((middlewareWarning as any).triggered) return
-      const registered =
-        getState()[api.reducerPath]?.config?.middlewareRegistered
-      if (registered !== undefined) {
-        ;(middlewareWarning as any).triggered = true
-      }
-      if (registered === false) {
+      const registered:
+        | ReturnType<typeof api.internalActions.internal_probeSubscription>
+        | boolean = dispatch(
+        api.internalActions.internal_probeSubscription({
+          queryCacheKey: 'DOES_NOT_EXIST',
+          requestId: 'DUMMY_REQUEST_ID',
+        })
+      )
+
+      ;(middlewareWarning as any).triggered = true
+
+      // The RTKQ middleware _should_ always return a boolean for `probeSubscription`
+      if (typeof registered !== 'boolean') {
+        // Otherwise, must not have been added
         throw new Error(
           `Warning: Middleware for RTK-Query API at reducerPath "${api.reducerPath}" has not been added to the store.
 You must add the middleware for RTK-Query to function correctly!`
@@ -346,7 +354,7 @@ You must add the middleware for RTK-Query to function correctly!`
         const thunkResult = dispatch(thunk)
         const stateAfter = selector(getState())
 
-        middlewareWarning(getState)
+        middlewareWarning(dispatch)
 
         const { requestId, abort } = thunkResult
 
@@ -440,7 +448,7 @@ You must add the middleware for RTK-Query to function correctly!`
           fixedCacheKey,
         })
         const thunkResult = dispatch(thunk)
-        middlewareWarning(getState)
+        middlewareWarning(dispatch)
         const { requestId, abort, unwrap } = thunkResult
         const returnValuePromise = thunkResult
           .unwrap()
diff --git a/packages/toolkit/src/query/tests/devWarnings.test.tsx b/packages/toolkit/src/query/tests/devWarnings.test.tsx
index 07954a3eb9..ce50aa9080 100644
--- a/packages/toolkit/src/query/tests/devWarnings.test.tsx
+++ b/packages/toolkit/src/query/tests/devWarnings.test.tsx
@@ -186,13 +186,16 @@ describe('missing reducer', () => {
   })
 })
 
-test('warns only for reducer if everything is missing', async () => {
+test('warns for reducer and also throws error if everything is missing', async () => {
   const store = configureStore({
     reducer: { x: () => 0 },
   })
   // @ts-expect-error
   api1.endpoints.q1.select(undefined)(store.getState())
-  await store.dispatch(api1.endpoints.q1.initiate(undefined))
+  const doDispatch = () => {
+    store.dispatch(api1.endpoints.q1.initiate(undefined))
+  }
+  expect(doDispatch).toThrowError(reMatchMissingMiddlewareError)
   expect(getLog().log).toBe(
     'Error: No data found at `state.api`. Did you forget to add the reducer to the store?'
   )