diff --git a/packages/toolkit/src/query/core/buildMiddleware/cacheCollection.ts b/packages/toolkit/src/query/core/buildMiddleware/cacheCollection.ts
index b5f9dbaef5..e043a8522a 100644
--- a/packages/toolkit/src/query/core/buildMiddleware/cacheCollection.ts
+++ b/packages/toolkit/src/query/core/buildMiddleware/cacheCollection.ts
@@ -65,10 +65,21 @@ export const buildCacheCollectionHandler: InternalHandlerBuilder = ({
   const { removeQueryResult, unsubscribeQueryResult, cacheEntriesUpserted } =
     api.internalActions
 
+  // A `condition`-based rejection means the query bailed out before it ever ran
+  // (e.g. `initiate(arg, { subscribe: false })` served fresh cached data). Nothing
+  // was fetched and no subscription changed, so we must not treat it as an
+  // unsubscribe trigger: rescheduling the removal timer here resets
+  // `keepUnusedDataFor` on every such call and prevents unused data from ever
+  // being collected. See https://github.com/reduxjs/redux-toolkit/issues/4480
+  const isQueryThunkRejectedByError = (
+    action: any,
+  ): action is ReturnType<typeof queryThunk.rejected> =>
+    queryThunk.rejected.match(action) && !action.meta.condition
+
   const canTriggerUnsubscribe = isAnyOf(
     unsubscribeQueryResult.match,
     queryThunk.fulfilled,
-    queryThunk.rejected,
+    isQueryThunkRejectedByError,
     cacheEntriesUpserted.match,
   )
 
diff --git a/packages/toolkit/src/query/tests/cacheCollection.test.ts b/packages/toolkit/src/query/tests/cacheCollection.test.ts
index c76e8c30a0..c67927f9ad 100644
--- a/packages/toolkit/src/query/tests/cacheCollection.test.ts
+++ b/packages/toolkit/src/query/tests/cacheCollection.test.ts
@@ -90,6 +90,38 @@ test(`query: handles large keepUnusedDataFor values over 32-bit ms`, async () =>
   expect(onCleanup).toHaveBeenCalled()
 })
 
+test(`query: repeated \`initiate\` with \`subscribe: false\` does not reset the removal timer (#4480)`, async () => {
+  const { store, api } = storeForApi(
+    createApi({
+      baseQuery: fetchBaseQuery({ baseUrl: 'https://example.com' }),
+      endpoints: (build) => ({
+        query: build.query<unknown, string>({
+          query: () => '/success',
+        }),
+      }),
+    }),
+  )
+
+  // First unsubscribed request fetches and schedules removal 60s from now.
+  await store.dispatch(api.endpoints.query.initiate('arg', { subscribe: false }))
+
+  vi.advanceTimersByTime(30000)
+  expect(onCleanup).not.toHaveBeenCalled()
+
+  // Second unsubscribed request hits the fresh cache and is rejected by the
+  // thunk `condition`. This must NOT restart the `keepUnusedDataFor` timer.
+  await store.dispatch(api.endpoints.query.initiate('arg', { subscribe: false }))
+
+  // Still not cleaned right before the original 60s deadline...
+  vi.advanceTimersByTime(29000)
+  expect(onCleanup).not.toHaveBeenCalled()
+
+  // ...but cleaned once the original deadline passes (would still be pending if
+  // the second call had reset the timer to fire at 90s).
+  vi.advanceTimersByTime(2000)
+  expect(onCleanup).toHaveBeenCalled()
+})
+
 describe(`query: await cleanup, keepUnusedDataFor set`, () => {
   const { store, api } = storeForApi(
     createApi({