cover html-limited bot metadata behavior in deployed PPR paths

test/e2e/app-dir/cache-components-bot-ua/cache-components-bot-ua.test.ts

@@ -5,13 +5,13 @@ describe('cache-components PPR bot static generation bypass', () => {
     files: __dirname,
   })
 
-  it('should bypass static generation for DOM bot requests to avoid SSG_BAILOUT', async () => {
+  it('should bypass static generation for bot requests to avoid SSG_BAILOUT', async () => {
     const res = await next.fetch('/foo', {
       headers: {
         'user-agent': 'Googlebot',
       },
     })
-    // With cache components + PPR enabled, DOM bots should behave like regular users
+    // With cache components + PPR enabled, bots should use dynamic rendering
     // and use the fallback cache mechanism. This allows them to handle dynamic content
     // like Math.random() without triggering SSG_BAILOUT errors.
     expect(res.status).toBe(200)
@@ -29,4 +29,71 @@ describe('cache-components PPR bot static generation bypass', () => {
     // With PPR, content is streamed, but the important thing is that
     // the page rendered without a 500 error
   })
+
+  it('should stream metadata for normal requests', async () => {
+    const $ = await next.render$('/foo')
+
+    expect($('head title').text()).not.toContain('Home')
+    expect($('body title').text()).toBe('Home')
+  })
+
+  for (const userAgent of ['Googlebot', 'Google-PageRenderer']) {
+    it(`should stream metadata for bot ${userAgent} without known head limitations`, async () => {
+      const $ = await next.render$('/foo', undefined, {
+        headers: {
+          'user-agent': userAgent,
+        },
+      })
+
+      expect($('head title').text()).not.toContain('Home')
+      expect($('body title').text()).toBe('Home')
+    })
+  }
+
+  it('should block metadata for an html limited bot', async () => {
+    const $ = await next.render$('/foo', undefined, {
+      headers: {
+        'user-agent': 'Discordbot',
+      },
+    })
+
+    expect($('head title').text()).toBe('Home')
+    expect($('body title').text()).not.toContain('Home')
+  })
+})
+
+describe('cache-components PPR customized htmlLimitedBots', () => {
+  const { next } = nextTestSetup({
+    files: __dirname,
+    overrideFiles: {
+      'next.config.js': `
+        module.exports = {
+          cacheComponents: true,
+          htmlLimitedBots: /Minibot/i,
+        }
+      `,
+    },
+  })
+
+  it('should block metadata for a configured html limited bot', async () => {
+    const $ = await next.render$('/foo', undefined, {
+      headers: {
+        'user-agent': 'Minibot',
+      },
+    })
+
+    expect($('head title').text()).toBe('Home')
+    expect($('body title').text()).not.toContain('Home')
+  })
+
+  it('should stream metadata for Googlebot when it is not in the configured rule', async () => {
+    const $ = await next.render$('/foo', undefined, {
+      headers: {
+        'user-agent': 'Googlebot',
+      },
+    })
+
+    expect($('head title').text()).not.toContain('Home')
+    expect($('body title').text()).toBe('Home')
+  })
 })

test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts

@@ -53,25 +53,44 @@ describe('app-dir - metadata-streaming', () => {
     })
   })
 
-  it('should send the blocking response for html limited bots', async () => {
-    const $ = await next.render$(
-      '/',
-      undefined, // no query
-      {
-        headers: {
-          'user-agent': 'Twitterbot',
-        },
-      }
-    )
-    expect(await $('title').text()).toBe('index page')
-  })
-
-  it('should send streaming response for headless browser bots', async () => {
-    const browser = await next.browser('/')
-    await retry(async () => {
-      expect(await browser.elementByCss('title').text()).toBe('index page')
+  for (const userAgent of ['Twitterbot', 'Mediapartners-Google']) {
+    it(`should send the blocking response for html limited bot ${userAgent}`, async () => {
+      const $ = await next.render$(
+        '/',
+        undefined, // no query
+        {
+          headers: {
+            'user-agent': userAgent,
+          },
+        }
+      )
+      expect($('head title').text()).toBe('index page')
+      expect($('body title').length).toBe(0)
     })
-  })
+  }
+
+  for (const userAgent of [
+    'Googlebot',
+    'Google-PageRenderer',
+    'Google-InspectionTool',
+    'AdsBot-Google',
+    'Storebot-Google',
+    'googleweblight',
+  ]) {
+    it(`should send a streaming response for bot ${userAgent} without known head limitations`, async () => {
+      const $ = await next.render$(
+        '/',
+        undefined, // no query
+        {
+          headers: {
+            'user-agent': userAgent,
+          },
+        }
+      )
+      expect($('head title').length).toBe(0)
+      expect($('body title').text()).toBe('index page')
+    })
+  }
 
   it('should only insert metadata once into head or body', async () => {
     const browser = await next.browser('/slow')