feat(blog-app): prerender the blog so crawlers see the content (BAPP-13)

infra/lib/blog-stack.ts

@@ -39,19 +39,57 @@ export class BlogStack extends Stack {
       blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
     });
 
-    // Redirect legacy blog.nakom.is requests to the canonical domain, preserving the path
-    const legacyRedirectFunction = new cloudfront.Function(this, 'LegacyDomainRedirect', {
+    // Two jobs in one function, because CloudFront permits only one function
+    // per event type per behaviour and both have to happen on viewer-request:
+    //
+    //  1. Redirect legacy blog.nakom.is requests to the canonical domain,
+    //     preserving the path.
+    //  2. Rewrite extensionless paths to the prerendered .html file (BAPP-13).
+    //     The site is prerendered to dist/<slug>.html, but readers and crawlers
+    //     ask for /<slug>. Without this rewrite the S3 key misses and the
+    //     request falls through to the error responses below — which is exactly
+    //     the empty-shell behaviour prerendering exists to end.
+    const viewerRequestFunction = new cloudfront.Function(this, 'LegacyDomainRedirect', {
       functionName: 'blog-legacy-domain-redirect',
       code: cloudfront.FunctionCode.fromInline(`
 function handler(event) {
-  if (event.request.headers.host.value === '${LEGACY_DOMAIN}') {
+  var request = event.request;
+
+  if (request.headers.host.value === '${LEGACY_DOMAIN}') {
     return {
       statusCode: 301,
       statusDescription: 'Moved Permanently',
-      headers: { location: { value: 'https://${CANONICAL_DOMAIN}' + event.request.uri } }
+      headers: { location: { value: 'https://${CANONICAL_DOMAIN}' + request.uri } }
     };
   }
-  return event.request;
+
+  var uri = request.uri;
+
+  // Leave the root alone — defaultRootObject already maps it to index.html.
+  if (uri === '/') {
+    return request;
+  }
+
+  // Serve /<slug>/ the same page as /<slug>, by rewrite rather than redirect.
+  // A 301 here would be tidier for canonicalisation but would drop the query
+  // string — CloudFront does not carry it into the location header — and that
+  // silently eats utm_* and gclid on any ad click that lands on the slashed
+  // form. The <link rel="canonical"> emitted on every prerendered page is the
+  // mechanism for telling search engines which URL is the real one.
+  if (uri.endsWith('/')) {
+    uri = uri.slice(0, -1);
+  }
+
+  // Anything with a file extension in its last segment is a real asset —
+  // /assets/index-abc123.js, /posts/some-post.md, /favicon.ico — and is
+  // fetched from S3 unchanged.
+  var lastSegment = uri.slice(uri.lastIndexOf('/') + 1);
+  if (lastSegment.indexOf('.') === -1) {
+    uri = uri + '.html';
+  }
+
+  request.uri = uri;
+  return request;
 }
 `),
       runtime: cloudfront.FunctionRuntime.JS_2_0,
@@ -106,7 +144,7 @@ function handler(event) {
         cachePolicy: cloudfront.CachePolicy.CACHING_OPTIMIZED,
         compress: true,
         functionAssociations: [{
-          function: legacyRedirectFunction,
+          function: viewerRequestFunction,
           eventType: cloudfront.FunctionEventType.VIEWER_REQUEST,
         }],
       },
@@ -141,17 +179,28 @@ function handler(event) {
       domainNames: [CANONICAL_DOMAIN, LEGACY_DOMAIN],
       certificate: certificate,
       defaultRootObject: 'index.html',
+      // Every route is prerendered to its own HTML file (BAPP-13), so a miss is
+      // now a genuine miss and says so.
+      //
+      // This previously mapped both statuses to /index.html with status **200**
+      // — necessary when the SPA had to boot and route client-side, but a soft
+      // 404: every typo URL looked to a crawler like a valid page carrying thin
+      // content, which Google treats as a quality problem across the site.
+      //
+      // 403 is listed as well as 404 because the bucket is private behind OAC
+      // and grants only s3:GetObject — with no s3:ListBucket, S3 answers a
+      // missing key with AccessDenied rather than NoSuchKey.
       errorResponses: [
         {
           httpStatus: 404,
-          responseHttpStatus: 200,
-          responsePagePath: '/index.html',
+          responseHttpStatus: 404,
+          responsePagePath: '/404.html',
           ttl: Duration.minutes(5),
         },
         {
           httpStatus: 403,
-          responseHttpStatus: 200,
-          responsePagePath: '/index.html',
+          responseHttpStatus: 404,
+          responsePagePath: '/404.html',
           ttl: Duration.minutes(5),
         },
       ],

web/.gitignore

@@ -1,7 +1,9 @@
 node_modules/
 dist/
+dist-ssr/
 *.tsbuildinfo
 .DS_Store
 *.swp
 src/content.generated.ts
 public/sitemap.xml
+public/llms.txt

web/package.json

@@ -6,7 +6,9 @@
   "type": "module",
   "scripts": {
     "dev": "vite",
-    "build": "npm run content && tsc -b && vite build",
+    "build": "npm run content && tsc -b && vite build && npm run build:ssr && npm run prerender",
+    "build:ssr": "vite build --ssr src/entry-server.tsx --outDir dist-ssr",
+    "prerender": "tsx scripts/prerender.ts",
     "preview": "vite preview",
     "content": "tsx scripts/buildContent.ts"
   },

web/scripts/buildContent.ts

@@ -3,6 +3,7 @@ import path from 'path';
 import matter from 'gray-matter';
 import { applyShortcodes } from '../src/shortcodes.js';
 import { createMarkdownProcessor } from '../src/utils/markdownPipeline.js';
+import { BASE_URL, SITE_DESCRIPTION, markdownUrl, postUrl } from '../src/siteConfig.js';
 
 async function processMarkdownContent(markdownContent: string, slug: string) {
   const { data: frontmatter, content } = matter(markdownContent);
@@ -18,8 +19,6 @@ async function processMarkdownContent(markdownContent: string, slug: string) {
   };
 }
 
-const BASE_URL = 'https://blog.nakomis.com';
-
 async function buildContent() {
   const contentDir = path.join(process.cwd(), 'content', 'blog');
   const outputPath = path.join(process.cwd(), 'src', 'content.generated.ts');
@@ -80,7 +79,7 @@ export const BLOG_POSTS = ${JSON.stringify(posts, null, 2)} as const;
   // Generate sitemap.xml
   const sitemapPath = path.join(process.cwd(), 'public', 'sitemap.xml');
   const postEntries = posts.map(post => {
-    const url = post.frontmatter.canonical ?? `${BASE_URL}/${post.slug}`;
+    const url = post.frontmatter.canonical ?? postUrl(post.slug);
     const lastmod = post.frontmatter.date ?? new Date().toISOString().split('T')[0];
     return `  <url>\n    <loc>${url}</loc>\n    <lastmod>${lastmod}</lastmod>\n  </url>`;
   }).join('\n');
@@ -97,6 +96,40 @@ ${postEntries}
 
   fs.writeFileSync(sitemapPath, sitemap);
   console.log(`✓ Generated sitemap.xml with ${posts.length + 1} URLs`);
+
+  // Generate llms.txt (BAPP-13).
+  //
+  // deploy.sh has always synced the raw markdown to /posts/*.md, but nothing
+  // linked to it, so the cleanest representation of every post was effectively
+  // undiscoverable. The sitemap deliberately does not list these — sitemaps are
+  // for canonical pages, and listing both would muddle canonicalisation.
+  //
+  // Generated from the same `posts` array as the sitemap so it cannot drift.
+  const llmsPath = path.join(process.cwd(), 'public', 'llms.txt');
+  const llmsEntries = posts.map(post => {
+    const excerpt = (post.frontmatter.excerpt ?? '').replace(/\s+/g, ' ').trim();
+    return `- [${post.frontmatter.title}](${markdownUrl(post.slug)})${excerpt ? `: ${excerpt}` : ''}`;
+  }).join('\n');
+
+  const llms = `# Martin Harris — Blog
+
+> ${SITE_DESCRIPTION}
+
+Written by Martin Harris. Every post below links to its raw markdown source,
+which is the same text the rendered page is built from — no navigation, no
+markup, no scripts. The HTML version of any post is at ${BASE_URL}/<slug>.
+
+## Posts
+
+${llmsEntries}
+
+## Other
+
+- [Sitemap](${BASE_URL}/sitemap.xml): all canonical HTML URLs.
+`;
+
+  fs.writeFileSync(llmsPath, llms);
+  console.log(`✓ Generated llms.txt with ${posts.length} posts`);
 }
 
 buildContent().catch(console.error);

web/scripts/prerender.ts

@@ -0,0 +1,139 @@
+/**
+ * Prerender every route to a real HTML file (BAPP-13).
+ *
+ * Runs after both Vite builds: the client build produces dist/index.html with
+ * the hashed asset links, and the SSR build produces dist-ssr/entry-server.js.
+ * This script uses the first as a template and the second as the renderer.
+ *
+ * Why this exists: blog.nakomis.com was a client-rendered SPA, so every URL
+ * returned the same ~2.3KB empty shell. Crawlers, link unfurlers and the
+ * AdSense reviewer all saw a page with no content on it.
+ */
+import fs from 'fs';
+import path from 'path';
+import { pathToFileURL } from 'url';
+import type { PrerenderRoute } from '../src/entry-server.js';
+
+const DIST = path.join(process.cwd(), 'dist');
+const SSR_ENTRY = path.join(process.cwd(), 'dist-ssr', 'entry-server.js');
+const TEMPLATE = path.join(DIST, 'index.html');
+
+/** Escape for use inside a double-quoted HTML attribute. */
+function attr(value: string): string {
+  return value
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+/** Escape for use as HTML text content. */
+function text(value: string): string {
+  return value.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function headTags(route: PrerenderRoute): string {
+  const tags = [
+    `<link rel="canonical" href="${attr(route.canonical)}" />`,
+    `<meta property="og:title" content="${attr(route.title)}" />`,
+    `<meta property="og:description" content="${attr(route.description)}" />`,
+    `<meta property="og:type" content="${route.ogType}" />`,
+    `<meta property="og:url" content="${attr(route.canonical)}" />`,
+    `<meta name="twitter:card" content="summary_large_image" />`,
+    `<meta name="twitter:title" content="${attr(route.title)}" />`,
+    `<meta name="twitter:description" content="${attr(route.description)}" />`,
+  ];
+
+  // Point machine readers at the markdown source. It is already served from
+  // /posts/*.md and is a far cleaner representation than the rendered page.
+  if (route.markdown) {
+    tags.push(`<link rel="alternate" type="text/markdown" href="${attr(route.markdown)}" />`);
+  }
+
+  return tags.map(tag => `  ${tag}`).join('\n');
+}
+
+function buildPage(template: string, route: PrerenderRoute, appHtml: string): string {
+  let html = template;
+
+  // Replace rather than append: the template already carries the site-wide
+  // title and description, and two of each would be ambiguous to a crawler.
+  html = html.replace(
+    /<title>[\s\S]*?<\/title>/,
+    `<title>${text(route.title)}</title>`,
+  );
+  html = html.replace(
+    /<meta\s+name="description"[\s\S]*?\/?>/,
+    `<meta name="description" content="${attr(route.description)}" />`,
+  );
+
+  html = html.replace('</head>', `${headTags(route)}\n</head>`);
+  html = html.replace('<div id="root"></div>', `<div id="root">${appHtml}</div>`);
+
+  return html;
+}
+
+async function prerender() {
+  if (!fs.existsSync(TEMPLATE)) {
+    throw new Error(`No client build found at ${TEMPLATE} — run "vite build" first.`);
+  }
+  if (!fs.existsSync(SSR_ENTRY)) {
+    throw new Error(`No SSR build found at ${SSR_ENTRY} — run "vite build --ssr" first.`);
+  }
+
+  const template = fs.readFileSync(TEMPLATE, 'utf-8');
+
+  // Guard the two anchors we splice into. If a future index.html edit renames
+  // or reformats them, the replace above would silently no-op and ship empty
+  // pages that look like a successful build.
+  if (!template.includes('<div id="root"></div>')) {
+    throw new Error('index.html no longer contains <div id="root"></div> — prerender cannot inject markup.');
+  }
+  if (!/<meta\s+name="description"/.test(template)) {
+    throw new Error('index.html no longer contains a description meta tag — prerender cannot replace it.');
+  }
+
+  const { render, getRoutes } = await import(pathToFileURL(SSR_ENTRY).href) as {
+    render: (url: string) => string;
+    getRoutes: () => PrerenderRoute[];
+  };
+
+  const routes = getRoutes();
+  console.log(`Prerendering ${routes.length} routes...`);
+
+  for (const route of routes) {
+    const appHtml = render(route.url);
+    if (!appHtml.trim()) {
+      throw new Error(`${route.url} rendered to nothing — refusing to write an empty page.`);
+    }
+    const outPath = path.join(DIST, route.outFile);
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, buildPage(template, route, appHtml));
+    console.log(`✓ ${route.outFile.padEnd(60)} ${(appHtml.length / 1024).toFixed(1)}KB`);
+  }
+
+  // A dedicated 404 page, served with a real 404 status by CloudFront. The
+  // distribution used to map every miss to index.html with status 200 — a soft
+  // 404, which Google treats as a quality problem because every typo URL looks
+  // like a valid page with thin content.
+  const notFound: PrerenderRoute = {
+    url: '/__not-found__',
+    outFile: '404.html',
+    title: 'Page not found | Martin Harris',
+    description: 'That page does not exist.',
+    canonical: `${routes[0].canonical}`,
+    ogType: 'website',
+  };
+  fs.writeFileSync(
+    path.join(DIST, notFound.outFile),
+    buildPage(template, notFound, render(notFound.url)),
+  );
+  console.log(`✓ ${notFound.outFile}`);
+
+  console.log(`\n✓ Prerendered ${routes.length + 1} pages`);
+}
+
+prerender().catch(error => {
+  console.error(error);
+  process.exit(1);
+});