From d6b419204e487a7bab8f206c10a7d83eee6983de Mon Sep 17 00:00:00 2001 From: Ryan Roemer Date: Fri, 22 May 2026 08:28:44 -0700 Subject: [PATCH 1/4] Chunks indicator similar posts table --- public/app/components/posts-found.js | 4 ++-- public/app/components/posts-table.js | 10 ++++++++++ public/app/hooks/use-chat-session.js | 3 +++ public/app/pages/chat.js | 3 ++- public/local/data/api/chat-session.js | 1 + public/local/data/api/rag.js | 1 + 6 files changed, 19 insertions(+), 3 deletions(-) diff --git a/public/app/components/posts-found.js b/public/app/components/posts-found.js index 142ad64..98c58e5 100644 --- a/public/app/components/posts-found.js +++ b/public/app/components/posts-found.js @@ -3,7 +3,7 @@ import { Modal } from "./modal.js"; import { PostsTable } from "./posts-table.js"; import { html } from "../util/html.js"; -export const PostsFound = ({ posts = [], analyticsDates }) => { +export const PostsFound = ({ posts = [], analyticsDates, usedChunks = [] }) => { const [isSimilarPostsModalOpen, setIsSimilarPostsModalOpen] = useState(false); return html` @@ -25,7 +25,7 @@ export const PostsFound = ({ posts = [], analyticsDates }) => { onClose=${() => setIsSimilarPostsModalOpen(false)} title="Similar Posts" > - ${(posts && html`<${PostsTable} posts=${posts} analyticsDates=${analyticsDates} />`) || html`

No results.

`} + ${(posts && html`<${PostsTable} posts=${posts} analyticsDates=${analyticsDates} usedChunks=${usedChunks} />`) || html`

No results.

`} `; diff --git a/public/app/components/posts-table.js b/public/app/components/posts-table.js index b243c70..be3a282 100644 --- a/public/app/components/posts-table.js +++ b/public/app/components/posts-table.js @@ -22,9 +22,11 @@ export const PostsTable = ({ heading, posts = [], analyticsDates = { start: null, end: null }, + usedChunks = [], }) => { const { getSortSymbol, handleColumnSort, sortItems } = useTableSort(); const [settings] = useSettings(); + const usedChunkSlugs = new Set(usedChunks.map((c) => c.slug)); // Short-circuit. if (posts.length === 0) { @@ -47,6 +49,9 @@ export const PostsTable = ({ + ${Object.entries(headings).map( ([key, label]) => html` + diff --git a/public/app/hooks/use-chat-session.js b/public/app/hooks/use-chat-session.js index 900d31d..63bf5eb 100644 --- a/public/app/hooks/use-chat-session.js +++ b/public/app/hooks/use-chat-session.js @@ -54,6 +54,7 @@ export const useChatSession = ({ start: null, end: null, }); + const [usedChunks, setUsedChunks] = useState([]); // Error state const [err, setErr] = useState(null); @@ -216,6 +217,7 @@ export const useChatSession = ({ } = event.message; searchMetadata = metadata; setSearchData({ posts: fetchedPosts, chunks, metadata }); + setUsedChunks(chatSessionRef.current.getUsedChunks()); setPosts(displayPosts); setAnalyticsDates(metadata?.analytics?.dates); } else if (event.type === "data") { @@ -373,6 +375,7 @@ export const useChatSession = ({ posts, searchData, analyticsDates, + usedChunks, err, contextExceededErr, isLoadingModelForChat, diff --git a/public/app/pages/chat.js b/public/app/pages/chat.js index 5caeeb8..2568589 100644 --- a/public/app/pages/chat.js +++ b/public/app/pages/chat.js @@ -148,6 +148,7 @@ export const Chat = () => { posts, searchData, analyticsDates, + usedChunks, err, contextExceededErr, isLoadingModelForChat, @@ -185,7 +186,7 @@ export const Chat = () => { <${DescriptionButton} /> <${SuggestedQueries} ...${{ suggestions: displayedSuggestions, isFetching }} /> - ${posts && html`<${PostsFound} ...${{ posts, analyticsDates }} />`} + ${posts && html`<${PostsFound} ...${{ posts, analyticsDates, usedChunks }} />`} ${err && html`<${Alert} type="error" err=${err}>${err.toString()}`} diff --git a/public/local/data/api/chat-session.js b/public/local/data/api/chat-session.js index f092fc8..68747c9 100644 --- a/public/local/data/api/chat-session.js +++ b/public/local/data/api/chat-session.js @@ -379,6 +379,7 @@ export const createChatSession = ({ provider, model, temperature }) => { getCapabilities: () => ({ ...capabilities }), canContinue: () => state.history.length === 0 || canContinue(state), getSearchData: () => state.searchData, + getUsedChunks: () => state.contextState?.usedChunks ?? [], getModel: () => ({ provider, model }), getTokenUsage: () => getTokenUsage(state), getHistory: () => [...state.history], diff --git a/public/local/data/api/rag.js b/public/local/data/api/rag.js index dd86fcb..7f508b4 100644 --- a/public/local/data/api/rag.js +++ b/public/local/data/api/rag.js @@ -73,6 +73,7 @@ export const performRagSearch = async ({ chunkCount: contextResult.chunkCount, tokenBreakdown: contextResult.tokenBreakdown, rawChunks: chunks, + usedChunks: contextResult.usedChunks, initialQuery: query, }; From 2d5230f7efa724ba6a72370512f66f40c46f9649 Mon Sep 17 00:00:00 2001 From: Ryan Roemer Date: Fri, 22 May 2026 08:35:17 -0700 Subject: [PATCH 2/4] Reset state --- public/app/hooks/use-chat-session.js | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/hooks/use-chat-session.js b/public/app/hooks/use-chat-session.js index 63bf5eb..383cc80 100644 --- a/public/app/hooks/use-chat-session.js +++ b/public/app/hooks/use-chat-session.js @@ -111,6 +111,7 @@ export const useChatSession = ({ setPosts(null); setSearchData(null); setAnalyticsDates({ start: null, end: null }); + setUsedChunks([]); setErr(null); setContextExceededErr(null); // Clean up chat session From cba6caf32c1634e9a4b32dacdbcdfd9fc2f56097 Mon Sep 17 00:00:00 2001 From: Ryan Roemer Date: Fri, 22 May 2026 09:02:13 -0700 Subject: [PATCH 3/4] Add drilldown for chunks --- public/app/components/posts-found.js | 9 +- public/app/components/posts-table.js | 132 ++++++++++++++++++-------- public/app/hooks/use-chat-session.js | 4 + public/app/pages/chat.js | 3 +- public/local/data/api/chat-session.js | 1 + public/local/data/api/chat.js | 7 +- public/local/data/api/rag.js | 1 + public/styles.css | 40 ++++++++ 8 files changed, 155 insertions(+), 42 deletions(-) diff --git a/public/app/components/posts-found.js b/public/app/components/posts-found.js index 98c58e5..7f81d20 100644 --- a/public/app/components/posts-found.js +++ b/public/app/components/posts-found.js @@ -3,7 +3,12 @@ import { Modal } from "./modal.js"; import { PostsTable } from "./posts-table.js"; import { html } from "../util/html.js"; -export const PostsFound = ({ posts = [], analyticsDates, usedChunks = [] }) => { +export const PostsFound = ({ + posts = [], + analyticsDates, + usedChunks = [], + chunkTexts = {}, +}) => { const [isSimilarPostsModalOpen, setIsSimilarPostsModalOpen] = useState(false); return html` @@ -25,7 +30,7 @@ export const PostsFound = ({ posts = [], analyticsDates, usedChunks = [] }) => { onClose=${() => setIsSimilarPostsModalOpen(false)} title="Similar Posts" > - ${(posts && html`<${PostsTable} posts=${posts} analyticsDates=${analyticsDates} usedChunks=${usedChunks} />`) || html`

No results.

`} + ${(posts && html`<${PostsTable} posts=${posts} analyticsDates=${analyticsDates} usedChunks=${usedChunks} chunkTexts=${chunkTexts} />`) || html`

No results.

`} `; diff --git a/public/app/components/posts-table.js b/public/app/components/posts-table.js index be3a282..faa5cf4 100644 --- a/public/app/components/posts-table.js +++ b/public/app/components/posts-table.js @@ -1,3 +1,4 @@ +import { Fragment, useState } from "react"; import { html } from "../util/html.js"; import { Category } from "./category.js"; import { Vertical } from "./vertical.js"; @@ -23,26 +24,41 @@ export const PostsTable = ({ posts = [], analyticsDates = { start: null, end: null }, usedChunks = [], + chunkTexts = {}, }) => { const { getSortSymbol, handleColumnSort, sortItems } = useTableSort(); const [settings] = useSettings(); + const [expandedSlug, setExpandedSlug] = useState(null); const usedChunkSlugs = new Set(usedChunks.map((c) => c.slug)); - // Short-circuit. - if (posts.length === 0) { - return html`
`; - } - - // Get the appropriate headings based on settings const headings = settings.displayAnalytics ? { ...BASE_HEADINGS, ...ANALYTICS_HEADINGS } : BASE_HEADINGS; + const colSpan = + 1 + Object.keys(headings).length + (settings.displayAnalytics ? 4 : 0); + const analyticsTitle = analyticsDates.start !== null && analyticsDates.end !== null ? `Analytics from ${new Date(analyticsDates.start).toLocaleDateString()} to ${new Date(analyticsDates.end).toLocaleDateString()}` : ""; + const getChunksForSlug = (slug) => { + const chunks = []; + for (const key of Object.keys(chunkTexts)) { + if (key.startsWith(`${slug}:`)) { + const [, start, end] = key.split(":"); + chunks.push({ key, text: chunkTexts[key], start, end }); + } + } + return chunks; + }; + + // Short-circuit. + if (posts.length === 0) { + return html`
`; + } + return html`

${heading}

@@ -81,39 +97,81 @@ export const PostsTable = ({ }, i, ) => { + const isExpanded = expandedSlug === slug; + const chunksForPost = getChunksForSlug(slug); return html` -
- - - - - - ${settings.displayAnalytics - ? html` - - - - + + + + + + ${ + settings.displayAnalytics + ? html` + + + + + ` + : null + } + + ${ + isExpanded && + html` + + - ` - : null} - + + ` + } + `; }, )} diff --git a/public/app/hooks/use-chat-session.js b/public/app/hooks/use-chat-session.js index 383cc80..3d67776 100644 --- a/public/app/hooks/use-chat-session.js +++ b/public/app/hooks/use-chat-session.js @@ -55,6 +55,7 @@ export const useChatSession = ({ end: null, }); const [usedChunks, setUsedChunks] = useState([]); + const [chunkTexts, setChunkTexts] = useState({}); // Error state const [err, setErr] = useState(null); @@ -112,6 +113,7 @@ export const useChatSession = ({ setSearchData(null); setAnalyticsDates({ start: null, end: null }); setUsedChunks([]); + setChunkTexts({}); setErr(null); setContextExceededErr(null); // Clean up chat session @@ -219,6 +221,7 @@ export const useChatSession = ({ searchMetadata = metadata; setSearchData({ posts: fetchedPosts, chunks, metadata }); setUsedChunks(chatSessionRef.current.getUsedChunks()); + setChunkTexts(chatSessionRef.current.getChunkTexts()); setPosts(displayPosts); setAnalyticsDates(metadata?.analytics?.dates); } else if (event.type === "data") { @@ -377,6 +380,7 @@ export const useChatSession = ({ searchData, analyticsDates, usedChunks, + chunkTexts, err, contextExceededErr, isLoadingModelForChat, diff --git a/public/app/pages/chat.js b/public/app/pages/chat.js index 2568589..5dc34c9 100644 --- a/public/app/pages/chat.js +++ b/public/app/pages/chat.js @@ -149,6 +149,7 @@ export const Chat = () => { searchData, analyticsDates, usedChunks, + chunkTexts, err, contextExceededErr, isLoadingModelForChat, @@ -186,7 +187,7 @@ export const Chat = () => { <${DescriptionButton} /> <${SuggestedQueries} ...${{ suggestions: displayedSuggestions, isFetching }} /> - ${posts && html`<${PostsFound} ...${{ posts, analyticsDates, usedChunks }} />`} + ${posts && html`<${PostsFound} ...${{ posts, analyticsDates, usedChunks, chunkTexts }} />`} ${err && html`<${Alert} type="error" err=${err}>${err.toString()}`} diff --git a/public/local/data/api/chat-session.js b/public/local/data/api/chat-session.js index 68747c9..d4ada14 100644 --- a/public/local/data/api/chat-session.js +++ b/public/local/data/api/chat-session.js @@ -380,6 +380,7 @@ export const createChatSession = ({ provider, model, temperature }) => { canContinue: () => state.history.length === 0 || canContinue(state), getSearchData: () => state.searchData, getUsedChunks: () => state.contextState?.usedChunks ?? [], + getChunkTexts: () => state.contextState?.chunkTexts ?? {}, getModel: () => ({ provider, model }), getTokenUsage: () => getTokenUsage(state), getHistory: () => [...state.history], diff --git a/public/local/data/api/chat.js b/public/local/data/api/chat.js index f77f376..632a4d1 100644 --- a/public/local/data/api/chat.js +++ b/public/local/data/api/chat.js @@ -85,7 +85,7 @@ export const BASE_TOKEN_ESTIMATE = estimateTokens( * @param {number} [options.maxChunks] - Optional max number of chunks to include * @param {boolean} [options.forMultiTurn=false] - Use larger cushion for multi-turn * @param {boolean} [options.isFirstTurn=false] - Skip ratio on first turn to maximize initial context - * @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>} + * @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, chunkTexts: Object, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>} */ export const buildContextFromChunks = async ({ chunks, @@ -163,12 +163,14 @@ export const buildContextFromChunks = async ({ // Each entry: { url, content, tokenCount } const contextEntries = []; const seenSlugs = new Map(); // slug -> index in contextEntries + const chunkTexts = {}; // {slug:start:end} -> text excerpt for (const chunk of chunksToProcess) { const post = await getPost(chunk.slug); const chunkText = getChunk(post.content, chunk.start, chunk.end).join( "\n\n", ); + chunkTexts[`${chunk.slug}:${chunk.start}:${chunk.end}`] = chunkText; // TODO(ESTIMATE): Per-chunk estimate affects which chunks are included. // Use markup factor since chunks will be wrapped in XML tags // (.........) @@ -278,6 +280,7 @@ export const buildContextFromChunks = async ({ context, usedChunks, chunkCount: usedChunks.length, + chunkTexts, tokenEstimate: totalContextTokensEst, // Granular token breakdown for UI display tokenBreakdown: { @@ -298,7 +301,7 @@ export const buildContextFromChunks = async ({ * @param {string} options.provider - LLM provider key * @param {string} options.model - Model ID * @param {number} options.targetChunkCount - Target number of chunks (will be clamped to MIN_CONTEXT_CHUNKS) - * @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>} + * @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, chunkTexts: Object, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>} */ export const rebuildContextWithLimit = async ({ chunks, diff --git a/public/local/data/api/rag.js b/public/local/data/api/rag.js index 7f508b4..354a122 100644 --- a/public/local/data/api/rag.js +++ b/public/local/data/api/rag.js @@ -74,6 +74,7 @@ export const performRagSearch = async ({ tokenBreakdown: contextResult.tokenBreakdown, rawChunks: chunks, usedChunks: contextResult.usedChunks, + chunkTexts: contextResult.chunkTexts, initialQuery: query, }; diff --git a/public/styles.css b/public/styles.css index b097fc0..11446dd 100644 --- a/public/styles.css +++ b/public/styles.css @@ -1454,3 +1454,43 @@ button.loading-status-icon-button { display: flex; align-items: center; } + +/* Chunk Excerpts */ +.chunk-excerpts { + max-height: 400px; + overflow-y: auto; + padding: 12px 16px; + background: var(--color-bg-light); +} + +.chunk-excerpt { + margin-bottom: 12px; + padding: 10px 12px; + background: var(--color-bg-white); + border: 1px solid var(--color-border-light); + border-radius: 4px; +} + +.chunk-excerpt:last-child { + margin-bottom: 0; +} + +.chunk-label { + font-size: 11px; + font-weight: 600; + color: var(--color-text-secondary); + margin-bottom: 6px; +} + +.chunk-label-num { + font-weight: 400; + color: var(--color-text-muted); +} + +.chunk-text { + font-size: 13px; + line-height: 1.5; + color: var(--color-text-primary); + white-space: pre-wrap; + word-wrap: break-word; +} From c55082fbfb6e7dd367f80a606719e42eabea3578 Mon Sep 17 00:00:00 2001 From: Ryan Roemer Date: Fri, 22 May 2026 09:22:43 -0700 Subject: [PATCH 4/4] Code review fixes --- public/app/components/posts-table.js | 13 +++++++------ public/local/data/api/chat.js | 3 ++- public/local/data/api/rag.js | 2 ++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/public/app/components/posts-table.js b/public/app/components/posts-table.js index faa5cf4..65318d1 100644 --- a/public/app/components/posts-table.js +++ b/public/app/components/posts-table.js @@ -35,8 +35,7 @@ export const PostsTable = ({ ? { ...BASE_HEADINGS, ...ANALYTICS_HEADINGS } : BASE_HEADINGS; - const colSpan = - 1 + Object.keys(headings).length + (settings.displayAnalytics ? 4 : 0); + const colSpan = 1 + Object.keys(headings).length; const analyticsTitle = analyticsDates.start !== null && analyticsDates.end !== null @@ -45,10 +44,12 @@ export const PostsTable = ({ const getChunksForSlug = (slug) => { const chunks = []; - for (const key of Object.keys(chunkTexts)) { - if (key.startsWith(`${slug}:`)) { - const [, start, end] = key.split(":"); - chunks.push({ key, text: chunkTexts[key], start, end }); + const slugChunks = usedChunks.filter((c) => c.slug === slug); + for (const chunk of slugChunks) { + const key = `${chunk.slug}:${chunk.start}:${chunk.end}`; + const text = chunkTexts[key]; + if (text) { + chunks.push({ key, text, start: chunk.start, end: chunk.end }); } } return chunks; diff --git a/public/local/data/api/chat.js b/public/local/data/api/chat.js index 632a4d1..8a29dcc 100644 --- a/public/local/data/api/chat.js +++ b/public/local/data/api/chat.js @@ -170,7 +170,6 @@ export const buildContextFromChunks = async ({ const chunkText = getChunk(post.content, chunk.start, chunk.end).join( "\n\n", ); - chunkTexts[`${chunk.slug}:${chunk.start}:${chunk.end}`] = chunkText; // TODO(ESTIMATE): Per-chunk estimate affects which chunks are included. // Use markup factor since chunks will be wrapped in XML tags // (.........) @@ -194,6 +193,7 @@ export const buildContextFromChunks = async ({ entry.content += CHUNK_COMBINE_SEPARATOR + chunkText; totalContextTokensEst += chunkTokensEst; usedChunks.push(chunk); + chunkTexts[`${chunk.slug}:${chunk.start}:${chunk.end}`] = chunkText; continue; } // "duplicate" mode falls through to add as new entry @@ -230,6 +230,7 @@ export const buildContextFromChunks = async ({ // Accumulate tokens and track chunk totalContextTokensEst += chunkTokensEst; usedChunks.push(chunk); + chunkTexts[`${chunk.slug}:${chunk.start}:${chunk.end}`] = chunkText; if (DEBUG_TOKENS) { // eslint-disable-next-line no-undef diff --git a/public/local/data/api/rag.js b/public/local/data/api/rag.js index 354a122..933bb7c 100644 --- a/public/local/data/api/rag.js +++ b/public/local/data/api/rag.js @@ -113,6 +113,8 @@ export const reduceContext = async ({ contextState, provider, model }) => { chunkCount: result.chunkCount, tokenBreakdown: result.tokenBreakdown, rawChunks, + usedChunks: result.usedChunks, + chunkTexts: result.chunkTexts, initialQuery, }; } catch (err) {
+ + { return html`
+ ${usedChunkSlugs.has(slug) && + html``} + ${date ? new Date(date).toISOString().substring(0, 10) : ""}
- ${usedChunkSlugs.has(slug) && - html``} - - ${date ? new Date(date).toISOString().substring(0, 10) : ""} - - ${title} - ${Category({ category: categories.primary })} - ${verticals?.primary && - Vertical({ vertical: verticals.primary })} - ${analytics.views}${analytics.users}${analytics.time.toFixed(2)} - ${(analytics.bounceRate * 100).toFixed(0)}% + <${Fragment}> +
+ ${ + usedChunkSlugs.has(slug) && + html` + setExpandedSlug(isExpanded ? null : slug)} + title=${isExpanded + ? "Click to collapse chunk excerpts" + : "Click to view chunk excerpts"} + >` + } + + ${date ? new Date(date).toISOString().substring(0, 10) : ""} + + ${title} + ${Category({ category: categories.primary })} + ${ + verticals?.primary && + Vertical({ vertical: verticals.primary }) + } + ${analytics.views}${analytics.users}${analytics.time.toFixed(2)} + ${(analytics.bounceRate * 100).toFixed(0)}% +
+
+ ${chunksForPost.map( + (chunk, idx) => html` +
+
+ Chunk ${idx + 1} ${" "}(${chunk.start}–${chunk.end}) +
+
${chunk.text}
+
+ `, + )} +