Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions public/app/components/posts-found.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@ import { Modal } from "./modal.js";
import { PostsTable } from "./posts-table.js";
import { html } from "../util/html.js";

export const PostsFound = ({ posts = [], analyticsDates }) => {
export const PostsFound = ({
posts = [],
analyticsDates,
usedChunks = [],
chunkTexts = {},
}) => {
const [isSimilarPostsModalOpen, setIsSimilarPostsModalOpen] = useState(false);

return html`
Expand All @@ -25,7 +30,7 @@ export const PostsFound = ({ posts = [], analyticsDates }) => {
onClose=${() => setIsSimilarPostsModalOpen(false)}
title="Similar Posts"
>
${(posts && html`<${PostsTable} posts=${posts} analyticsDates=${analyticsDates} />`) || html`<p className="status">No results.</p>`}
${(posts && html`<${PostsTable} posts=${posts} analyticsDates=${analyticsDates} usedChunks=${usedChunks} chunkTexts=${chunkTexts} />`) || html`<p className="status">No results.</p>`}
</${Modal}>
</${Fragment}>
`;
Expand Down
135 changes: 102 additions & 33 deletions public/app/components/posts-table.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { Fragment, useState } from "react";
import { html } from "../util/html.js";
import { Category } from "./category.js";
import { Vertical } from "./vertical.js";
Expand All @@ -22,31 +23,52 @@ export const PostsTable = ({
heading,
posts = [],
analyticsDates = { start: null, end: null },
usedChunks = [],
chunkTexts = {},
}) => {
const { getSortSymbol, handleColumnSort, sortItems } = useTableSort();
const [settings] = useSettings();
const [expandedSlug, setExpandedSlug] = useState(null);
const usedChunkSlugs = new Set(usedChunks.map((c) => c.slug));

// Short-circuit.
if (posts.length === 0) {
return html`<div />`;
}

// Get the appropriate headings based on settings
const headings = settings.displayAnalytics
? { ...BASE_HEADINGS, ...ANALYTICS_HEADINGS }
: BASE_HEADINGS;

const colSpan = 1 + Object.keys(headings).length;

const analyticsTitle =
analyticsDates.start !== null && analyticsDates.end !== null
? `Analytics from ${new Date(analyticsDates.start).toLocaleDateString()} to ${new Date(analyticsDates.end).toLocaleDateString()}`
: "";

const getChunksForSlug = (slug) => {
const chunks = [];
const slugChunks = usedChunks.filter((c) => c.slug === slug);
for (const chunk of slugChunks) {
const key = `${chunk.slug}:${chunk.start}:${chunk.end}`;
const text = chunkTexts[key];
if (text) {
chunks.push({ key, text, start: chunk.start, end: chunk.end });
}
}
return chunks;
Comment thread
ryan-roemer marked this conversation as resolved.
};

// Short-circuit.
if (posts.length === 0) {
return html`<div />`;
}

return html`
<div>
<h2 className="content-subhead">${heading}</h2>
<table className="pure-table pure-table-bordered">
<thead>
<tr>
<th title="Included in prompt context">
<i className="iconoir-quote"></i>
</th>
${Object.entries(headings).map(
([key, label]) =>
html`<th
Expand All @@ -67,6 +89,7 @@ export const PostsTable = ({
date,
title,
href,
slug,
categories,
verticals,
analytics,
Expand All @@ -75,35 +98,81 @@ export const PostsTable = ({
},
i,
) => {
const isExpanded = expandedSlug === slug;
const chunksForPost = getChunksForSlug(slug);
return html`
<tr key=${`post-item-${i}`}>
<td style=${{ minWidth: "90px" }}>
${date ? new Date(date).toISOString().substring(0, 10) : ""}
</td>
<td
title=${JSON.stringify({
embeddingNumTokens,
similarity,
})}
>
<a href="${href}">${title}</a>
</td>
<td>${Category({ category: categories.primary })}</td>
<td>
${verticals?.primary &&
Vertical({ vertical: verticals.primary })}
</td>
${settings.displayAnalytics
? html`
<td key="views">${analytics.views}</td>
<td key="users">${analytics.users}</td>
<td key="time">${analytics.time.toFixed(2)}</td>
<td key="bounceRate">
${(analytics.bounceRate * 100).toFixed(0)}%
<${Fragment}>
<tr key=${`post-item-${i}`}>
<td>
${
usedChunkSlugs.has(slug) &&
html`<i
class="iconoir-quote"
style=${{ cursor: "pointer" }}
onClick=${() =>
setExpandedSlug(isExpanded ? null : slug)}
title=${isExpanded
? "Click to collapse chunk excerpts"
: "Click to view chunk excerpts"}
></i>`
}
</td>
<td style=${{ minWidth: "90px" }}>
${date ? new Date(date).toISOString().substring(0, 10) : ""}
</td>
<td
title=${JSON.stringify({
embeddingNumTokens,
similarity,
})}
>
<a href="${href}">${title}</a>
</td>
<td>${Category({ category: categories.primary })}</td>
<td>
${
verticals?.primary &&
Vertical({ vertical: verticals.primary })
}
</td>
${
settings.displayAnalytics
? html`
<td key="views">${analytics.views}</td>
<td key="users">${analytics.users}</td>
<td key="time">${analytics.time.toFixed(2)}</td>
<td key="bounceRate">
${(analytics.bounceRate * 100).toFixed(0)}%
</td>
`
: null
}
</tr>
${
isExpanded &&
html`
<tr key=${`post-item-${i}-expansion`}>
<td colspan=${colSpan} style=${{ padding: "0" }}>
<div class="chunk-excerpts">
${chunksForPost.map(
(chunk, idx) => html`
<div class="chunk-excerpt">
<div class="chunk-label">
Chunk ${idx + 1} ${" "}<span
class="chunk-label-num"
>(${chunk.start}–${chunk.end})</span
>
</div>
<div class="chunk-text">${chunk.text}</div>
</div>
`,
)}
</div>
</td>
`
: null}
</tr>
</tr>
`
}
</${Fragment}>
`;
},
)}
Expand Down
8 changes: 8 additions & 0 deletions public/app/hooks/use-chat-session.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ export const useChatSession = ({
start: null,
end: null,
});
const [usedChunks, setUsedChunks] = useState([]);
const [chunkTexts, setChunkTexts] = useState({});

// Error state
const [err, setErr] = useState(null);
Expand Down Expand Up @@ -110,6 +112,8 @@ export const useChatSession = ({
setPosts(null);
setSearchData(null);
setAnalyticsDates({ start: null, end: null });
setUsedChunks([]);
setChunkTexts({});
setErr(null);
setContextExceededErr(null);
// Clean up chat session
Expand Down Expand Up @@ -216,6 +220,8 @@ export const useChatSession = ({
} = event.message;
searchMetadata = metadata;
setSearchData({ posts: fetchedPosts, chunks, metadata });
setUsedChunks(chatSessionRef.current.getUsedChunks());
setChunkTexts(chatSessionRef.current.getChunkTexts());
setPosts(displayPosts);
setAnalyticsDates(metadata?.analytics?.dates);
} else if (event.type === "data") {
Expand Down Expand Up @@ -373,6 +379,8 @@ export const useChatSession = ({
posts,
searchData,
analyticsDates,
usedChunks,
chunkTexts,
err,
contextExceededErr,
isLoadingModelForChat,
Expand Down
4 changes: 3 additions & 1 deletion public/app/pages/chat.js
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ export const Chat = () => {
posts,
searchData,
analyticsDates,
usedChunks,
chunkTexts,
err,
contextExceededErr,
isLoadingModelForChat,
Expand Down Expand Up @@ -185,7 +187,7 @@ export const Chat = () => {

<${DescriptionButton} />
<${SuggestedQueries} ...${{ suggestions: displayedSuggestions, isFetching }} />
${posts && html`<${PostsFound} ...${{ posts, analyticsDates }} />`}
${posts && html`<${PostsFound} ...${{ posts, analyticsDates, usedChunks, chunkTexts }} />`}

${err && html`<${Alert} type="error" err=${err}>${err.toString()}</${Alert}>`}

Expand Down
2 changes: 2 additions & 0 deletions public/local/data/api/chat-session.js
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,8 @@ export const createChatSession = ({ provider, model, temperature }) => {
getCapabilities: () => ({ ...capabilities }),
canContinue: () => state.history.length === 0 || canContinue(state),
getSearchData: () => state.searchData,
getUsedChunks: () => state.contextState?.usedChunks ?? [],
getChunkTexts: () => state.contextState?.chunkTexts ?? {},
getModel: () => ({ provider, model }),
getTokenUsage: () => getTokenUsage(state),
getHistory: () => [...state.history],
Expand Down
8 changes: 6 additions & 2 deletions public/local/data/api/chat.js
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ export const BASE_TOKEN_ESTIMATE = estimateTokens(
* @param {number} [options.maxChunks] - Optional max number of chunks to include
* @param {boolean} [options.forMultiTurn=false] - Use larger cushion for multi-turn
* @param {boolean} [options.isFirstTurn=false] - Skip ratio on first turn to maximize initial context
* @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>}
* @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, chunkTexts: Object, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>}
*/
export const buildContextFromChunks = async ({
chunks,
Expand Down Expand Up @@ -163,6 +163,7 @@ export const buildContextFromChunks = async ({
// Each entry: { url, content, tokenCount }
const contextEntries = [];
const seenSlugs = new Map(); // slug -> index in contextEntries
const chunkTexts = {}; // {slug:start:end} -> text excerpt

for (const chunk of chunksToProcess) {
const post = await getPost(chunk.slug);
Expand Down Expand Up @@ -192,6 +193,7 @@ export const buildContextFromChunks = async ({
entry.content += CHUNK_COMBINE_SEPARATOR + chunkText;
totalContextTokensEst += chunkTokensEst;
usedChunks.push(chunk);
chunkTexts[`${chunk.slug}:${chunk.start}:${chunk.end}`] = chunkText;
continue;
}
// "duplicate" mode falls through to add as new entry
Expand Down Expand Up @@ -228,6 +230,7 @@ export const buildContextFromChunks = async ({
// Accumulate tokens and track chunk
totalContextTokensEst += chunkTokensEst;
usedChunks.push(chunk);
chunkTexts[`${chunk.slug}:${chunk.start}:${chunk.end}`] = chunkText;

if (DEBUG_TOKENS) {
// eslint-disable-next-line no-undef
Expand Down Expand Up @@ -278,6 +281,7 @@ export const buildContextFromChunks = async ({
context,
usedChunks,
chunkCount: usedChunks.length,
chunkTexts,
tokenEstimate: totalContextTokensEst,
// Granular token breakdown for UI display
tokenBreakdown: {
Expand All @@ -298,7 +302,7 @@ export const buildContextFromChunks = async ({
* @param {string} options.provider - LLM provider key
* @param {string} options.model - Model ID
* @param {number} options.targetChunkCount - Target number of chunks (will be clamped to MIN_CONTEXT_CHUNKS)
* @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>}
* @returns {Promise<{context: string, usedChunks: Array, chunkCount: number, chunkTexts: Object, tokenEstimate: number, tokenBreakdown: {basePromptTokens: number, queryTokens: number, chunksTokens: number, totalTokens: number}}>}
*/
export const rebuildContextWithLimit = async ({
chunks,
Expand Down
4 changes: 4 additions & 0 deletions public/local/data/api/rag.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ export const performRagSearch = async ({
chunkCount: contextResult.chunkCount,
tokenBreakdown: contextResult.tokenBreakdown,
rawChunks: chunks,
usedChunks: contextResult.usedChunks,
chunkTexts: contextResult.chunkTexts,
initialQuery: query,
};

Expand Down Expand Up @@ -111,6 +113,8 @@ export const reduceContext = async ({ contextState, provider, model }) => {
chunkCount: result.chunkCount,
tokenBreakdown: result.tokenBreakdown,
rawChunks,
usedChunks: result.usedChunks,
chunkTexts: result.chunkTexts,
initialQuery,
};
} catch (err) {
Expand Down
40 changes: 40 additions & 0 deletions public/styles.css
Original file line number Diff line number Diff line change
Expand Up @@ -1454,3 +1454,43 @@ button.loading-status-icon-button {
display: flex;
align-items: center;
}

/* Chunk Excerpts */
.chunk-excerpts {
max-height: 400px;
overflow-y: auto;
padding: 12px 16px;
background: var(--color-bg-light);
}

.chunk-excerpt {
margin-bottom: 12px;
padding: 10px 12px;
background: var(--color-bg-white);
border: 1px solid var(--color-border-light);
border-radius: 4px;
}

.chunk-excerpt:last-child {
margin-bottom: 0;
}

.chunk-label {
font-size: 11px;
font-weight: 600;
color: var(--color-text-secondary);
margin-bottom: 6px;
}

.chunk-label-num {
font-weight: 400;
color: var(--color-text-muted);
}

.chunk-text {
font-size: 13px;
line-height: 1.5;
color: var(--color-text-primary);
white-space: pre-wrap;
word-wrap: break-word;
}
Loading