From 7a9363b502fb18299b5a4ec9cf6e5cf10a37fea5 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Wed, 12 Aug 2026 13:07:32 -0400 Subject: [PATCH] feat(app): histogram and filters across every selected source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The results table already spanned sources; the histogram and the filters sidebar still picked a component per case at the call site. The page now renders SearchHistogram and SearchTotalCount, which take the same per-source list the table does. One source keeps the full DBTimeChart — severity grouping, series drill-down and focus, the pinned tooltip — because a single source has a severity vocabulary to group by. Several sources can't share one, so they stack a count() series each. DBSearchPageFilters takes a list too, so the sidebar people already use works across a selection: facet fields and values merge, value counts are summed so a percentage describes the whole search, "load more" fans out and unions, and pins read as a union and write to every selected source. A source whose table lacks a filtered column is excluded from the results with the reason on its status chip, rather than quietly returning rows that ignore the filter. With one source every path is the one it was before: analysis-mode tabs, denoise, shared filters, and dropping percentages when a distribution query fails. --- .changeset/multi-source-filters.md | 12 + packages/app/src/DBSearchPage.tsx | 156 +++++-- .../DBSearchPage.directTrace.test.tsx | 4 + .../src/components/DBSearchPageFilters.tsx | 119 +++-- .../DBSearchPageFilters/NestedFilterGroup.tsx | 7 +- .../components/DBSearchPageFilters/hooks.ts | 145 ++++++ .../app/src/components/SearchHistogram.tsx | 421 ++++++++++++++++++ .../__tests__/DBSearchPageFilters.test.tsx | 59 +-- packages/app/src/hooks/useMetadata.tsx | 76 +++- packages/app/src/searchFilters.tsx | 100 +++++ 10 files changed, 1002 insertions(+), 97 deletions(-) create mode 100644 .changeset/multi-source-filters.md create mode 100644 packages/app/src/components/SearchHistogram.tsx diff --git a/.changeset/multi-source-filters.md b/.changeset/multi-source-filters.md new file mode 100644 index 0000000000..d933bb2629 --- /dev/null +++ b/.changeset/multi-source-filters.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/app': minor +--- + +The filters sidebar works across every selected source. Facet fields and +values merge across sources, value counts are summed, "load more" fans out, +and pins (personal and team-shared) read as a union and apply to the whole +selection. Checking a value filters every source that has the field; a source +whose table lacks a filtered column is excluded from the results with a +visible reason on its status chip instead of silently returning unfiltered +rows. Filter pills and add-to-filter from the row side panel work across +sources too. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index c11ff3efe1..caa99bf8df 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -32,6 +32,7 @@ import { } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { + ALERT_COUNT_DEFAULT_SELECT, buildMultiSourceSearchConfig, buildSearchChartConfig, } from '@hyperdx/common-utils/dist/core/searchChartConfig'; @@ -92,7 +93,7 @@ import { AlertStatusIcon } from '@/components/AlertStatusIcon'; import { ContactSupportText } from '@/components/ContactSupportText'; import { DBSearchPageFilters } from '@/components/DBSearchPageFilters'; import { cleanClickHouseExpression } from '@/components/DBSearchPageFilters/utils'; -import { DBTimeChart, type SeriesGroupFilter } from '@/components/DBTimeChart'; +import { type SeriesGroupFilter } from '@/components/DBTimeChart'; import EmptyState from '@/components/EmptyState'; import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; @@ -100,12 +101,16 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover' import { InputControlled } from '@/components/InputControlled'; import MultiSourceColumnPicker from '@/components/MultiSourceColumnPicker'; import OnboardingModal from '@/components/OnboardingModal'; +import { + SearchHistogram, + type SearchHistogramSpec, + SearchTotalCount, +} from '@/components/SearchHistogram'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; import SearchPageActionBar from '@/components/SearchPageActionBar'; import SearchResultsTable from '@/components/SearchResultsTable'; -import SearchTotalCountChart from '@/components/SearchTotalCountChart'; import { SourceMultiSelectControlled } from '@/components/SourceMultiSelect'; import { TableSourceForm } from '@/components/Sources/SourceForm'; import { SourceSelectControlled } from '@/components/SourceSelect'; @@ -343,12 +348,12 @@ function ExpandFiltersButton({ onExpand }: { onExpand: () => void }) { function SearchResultsCountGroup({ isFilterSidebarCollapsed, onExpandFilters, - histogramTimeChartConfig, + histogramSpecs, enableParallelQueries, }: { isFilterSidebarCollapsed: boolean; onExpandFilters: () => void; - histogramTimeChartConfig: BuilderChartConfigWithDateRange; + histogramSpecs: SearchHistogramSpec[]; enableParallelQueries?: boolean; }) { return ( @@ -356,8 +361,8 @@ function SearchResultsCountGroup({ {isFilterSidebarCollapsed && ( )} - @@ -1785,6 +1790,45 @@ export function DBSearchPage() { searchedTimeRange, ]); + const multiHistogramSpecs = useMemo(() => { + if (!isMultiSource || isMultiSourceSqlBlocked) return []; + if ( + multiSourceFilters.length > 0 && + columnsBySourceId.size < searchedMultiSources.length + ) { + return []; + } + const where = searchedConfig.where ?? ''; + return searchedMultiSources.map(source => ({ + source, + disabledReason: multiDisabledReasons.get(source.id), + config: { + ...buildMultiSourceSearchConfig(source, { + where, + whereLanguage: 'lucene', + filters: multiSourceFilters, + }), + select: ALERT_COUNT_DEFAULT_SELECT, + orderBy: undefined, + granularity: 'auto' as const, + dateRange: searchedTimeRange, + displayType: DisplayType.StackedBar, + // Match the single-source histogram: reflect the user's exact range + // so chart and table counts agree (see histogramTimeChartConfig). + alignDateRangeToGranularity: false, + dateRangeEndInclusive: true, + }, + })); + }, [ + isMultiSource, + isMultiSourceSqlBlocked, + searchedMultiSources, + searchedConfig.where, + multiSourceFilters, + multiDisabledReasons, + columnsBySourceId, + searchedTimeRange, + ]); // --- End multi-source search --------------------------------------------- // query error handling @@ -2150,6 +2194,20 @@ export function DBSearchPage() { searchedConfig.select, ]); + // The chart/count query plan for however many sources are selected: one + // source keeps its severity-grouped histogram, several get one count() + // series each (stacked by source). + const histogramSpecs = useMemo(() => { + if (isMultiSource) return multiHistogramSpecs; + if (searchedSource == null || histogramTimeChartConfig == null) return []; + return [{ source: searchedSource, config: histogramTimeChartConfig }]; + }, [ + isMultiSource, + multiHistogramSpecs, + searchedSource, + histogramTimeChartConfig, + ]); + const onFormSubmit = useCallback>( e => { e.preventDefault(); @@ -2231,6 +2289,22 @@ export function DBSearchPage() { }; }, [chartConfig, searchedTimeRange, aliasWith]); + // The sidebar reads facets, values, and pins across everything selected; + // with one source that is exactly the single-source sidebar. + const filterSidebarSources = useMemo(() => { + if (isMultiSource) { + // Facet queries want each source's search shape (its own FROM, + // connection, and WHERE), not the aggregated histogram config. + return searchStreamSpecs.map(({ source, config }) => ({ + source, + config: { ...config, orderBy: undefined }, + })); + } + return searchedSource != null + ? [{ source: searchedSource, config: filtersChartConfig }] + : []; + }, [isMultiSource, searchStreamSpecs, searchedSource, filtersChartConfig]); + const openNewSourceModal = useCallback(() => { setNewSourceModalOpened(true); }, []); @@ -2789,7 +2863,7 @@ export function DBSearchPage() { height: '100%', }} > - {!isFilterSidebarCollapsed && !isMultiSource && ( + {!isFilterSidebarCollapsed && ( setIsFilterSidebarCollapsed(false) } - histogramTimeChartConfig={histogramTimeChartConfig} + histogramSpecs={histogramSpecs} /> - ) : ( <> - {/* The histogram, total count, and filters sidebar come - with the next change; searching several sources - returns the merged results table on its own. */} + + + + {isFilterSidebarCollapsed && ( + + setIsFilterSidebarCollapsed(false) + } + /> + )} + + + {shouldShowLiveModeHint && ( + + )} + + + + + setIsFilterSidebarCollapsed(false) } - histogramTimeChartConfig={histogramTimeChartConfig} + histogramSpecs={histogramSpecs} enableParallelQueries /> @@ -2990,14 +3095,9 @@ export function DBSearchPage() { className={searchPageStyles.timeChartContainer} mih="0" > - () =>
); // Multi-source components pull in DBRowSidePanel (and its deep import graph), // which this test isolates away just like DBSqlRowTableWithSidebar above. jest.mock('../components/SearchResultsTable', () => () =>
); +jest.mock('../components/SearchHistogram', () => ({ + SearchHistogram: () =>
, + SearchTotalCount: () =>
, +})); jest.mock('../components/PatternTable', () => () =>
); jest.mock('../components/Search/DBSearchHeatmapChart', () => ({ DBSearchHeatmapChart: () =>
, diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 67a154aae7..55a60c41f7 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -8,6 +8,7 @@ import { FilterState } from '@hyperdx/common-utils/dist/filters'; import { BuilderChartConfigWithDateRange, SourceKind, + TSource, } from '@hyperdx/common-utils/dist/types'; import { Accordion, @@ -51,22 +52,23 @@ import { import { IS_CLICKHOUSE_BUILD } from '@/config'; import { useColumns, - useGetValuesDistribution, useJsonColumns, + useMergedValuesDistribution, useTableMetadata, } from '@/hooks/useMetadata'; +import { useMultiSourceColumns } from '@/hooks/useMultiSourceSearch'; import useResizable from '@/hooks/useResizable'; import { usePinnedFiltersApi } from '@/pinnedFilters'; import { FilterStateHook, IS_ROOT_SPAN_COLUMN_NAME, - usePinnedFilters, + usePinnedFiltersForSources, } from '@/searchFilters'; import { useSource } from '@/source'; import { useLocalStorage } from '@/utils'; import { FilterSettingsPanel } from './DBSearchPageFilters/FilterSettingsPopover'; -import { useFetchFacets } from './DBSearchPageFilters/hooks'; +import { useFetchFacetsForSources } from './DBSearchPageFilters/hooks'; import { NestedFilterGroup } from './DBSearchPageFilters/NestedFilterGroup'; import { PinShareIndicator, @@ -82,6 +84,17 @@ import { import resizeStyles from '@styles/ResizablePanel.module.scss'; import classes from '@styles/SearchPage.module.scss'; +// Placeholder used when no source is selected yet; nothing queries with it. +const EMPTY_FILTER_CHART_CONFIG = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql' as const, + dateRange: [new Date(0), new Date(0)] as [Date, Date], +}; + /* The initial number of values per filter to render */ const INITIAL_MAX_VALUES_DISPLAYED = 10; @@ -402,7 +415,8 @@ export type FilterGroupProps = { isDefaultExpanded?: boolean; showFilterCounts?: boolean; 'data-testid'?: string; - chartConfig: BuilderChartConfigWithDateRange; + /** One config per selected source; value counts are summed across them. */ + chartConfigs: BuilderChartConfigWithDateRange[]; isLive?: boolean; onRangeChange?: (range: { min: number; max: number }) => void; distributionKey?: string; @@ -428,7 +442,7 @@ const FilterGroupBody = ({ onLoadMore, loadMoreLoading, hasLoadedMore, - chartConfig, + chartConfigs, isLive, distributionKey, showDistributions, @@ -449,7 +463,7 @@ const FilterGroupBody = ({ onLoadMore: (key: string) => void; loadMoreLoading: boolean; hasLoadedMore: boolean; - chartConfig: BuilderChartConfigWithDateRange; + chartConfigs: BuilderChartConfigWithDateRange[]; isLive?: boolean; distributionKey?: string; showDistributions: boolean; @@ -463,16 +477,19 @@ const FilterGroupBody = ({ const [recentlyMoved, setRecentlyMoved] = useState>( new Set(), ); - // For live searches, don't refresh percentages when date range changes + // For live searches, don't refresh percentages when date range changes. + // Every source's config carries the same searched range, so the first one + // speaks for all of them. + const primaryDateRange = chartConfigs[0]?.dateRange; const [dateRange, setDateRange] = useState<[Date, Date]>( - chartConfig.dateRange, + primaryDateRange ?? EMPTY_FILTER_CHART_CONFIG.dateRange, ); useEffect(() => { - if (!isLive) { - setDateRange(chartConfig.dateRange); + if (!isLive && primaryDateRange != null) { + setDateRange(primaryDateRange); } - }, [chartConfig.dateRange, isLive]); + }, [primaryDateRange, isLive]); const handleSetSearch = useCallback( (value: string) => { @@ -484,25 +501,23 @@ const FilterGroupBody = ({ [hasLoadedMore, name, onLoadMore], ); + const distributionConfigs = useMemo( + () => chartConfigs.map(config => ({ ...config, dateRange })), + [chartConfigs, dateRange], + ); const { data: distributionData, isFetching: isFetchingDistribution, error: distributionError, - } = useGetValuesDistribution( + } = useMergedValuesDistribution( { - chartConfig: { ...chartConfig, dateRange }, + chartConfigs: distributionConfigs, key: distributionKey || name, limit: 100, // The 100 most common values are enough to find any values that are present in at least 1% of rows }, - { - enabled: showDistributions, - }, + { enabled: showDistributions }, ); - useEffect(() => { - onFetchingDistributionChange(isFetchingDistribution); - }, [isFetchingDistribution, onFetchingDistributionChange]); - useEffect(() => { if (distributionError) { notifications.show({ @@ -515,6 +530,10 @@ const FilterGroupBody = ({ } }, [distributionError, onDistributionError]); + useEffect(() => { + onFetchingDistributionChange(isFetchingDistribution); + }, [isFetchingDistribution, onFetchingDistributionChange]); + const totalAppliedFiltersSize = selectedValues.included.size + selectedValues.excluded.size + @@ -900,7 +919,7 @@ export const FilterGroup = ({ isDefaultExpanded, showFilterCounts, 'data-testid': dataTestId, - chartConfig, + chartConfigs, isLive, distributionKey, onRangeChange, @@ -1040,7 +1059,7 @@ export const FilterGroup = ({ onLoadMore={onLoadMore} loadMoreLoading={loadMoreLoading} hasLoadedMore={hasLoadedMore} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} distributionKey={distributionKey} showDistributions={showDistributions} @@ -1061,10 +1080,9 @@ const DBSearchPageFiltersComponent = ({ clearFilter, setFilterValue: _setFilterValue, isLive, - chartConfig, + sources, analysisMode, setAnalysisMode, - sourceId, showDelta, denoiseResults, setDenoiseResults, @@ -1076,8 +1094,11 @@ const DBSearchPageFiltersComponent = ({ analysisMode: 'results' | 'delta' | 'pattern'; setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void; isLive: boolean; - chartConfig: BuilderChartConfigWithDateRange; - sourceId?: string; + /** + * One entry per selected source. Facets, values, and pins merge across all + * of them; a single source behaves exactly as before. + */ + sources: { source: TSource; config: BuilderChartConfigWithDateRange }[]; showDelta: boolean; denoiseResults: boolean; setDenoiseResults: (denoiseResults: boolean) => void; @@ -1096,6 +1117,23 @@ const DBSearchPageFiltersComponent = ({ }, [_setFilterValue], ); + // The first selected source is the "primary": it anchors the things that + // are inherently about one table (schema preview, the analysis-mode tabs). + // Everything users read or click in the list itself merges across sources. + const primarySource = sources[0]?.source; + const sourceId = primarySource?.id; + const chartConfig = sources[0]?.config ?? EMPTY_FILTER_CHART_CONFIG; + const sourceIds = useMemo(() => sources.map(s => s.source.id), [sources]); + const chartConfigs = useMemo(() => sources.map(s => s.config), [sources]); + const facetSpecs = useMemo( + () => + sources.map(({ source, config }) => ({ + sourceId: source.id, + chartConfig: config, + })), + [sources], + ); + const { toggleFilterPin, toggleFieldPin, @@ -1111,7 +1149,7 @@ const DBSearchPageFiltersComponent = ({ resetSharedFilters, hasPersonalPins, hasSharedPins, - } = usePinnedFilters(sourceId ?? null); + } = usePinnedFiltersForSources(sourceIds); const { data: pinnedFiltersApiData } = usePinnedFiltersApi(sourceId ?? null); const [isSharedFiltersVisible, setSharedFiltersVisible] = useLocalStorage( 'hdx-shared-filters-visible', @@ -1144,6 +1182,11 @@ const DBSearchPageFiltersComponent = ({ chartConfig.dateRange, ); + // Filter keys can come from any selected source's schema, so escaping is + // resolved against the union of their columns. + const { columnsBySourceId } = useMultiSourceColumns( + useMemo(() => sources.map(s => s.source), [sources]), + ); const { data: columns } = useColumns({ databaseName: chartConfig.from.databaseName, tableName: chartConfig.from.tableName, @@ -1164,10 +1207,13 @@ const DBSearchPageFiltersComponent = ({ // Conditionally backtick-quote facet keys that contain special characters and // match known column names, so they can be used in the ClickHouse query to get // key values. - const knownColumns = useMemo( - () => (columns ? new Set(columns.map(c => c.name)) : new Set()), - [columns], - ); + const knownColumns = useMemo(() => { + const names = new Set(columns?.map(c => c.name) ?? []); + for (const sourceColumns of columnsBySourceId.values()) { + for (const name of sourceColumns) names.add(name); + } + return names; + }, [columns, columnsBySourceId]); const [showMoreFields, setShowMoreFields] = useState(false); const { @@ -1178,9 +1224,8 @@ const DBSearchPageFiltersComponent = ({ loadMoreFacetsForKey, loadMoreLoadingKeys, extraFacetKeys, - } = useFetchFacets({ - chartConfig, - sourceId: sourceId ?? null, + } = useFetchFacetsForSources({ + specs: facetSpecs, dateRange, mode: showAllValues ? 'all' : 'exact', filterState, @@ -1516,7 +1561,7 @@ const DBSearchPageFiltersComponent = ({ ); }) } - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} /> ))} @@ -1571,7 +1616,7 @@ const DBSearchPageFiltersComponent = ({ entry.range != null))) ); })()} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} onRangeChange={range => setFilterRange(facet.key, range)} /> @@ -1598,7 +1643,7 @@ const DBSearchPageFiltersComponent = ({ loadMoreLoadingKeys, showFilterCounts, isFacetsLoading, - chartConfig, + chartConfigs, isLive, setFilterRange, tableMetadata, diff --git a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx index 4145d8e917..6291a6136c 100644 --- a/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx +++ b/packages/app/src/components/DBSearchPageFilters/NestedFilterGroup.tsx @@ -39,7 +39,8 @@ type NestedFilterGroupProps = { hasLoadedMore: Record; isDefaultExpanded?: boolean; 'data-testid'?: string; - chartConfig: any; // Using any to avoid importing ChartConfigWithDateRange + /** One config per selected source; counts are summed across them. */ + chartConfigs: any[]; // `any` avoids importing ChartConfigWithDateRange isLive?: boolean; }; @@ -70,7 +71,7 @@ export const NestedFilterGroup = ({ hasLoadedMore, isDefaultExpanded, 'data-testid': dataTestId, - chartConfig, + chartConfigs, isLive, }: NestedFilterGroupProps) => { const selectedValues: FilterState = useMemo( @@ -253,7 +254,7 @@ export const NestedFilterGroup = ({ hasLoadedMore={hasLoadedMore[child.key] || false} showFilterCounts={showFilterCounts} isDefaultExpanded={childHasSelections} - chartConfig={chartConfig} + chartConfigs={chartConfigs} isLive={isLive} />
diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index a9d676102a..d9dd4319e6 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -17,6 +17,7 @@ import { useMapColumns, useMetadataWithSettings, } from '@/hooks/useMetadata'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; import { escapeFilterStateKeys, usePinnedFilters } from '@/searchFilters'; import { useSource } from '@/source'; import { mergePath } from '@/utils'; @@ -372,3 +373,147 @@ export function useFetchFacets({ extraFacetKeys, }; } + +export type SourceFacetSpec = { + sourceId: string; + chartConfig: BuilderChartConfigWithDateRange; +}; + +/** Slot hook: the full facet pipeline for one selected source. */ +function useSourceFacetsSlot( + spec: SourceFacetSpec | undefined, + opts: { + dateRange: [Date, Date]; + mode: 'all' | 'exact'; + filterState?: FilterState; + showMoreFields?: boolean; + }, +) { + const query = useFetchFacets({ + chartConfig: spec?.chartConfig ?? STUB_FACET_CONFIG, + sourceId: spec?.sourceId ?? null, + dateRange: opts.dateRange, + mode: opts.mode, + filterState: opts.filterState, + showMoreFields: opts.showMoreFields, + enabled: spec != null, + }); + return query; +} + +const STUB_FACET_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +/** + * Facets across every selected source: fields and values merged by field + * path, values unioned in first-seen order. "Load more" fans out to each + * source and unions what comes back, so a high-cardinality field expands + * across the whole search rather than one table. + * + * With a single source this is `useFetchFacets` for that source, unchanged. + */ +export function useFetchFacetsForSources({ + specs, + dateRange, + mode, + filterState, + showMoreFields, +}: { + specs: SourceFacetSpec[]; + dateRange: [Date, Date]; + mode: 'all' | 'exact'; + filterState?: FilterState; + showMoreFields?: boolean; +}) { + const slots = useMultiSourceSlots(specs, useSourceFacetsSlot, { + dateRange, + mode, + filterState, + showMoreFields, + }); + + const merged = useMemo(() => { + const byKey = new Map< + string, + { values: (string | boolean)[]; seen: Set } + >(); + let sawAny = false; + for (const slot of slots) { + const facets = slot.data.keyValues; + if (facets == null) continue; + sawAny = true; + for (const facet of facets) { + let entry = byKey.get(facet.key); + if (entry == null) { + entry = { values: [], seen: new Set() }; + byKey.set(facet.key, entry); + } + for (const value of facet.value) { + if (!entry.seen.has(value)) { + entry.seen.add(value); + entry.values.push(value); + } + } + } + } + const keyValues = sawAny + ? [...byKey.entries()].map(([key, entry]) => ({ + key, + value: entry.values, + })) + : undefined; + + const keys = slots.flatMap(slot => slot.data.keys ?? []); + const seenPaths = new Set(); + const mergedKeys = keys.filter(field => { + const id = `${field.path.join('.')}|${field.type}`; + if (seenPaths.has(id)) return false; + seenPaths.add(id); + return true; + }); + + return { keys: mergedKeys.length > 0 ? mergedKeys : undefined, keyValues }; + }, [slots]); + + const loadMoreFacetsForKey = useCallback( + async (key: string) => { + await Promise.all(slots.map(slot => slot.loadMoreFacetsForKey(key))); + }, + [slots], + ); + + const loadMoreLoadingKeys = useMemo(() => { + const keys = new Set(); + for (const slot of slots) { + for (const key of slot.loadMoreLoadingKeys) keys.add(key); + } + return keys; + }, [slots]); + + const extraFacetKeys = useMemo(() => { + const keys = new Set(); + for (const slot of slots) { + for (const key of slot.extraFacetKeys) keys.add(key); + } + return keys; + }, [slots]); + + return { + data: merged, + isLoading: slots.some(s => s.isLoading), + isFetching: slots.some(s => s.isFetching), + // A single failing source shouldn't blank the sidebar; surface the first. + error: slots.find(s => s.error != null)?.error, + loadMoreFacetsForKey, + loadMoreLoadingKeys, + extraFacetKeys, + areExtraFacetsLoading: slots.some(s => s.areExtraFacetsLoading), + }; +} diff --git a/packages/app/src/components/SearchHistogram.tsx b/packages/app/src/components/SearchHistogram.tsx new file mode 100644 index 0000000000..55dbaf1481 --- /dev/null +++ b/packages/app/src/components/SearchHistogram.tsx @@ -0,0 +1,421 @@ +import { useMemo, useState } from 'react'; +import { + ColumnMetaType, + filterColumnMetaByType, + JSDataType, + ResponseJSON, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { + BuilderChartConfigWithDateRange, + DisplayType, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { Text } from '@mantine/core'; +import { keepPreviousData } from '@tanstack/react-query'; + +import api from '@/api'; +import { + convertToTimeChartConfig, + formatResponseForTimeChart, + useTimeChartSettings, +} from '@/ChartUtils'; +import ChartContainer from '@/components/charts/ChartContainer'; +import ChartErrorState from '@/components/charts/ChartErrorState'; +import { type ActiveClickPayload, MemoChart } from '@/HDXMultiSeriesTimeChart'; +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; +import type { NumberFormat } from '@/types'; + +import { DBTimeChart, type SeriesGroupFilter } from './DBTimeChart'; +import { getMultiSourceColor } from './MultiSourceBadge'; +import SearchTotalCountChart from './SearchTotalCountChart'; + +/** Synthetic group column tagged onto each source's histogram rows. */ +const SOURCE_GROUP_COLUMN = '__hdx_source'; + +export type SearchHistogramSpec = { + source: TSource; + /** Per-source count() histogram config (canonical WHERE, no groupBy). */ + config: BuilderChartConfigWithDateRange; + /** When set, the source doesn't run (mirrors MultiSourceStreamSpec). */ + disabledReason?: string; +}; + +// Placeholder for unused hook slots; never queried (enabled: false). +const STUB_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +type HistogramSlotState = { + data: ReturnType['data']; + isLoading: boolean; + isError: boolean; + error: Error | null; +}; + +function useHistogramSlot( + spec: SearchHistogramSpec | undefined, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible, + }: { + enabled: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + parallelizeWhenPossible?: boolean; + }, +): HistogramSlotState { + const queriedConfig = useMemo( + () => convertToTimeChartConfig(spec?.config ?? STUB_CONFIG), + [spec?.config], + ); + + const { data, isLoading, isError, error } = useQueriedChartConfig( + queriedConfig, + { + // Key shape mirrors DBTimeChart/SearchTotalCountChart so TanStack can + // de-dupe the histogram and total-count consumers of the same source. + queryKey: [ + queryKeyPrefix, + queriedConfig, + 'chunked', + { + disableQueryChunking: false, + enableParallelQueries, + parallelizeWhenPossible, + }, + ], + placeholderData: keepPreviousData, + enableQueryChunking: true, + enableParallelQueries: enableParallelQueries && parallelizeWhenPossible, + enabled: enabled && spec != null && spec.disabledReason == null, + }, + ); + + // Stable identity per content change so the slots array (and everything + // memoized on it) doesn't churn on unrelated renders. + return useMemo( + () => ({ data, isLoading, isError, error: error ?? null }), + [data, isLoading, isError, error], + ); +} + +/** + * Runs one count() histogram query per selected source (one hook slot per + * source, see useMultiSourceSlots) and merges the responses into a single + * response shape with a synthetic source-name group column — so the standard + * time-chart transform naturally yields one series per source. + */ +function useMultiSourceHistogram( + specs: SearchHistogramSpec[], + { + enabled = true, + queryKeyPrefix, + enableParallelQueries, + }: { + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + }, +) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const slots = useMultiSourceSlots(specs, useHistogramSlot, { + enabled: enabled && !isLoadingMe, + queryKeyPrefix, + enableParallelQueries, + parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, + }); + + const isLoading = slots.some(s => s.isLoading); + const allFailed = slots.length > 0 && slots.every(s => s.isError); + const anyError = slots.some(s => s.isError); + const error = slots.find(s => s.error != null)?.error ?? undefined; + const isComplete = + slots.length > 0 && slots.every(s => s.isError || !!s.data?.isComplete); + + const mergedResponse: ResponseJSON> | undefined = + useMemo(() => { + let meta: ColumnMetaType[] | undefined; + const data: Record[] = []; + for (let i = 0; i < specs.length; i++) { + const response = slots[i]?.data; + if (response?.meta == null || response.meta.length === 0) continue; + if (meta == null) { + meta = [ + ...response.meta, + { name: SOURCE_GROUP_COLUMN, type: 'String' }, + ]; + } + const sourceName = specs[i].source.name; + for (const row of response.data ?? []) { + data.push({ ...row, [SOURCE_GROUP_COLUMN]: sourceName }); + } + } + return meta ? { data, meta, rows: data.length } : undefined; + }, [slots, specs]); + + return { mergedResponse, isLoading, allFailed, anyError, error, isComplete }; +} + +const EMPTY_NUMBER_FORMATS = new Map(); + +/** + * The multi-source search histogram: one stacked count() series per selected + * source, colored consistently with the results-table badges. A thin + * counterpart to DBTimeChart — drag-to-zoom and the legend work; per-series + * drill-down/pinned tooltips are single-source features and are omitted. + */ +function MergedSourcesTimeChart({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + showLegend = true, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + showLegend?: boolean; +}) { + const { mergedResponse, isLoading, allFailed, error, isComplete } = + useMultiSourceHistogram(specs, { + enabled, + queryKeyPrefix, + enableParallelQueries, + }); + + const firstConfig = specs[0]?.config; + const { dateRange, granularity } = useTimeChartSettings( + firstConfig ?? STUB_CONFIG, + ); + + const [activeClickPayload, setActiveClickPayload] = useState< + ActiveClickPayload | undefined + >(); + + const colorBySourceName = useMemo( + () => + new Map( + specs.map((spec, i) => [spec.source.name, getMultiSourceColor(i)]), + ), + [specs], + ); + + const formatted = useMemo(() => { + if (mergedResponse == null) { + return null; + } + try { + const result = formatResponseForTimeChart({ + currentPeriodResponse: mergedResponse, + dateRange, + granularity, + generateEmptyBuckets: true, + }); + // One series per source: recolor to the shared per-source palette so + // the histogram matches the table badges and status chips. + for (const line of result.lineData) { + const color = colorBySourceName.get(line.dataKey); + if (color != null) { + line.color = color; + } + } + return result; + } catch (e) { + console.error(e); + return null; + } + }, [mergedResponse, dateRange, granularity, colorBySourceName]); + + if (allFailed && error) { + return ; + } + + return ( + + {isLoading && formatted == null ? ( +
+ Loading Chart Data... +
+ ) : formatted == null || formatted.graphResults.length === 0 ? ( +
+ No data found within time range. +
+ ) : ( + + )} + + ); +} + +/** + * Summed "N Results" across every selected source, sharing the histogram's + * per-source queries (identical query keys) so it adds no ClickHouse load. + */ +function MergedSourcesTotalCount({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + const { mergedResponse, isLoading, allFailed } = useMultiSourceHistogram( + specs, + { + enabled, + queryKeyPrefix, + enableParallelQueries, + }, + ); + + const totalCount = useMemo(() => { + if (mergedResponse == null) return undefined; + // The count column may be renamed (e.g. via materialized views); fall back + // to the first numeric column, mirroring SearchTotalCountChart. + const countColumn = + mergedResponse.meta?.find(c => c.name === 'count()')?.name ?? + filterColumnMetaByType(mergedResponse.meta ?? [], [ + JSDataType.Number, + ])?.[0]?.name ?? + 'count()'; + return mergedResponse.data.reduce( + (sum: number, row: any) => sum + (Number.parseInt(row[countColumn]) || 0), + 0, + ); + }, [mergedResponse]); + + return ( + + {isLoading && totalCount == null ? ( + ··· Results + ) : totalCount != null && !allFailed ? ( + `${totalCount.toLocaleString()} Results` + ) : ( + '0 Results' + )} + + ); +} + +/** + * The search page's histogram, for any number of selected sources. + * + * One source keeps the full-featured DBTimeChart — series drill-down, focus, + * the pinned tooltip, MV optimization — grouped by severity/status, which is + * what a single source's chart has always shown. Several sources can't share + * a severity vocabulary, so they stack one count() series per source instead, + * and the merged chart trades the per-series drill-down for that. + */ +export function SearchHistogram({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, + onTimeRangeSelect, + onFocusSeries, + showLegend, +}: { + /** One spec per selected source; N=1 is the single-source histogram. */ + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; + onTimeRangeSelect?: (start: Date, end: Date) => void; + /** Focus a severity/status series into the search (single source only). */ + onFocusSeries?: (filters: SeriesGroupFilter[]) => void; + showLegend?: boolean; +}) { + if (specs.length === 1) { + return ( + + ); + } + + return ( + + ); +} + +/** + * "N Results" for any number of sources: the single-source count query, or the + * sum across sources. Shares the histogram's per-source queries either way, so + * it adds no ClickHouse load. + */ +export function SearchTotalCount({ + specs, + enabled = true, + queryKeyPrefix, + enableParallelQueries, +}: { + specs: SearchHistogramSpec[]; + enabled?: boolean; + queryKeyPrefix: string; + enableParallelQueries?: boolean; +}) { + if (specs.length === 1) { + return ( + + ); + } + + return ( + + ); +} diff --git a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx index 0353c68d4f..a1d4359c93 100644 --- a/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx +++ b/packages/app/src/components/__tests__/DBSearchPageFilters.test.tsx @@ -1,4 +1,3 @@ -import { UseQueryResult } from '@tanstack/react-query'; import { screen, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -13,7 +12,7 @@ import { groupFacetsByBaseName, parseMapFieldName, } from '@/components/DBSearchPageFilters/utils'; -import { useGetValuesDistribution } from '@/hooks/useMetadata'; +import { useMergedValuesDistribution } from '@/hooks/useMetadata'; describe('cleanClickHouseExpression', () => { it('should remove toString wrapper', () => { @@ -50,9 +49,9 @@ describe('cleanClickHouseExpression', () => { }); jest.mock('@/hooks/useMetadata', () => ({ - useGetValuesDistribution: jest + useMergedValuesDistribution: jest .fn() - .mockReturnValue({ data: undefined, isFetching: false, error: undefined }), + .mockReturnValue({ data: undefined, isFetching: false, error: null }), })); describe('cleanedFacetName', () => { @@ -396,18 +395,20 @@ describe('FilterGroup', () => { loadMoreLoading: false, hasLoadedMore: false, isDefaultExpanded: true, - chartConfig: { - from: { - databaseName: 'test_db', - tableName: 'test_table', + chartConfigs: [ + { + from: { + databaseName: 'test_db', + tableName: 'test_table', + }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: '', + connection: 'test_connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], }, - select: '', - where: '', - whereLanguage: 'sql', - timestampValueExpression: '', - connection: 'test_connection', - dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], - }, + ], }; it('should sort options alphabetically by default', () => { @@ -441,7 +442,7 @@ describe('FilterGroup', () => { }); it('should show selected items first, then sort by counts, if percentages when they are enabled', () => { - jest.mocked(useGetValuesDistribution).mockReturnValue({ + jest.mocked(useMergedValuesDistribution).mockReturnValue({ data: new Map([ ['apple', 30], ['banana', 20], @@ -449,7 +450,7 @@ describe('FilterGroup', () => { ]), isFetching: false, error: null, - } as UseQueryResult>); + }); renderWithMantine( { }); it('should show percentages, if enabled', async () => { - jest.mocked(useGetValuesDistribution).mockReturnValue({ + jest.mocked(useMergedValuesDistribution).mockReturnValue({ data: new Map([ ['apple', 99.2], ['zebra', 0.6], ]), isFetching: false, error: null, - } as UseQueryResult>); + }); renderWithMantine( { onLoadMore: jest.fn(), loadMoreLoading: {} as Record, hasLoadedMore: {} as Record, - chartConfig: { - from: { databaseName: 'test_db', tableName: 'test_table' }, - select: '', - where: '', - whereLanguage: 'sql', - timestampValueExpression: '', - connection: 'test_connection', - dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], - }, + chartConfigs: [ + { + from: { databaseName: 'test_db', tableName: 'test_table' }, + select: '', + where: '', + whereLanguage: 'sql', + timestampValueExpression: '', + connection: 'test_connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], + }, + ], }; it('should not render child FilterGroups when collapsed', () => { diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index fca1559d5d..7e9acfda89 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -32,6 +32,7 @@ import api from '@/api'; import { IS_LOCAL_MODE } from '@/config'; import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT } from '@/defaults'; +import { useMultiSourceSlots } from '@/hooks/useSourceSlots'; import { getMetadata } from '@/metadata'; import { useSource, useSources } from '@/source'; import { toArray } from '@/utils'; @@ -449,7 +450,7 @@ export function useMultipleGetKeyValues( }; } -export function useGetValuesDistribution( +function useGetValuesDistribution( { chartConfig, key, @@ -484,6 +485,79 @@ export function useGetValuesDistribution( }); } +/** Slot hook: value→count distribution for one source's config. */ +function useValuesDistributionSlot( + chartConfig: BuilderChartConfigWithDateRange | undefined, + { key, limit, enabled }: { key: string; limit: number; enabled: boolean }, +) { + const { data, isFetching, error } = useGetValuesDistribution( + { + chartConfig: chartConfig ?? STUB_DISTRIBUTION_CONFIG, + key, + limit, + }, + { enabled: enabled && chartConfig != null }, + ); + return useMemo( + () => ({ data, isFetching, error }), + [data, isFetching, error], + ); +} + +const STUB_DISTRIBUTION_CONFIG: BuilderChartConfigWithDateRange = { + connection: '', + from: { databaseName: '', tableName: '' }, + timestampValueExpression: '', + select: '', + where: '', + whereLanguage: 'sql', + dateRange: [new Date(0), new Date(0)], +}; + +/** + * Value→count distribution for a filter key across every selected source, + * summed. With one source this is exactly `useGetValuesDistribution`; with + * several, a value's count is the total across the sources that have it, so + * the sidebar's percentages describe the whole search rather than one table. + */ +export function useMergedValuesDistribution( + { + chartConfigs, + key, + limit, + }: { + chartConfigs: BuilderChartConfigWithDateRange[]; + key: string; + limit: number; + }, + options?: { enabled?: boolean }, +) { + const slots = useMultiSourceSlots(chartConfigs, useValuesDistributionSlot, { + key, + limit, + enabled: options?.enabled ?? true, + }); + + return useMemo(() => { + const merged = new Map(); + let any = false; + for (const slot of slots) { + if (slot.data == null) continue; + any = true; + for (const [value, count] of slot.data) { + merged.set(value, (merged.get(value) ?? 0) + count); + } + } + return { + data: any ? merged : undefined, + isFetching: slots.some(s => s.isFetching), + // Surface the first failure so the group can drop percentages rather + // than showing counts that silently omit a source. + error: slots.find(s => s.error != null)?.error ?? null, + }; + }, [slots]); +} + export function useGetKeyValues( { chartConfig, diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index ac193b7f26..5492f16a71 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -11,6 +11,7 @@ import { cleanClickHouseExpression, toQuotedClickHouseKeyExpression, } from './components/DBSearchPageFilters/utils'; +import { useMultiSourceSlots } from './hooks/useSourceSlots'; import { usePinnedFiltersApi, useUpdatePinnedFilters } from './pinnedFilters'; import { useLocalStorage } from './utils'; @@ -658,3 +659,102 @@ export function usePinnedFilters(sourceId: string | null) { hasSharedPins, }; } + +/** + * Pinned filters across every selected source. + * + * Pins are stored per source (personal pins in localStorage, shared pins in + * Mongo), so a search spanning several sources reads the union of their pins + * and writes to all of them: pinning `ServiceName` while searching logs and + * traces keeps it pinned in both, and unpinning clears it from both. With one + * source this is exactly `usePinnedFilters` for that source. + */ +export function usePinnedFiltersForSources(sourceIds: string[]) { + const slots = useMultiSourceSlots(sourceIds, usePinnedFiltersSlot, undefined); + const active = useMemo( + () => slots.slice(0, sourceIds.length), + [slots, sourceIds.length], + ); + + return useMemo(() => { + // Reads: a pin on any selected source shows in the sidebar. + const anyOf = + ( + pick: (s: PinnedFiltersHook) => (...a: A) => boolean, + ) => + (...args: A) => + active.some(slot => pick(slot)(...args)); + // Writes: fan out so a pin applies to the whole selection. Toggling is + // resolved against the merged view first, so a mixed state (pinned on one + // source, not another) resolves to "pin everywhere" rather than flipping + // each source independently. + const fanOut = + ( + pick: (s: PinnedFiltersHook) => (...a: A) => void, + isSet: (s: PinnedFiltersHook, ...a: A) => boolean, + ) => + (...args: A) => { + const shouldPin = !active.some(slot => isSet(slot, ...args)); + for (const slot of active) { + if (isSet(slot, ...args) !== shouldPin) { + pick(slot)(...args); + } + } + }; + + return { + pinnedFilters: active.reduce( + (acc, slot) => mergePinnedFilterValues(acc, slot.pinnedFilters), + {}, + ), + getPinnedFields: () => [ + ...new Set(active.flatMap(slot => slot.getPinnedFields())), + ], + isFilterPinned: anyOf(s => s.isFilterPinned), + isFieldPinned: anyOf(s => s.isFieldPinned), + isSharedFilterPinned: anyOf(s => s.isSharedFilterPinned), + isSharedFieldPinned: anyOf(s => s.isSharedFieldPinned), + toggleFilterPin: fanOut( + s => s.toggleFilterPin, + (s, property: string, value: string | boolean) => + s.isFilterPinned(property, value), + ), + toggleFieldPin: fanOut( + s => s.toggleFieldPin, + (s, key: string) => s.isFieldPinned(key), + ), + toggleSharedFilterPin: fanOut( + s => s.toggleSharedFilterPin, + (s, property: string, value: string | boolean) => + s.isSharedFilterPinned(property, value), + ), + toggleSharedFieldPin: fanOut( + s => s.toggleSharedFieldPin, + (s, key: string) => s.isSharedFieldPinned(key), + ), + resetPersonalPins: () => active.forEach(s => s.resetPersonalPins()), + resetSharedFilters: () => active.forEach(s => s.resetSharedFilters()), + hasPersonalPins: active.some(s => s.hasPersonalPins), + hasSharedPins: active.some(s => s.hasSharedPins), + }; + }, [active]); +} + +type PinnedFiltersHook = ReturnType; + +/** Slot hook: one source's pins. Unused slots pass null and stay inert. */ +function usePinnedFiltersSlot(sourceId: string | undefined) { + return usePinnedFilters(sourceId ?? null); +} + +/** Union two pinned-filter maps, deduping values per key. */ +function mergePinnedFilterValues( + a: PinnedFilters, + b: PinnedFilters, +): PinnedFilters { + const out: PinnedFilters = { ...a }; + for (const [key, values] of Object.entries(b)) { + out[key] = [...new Set([...(out[key] ?? []), ...values])]; + } + return out; +}