From 9d110ebae427fd37663dca7c0a8be14df393dbd4 Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Fri, 24 Jul 2026 14:15:40 +0200 Subject: [PATCH 01/18] BE-705: Add a dedicated entities-table query endpoint to the Graph The entities table no longer assembles its rows from a subgraph: the new /entities/query/table endpoint serves flat rows built from materialized columns, with the type summary, the count, and the page read in one repeatable-read transaction. When no explicit type filter is given, the visible-type universe is derived from the summary and drives the page query as an indexable include clause. The count rides along in the summary scan when the page carries no narrowing filters. Follow-up pages continue on the first page's database state through an opaque cursor sealing two snapshot instants, the type universe, the sort, the draft visibility, and the keyset position. Property filters pass through the filter-protection rewrite, rows are masked like the generic read path, and link endpoints pass the knowledge-edge permission check. The endpoint is exposed through the Node API as queryEntitiesTable and branded in the TypeScript SDK. --- .gitattributes | 1 + Cargo.lock | 1 + Cargo.toml | 1 + apps/hash-api/src/graphql/resolvers/index.ts | 4 + .../resolvers/knowledge/entity/entity.ts | 14 + libs/@local/graph/api/openapi/openapi.json | 875 ++++++++++- libs/@local/graph/api/src/rest/entity/mod.rs | 27 +- .../graph/api/src/rest/entity/query/mod.rs | 52 +- .../store/postgres/knowledge/entity/mod.rs | 70 +- .../postgres/knowledge/entity/summary.rs | 196 ++- .../store/postgres/knowledge/entity/table.rs | 1362 +++++++++++++++++ .../src/store/postgres/query/ast/mod.rs | 2 +- .../src/store/postgres/query/ast/non_empty.rs | 10 +- .../src/store/postgres/query/mod.rs | 2 +- .../@local/graph/sdk/typescript/src/entity.ts | 115 +- libs/@local/graph/store/Cargo.toml | 1 + libs/@local/graph/store/src/entity/mod.rs | 7 + libs/@local/graph/store/src/entity/store.rs | 27 +- libs/@local/graph/store/src/entity/table.rs | 752 +++++++++ libs/@local/graph/store/src/query/ordering.rs | 2 +- libs/@local/graph/type-fetcher/src/store.rs | 16 +- .../src/graphql/scalar-mapping.ts | 4 + .../type-defs/knowledge/entity.typedef.ts | 6 + .../postgres/email_filter_protection.rs | 122 +- tests/graph/integration/postgres/lib.rs | 17 +- tests/graph/integration/postgres/table.rs | 1248 +++++++++++++++ 26 files changed, 4766 insertions(+), 168 deletions(-) create mode 100644 libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs create mode 100644 libs/@local/graph/store/src/entity/table.rs create mode 100644 tests/graph/integration/postgres/table.rs diff --git a/.gitattributes b/.gitattributes index 54651959e75..7f684d85ebd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ **/docs/dependency-diagram.mmd linguist-generated=true **/*.snap linguist-generated=true **/*.snap.* linguist-generated=true +libs/@local/graph/api/openapi/** linguist-generated=true diff --git a/Cargo.lock b/Cargo.lock index 44052f0507a..94d55ed6005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3803,6 +3803,7 @@ dependencies = [ name = "hash-graph-store" version = "0.0.0" dependencies = [ + "base64", "bytes", "derive-where", "derive_more", diff --git a/Cargo.toml b/Cargo.toml index 5b3342dbe59..b8aacae3bba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,6 +117,7 @@ aws-types = { version = "1.3.10", default-features = fa axum = { version = "0.8.6" } axum-core = { version = "0.5.5" } base16ct = { version = "1.0.0", default-features = false } +base64 = { version = "0.22.1" } bitvec = { version = "1.0.1", default-features = false } bon = { version = "3.8.1", default-features = false, features = ["implied-bounds"] } bstr = { version = "1.12.1" } diff --git a/apps/hash-api/src/graphql/resolvers/index.ts b/apps/hash-api/src/graphql/resolvers/index.ts index 7fa9911468f..719d950ae4d 100644 --- a/apps/hash-api/src/graphql/resolvers/index.ts +++ b/apps/hash-api/src/graphql/resolvers/index.ts @@ -41,6 +41,7 @@ import { createEntityResolver, isEntityPublicResolver, queryEntitiesResolver, + queryEntitiesTableResolver, searchEntitiesResolver, queryEntitySubgraphResolver, removeEntityViewerResolver, @@ -165,6 +166,9 @@ export const resolvers: Omit & { }), summarizeEntities: loggedInAndSignedUpMiddleware(summarizeEntitiesResolver), queryEntities: loggedInAndSignedUpMiddleware(queryEntitiesResolver), + queryEntitiesTable: loggedInAndSignedUpMiddleware( + queryEntitiesTableResolver, + ), searchEntities: loggedInAndSignedUpMiddleware(searchEntitiesResolver), queryEntitySubgraph: loggedInAndSignedUpMiddleware( queryEntitySubgraphResolver, diff --git a/apps/hash-api/src/graphql/resolvers/knowledge/entity/entity.ts b/apps/hash-api/src/graphql/resolvers/knowledge/entity/entity.ts index 65b8668e17c..b8166a0f152 100644 --- a/apps/hash-api/src/graphql/resolvers/knowledge/entity/entity.ts +++ b/apps/hash-api/src/graphql/resolvers/knowledge/entity/entity.ts @@ -11,6 +11,7 @@ import { serializeQueryEntitiesResponse, serializeQueryEntitySubgraphResponse, serializeSearchEntitiesResponse, + queryEntitiesTable, summarizeEntities, type CreateEntityParameters, } from "@local/hash-graph-sdk/entity"; @@ -46,6 +47,7 @@ import type { MutationUpdateEntitiesArgs, MutationUpdateEntityArgs, Query, + QueryQueryEntitiesTableArgs, QuerySummarizeEntitiesArgs, QueryIsEntityPublicArgs, QueryQueryEntitiesArgs, @@ -151,6 +153,18 @@ export const summarizeEntitiesResolver: ResolverFn< request, ); +export const queryEntitiesTableResolver: ResolverFn< + Query["queryEntitiesTable"], + Record, + GraphQLContext, + QueryQueryEntitiesTableArgs +> = async (_, { request }, graphQLContext) => + queryEntitiesTable( + graphQLContextToImpureGraphContext(graphQLContext), + graphQLContext.authentication, + request, + ); + export const queryEntitiesResolver: ResolverFn< Query["queryEntities"], Record, diff --git a/libs/@local/graph/api/openapi/openapi.json b/libs/@local/graph/api/openapi/openapi.json index 8063c5b05c2..d6414a5b399 100644 --- a/libs/@local/graph/api/openapi/openapi.json +++ b/libs/@local/graph/api/openapi/openapi.json @@ -1823,6 +1823,54 @@ } } }, + "/entities/query/table": { + "post": { + "tags": [ + "Graph", + "Entity" + ], + "operationId": "query_entities_table", + "parameters": [ + { + "name": "X-Authenticated-User-Actor-Id", + "in": "header", + "description": "The ID of the actor which is used to authorize the request", + "required": true, + "schema": { + "$ref": "#/components/schemas/ActorEntityUuid" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryEntitiesTableParams" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryEntitiesTableResponse" + } + } + } + }, + "422": { + "description": "Provided query is invalid" + }, + "500": { + "description": "Store error occurred" + } + } + } + }, "/entities/search": { "post": { "tags": [ @@ -4835,103 +4883,688 @@ "firstNonDraftCreatedAtDecisionTime": { "$ref": "#/components/schemas/Timestamp" }, - "firstNonDraftCreatedAtTransactionTime": { - "$ref": "#/components/schemas/Timestamp" + "firstNonDraftCreatedAtTransactionTime": { + "$ref": "#/components/schemas/Timestamp" + } + } + }, + "EntityQueryCursor": { + "type": "array", + "items": { + "type": "object" + } + }, + "EntityQuerySortingPath": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/components/schemas/EntityQuerySortingToken" + }, + { + "type": "string" + }, + { + "type": "number" + } + ] + } + }, + "EntityQuerySortingRecord": { + "type": "object", + "required": [ + "path", + "ordering", + "nulls" + ], + "properties": { + "nulls": { + "$ref": "#/components/schemas/NullOrdering" + }, + "ordering": { + "$ref": "#/components/schemas/Ordering" + }, + "path": { + "$ref": "#/components/schemas/EntityQuerySortingPath" + } + } + }, + "EntityQuerySortingToken": { + "type": "string", + "enum": [ + "uuid", + "archived", + "label", + "editionCreatedAtTransactionTime", + "editionCreatedAtDecisionTime", + "createdAtTransactionTime", + "createdAtDecisionTime", + "typeTitle" + ] + }, + "EntityQueryToken": { + "type": "string", + "description": "A single token in an [`EntityQueryPath`].", + "enum": [ + "uuid", + "editionId", + "draftId", + "archived", + "webId", + "type", + "properties", + "label", + "createdById", + "editionCreatedById", + "createdAtTransactionTime", + "createdAtDecisionTime", + "provenance", + "editionProvenance", + "embedding", + "incomingLinks", + "outgoingLinks", + "leftEntity", + "rightEntity" + ] + }, + "EntityRecordId": { + "type": "object", + "required": [ + "entityId", + "editionId" + ], + "properties": { + "editionId": { + "$ref": "#/components/schemas/EntityEditionId" + }, + "entityId": { + "$ref": "#/components/schemas/EntityId" + } + } + }, + "EntityTableCursor": { + "type": "string", + "description": "An opaque continuation token for the entities table" + }, + "EntityTableFilter": { + "type": "object", + "description": "The scope of the entities table.", + "properties": { + "includeArchived": { + "type": "boolean" + }, + "includeDrafts": { + "type": "boolean" + }, + "propertyFilters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityTablePropertyFilter" + }, + "description": "Conditions on property columns, all of which a row has to satisfy." + }, + "types": { + "$ref": "#/components/schemas/EntityTableTypeScope" + }, + "webs": { + "$ref": "#/components/schemas/EntityTableWebScope" + } + }, + "additionalProperties": false + }, + "EntityTableLinkEndpoint": { + "type": "object", + "description": "One endpoint of a link row: its identity, display label, and direct types.", + "required": [ + "entityId", + "entityTypeIds" + ], + "properties": { + "entityId": { + "$ref": "#/components/schemas/EntityId" + }, + "entityTypeIds": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VersionedUrl" + }, + "description": "The endpoint's direct types." + }, + "label": { + "type": "string" + } + } + }, + "EntityTablePropertyFilter": { + "oneOf": [ + { + "type": "object", + "required": [ + "property", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "hasAnyValue" + ] + } + } + }, + { + "type": "object", + "required": [ + "property", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "isEmpty" + ] + } + } + }, + { + "type": "object", + "required": [ + "property", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "isTrue" + ] + } + } + }, + { + "type": "object", + "required": [ + "property", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "isFalse" + ] + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "equals" + ] + }, + "value": { + "$ref": "#/components/schemas/EntityTablePropertyValue" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "notEquals" + ] + }, + "value": { + "$ref": "#/components/schemas/EntityTablePropertyValue" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "greaterThan" + ] + }, + "value": { + "$ref": "#/components/schemas/Real" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "greaterThanOrEqual" + ] + }, + "value": { + "$ref": "#/components/schemas/Real" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "lessThan" + ] + }, + "value": { + "$ref": "#/components/schemas/Real" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "lessThanOrEqual" + ] + }, + "value": { + "$ref": "#/components/schemas/Real" + } + } + }, + { + "type": "object", + "description": "Matches anywhere inside the property's text.", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "containsSegment" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "startsWith" + ] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "property", + "value", + "type" + ], + "properties": { + "property": { + "$ref": "#/components/schemas/BaseUrl" + }, + "type": { + "type": "string", + "enum": [ + "endsWith" + ] + }, + "value": { + "type": "string" + } + } + } + ], + "description": "A filter on one of the table's property columns, mirroring the operators the table's filter\nUI offers.\n\nThe `type` tag selects the operator, `property` names the column, and each operator carries\nexactly the value fields it needs, so a value-less operator with a value (or the reverse) is\nunrepresentable.", + "discriminator": { + "propertyName": "type" + } + }, + "EntityTablePropertyValue": { + "oneOf": [ + { + "$ref": "#/components/schemas/Real" + }, + { + "type": "string" + } + ], + "description": "A property value a table filter compares against." + }, + "EntityTableRow": { + "type": "object", + "description": "One row of the entities table, built entirely from materialized columns\nplus the raw property object.", + "required": [ + "entityId", + "entityEditionId", + "entityTypeIds", + "entityTypeTitles", + "createdAtTransactionTime", + "createdAtDecisionTime", + "editionCreatedAtDecisionTime", + "createdBy", + "lastEditedBy", + "archived", + "properties", + "propertiesMetadata" + ], + "properties": { + "archived": { + "type": "boolean" + }, + "createdAtDecisionTime": { + "$ref": "#/components/schemas/Timestamp" + }, + "createdAtTransactionTime": { + "$ref": "#/components/schemas/Timestamp" + }, + "createdBy": { + "$ref": "#/components/schemas/ActorEntityUuid" + }, + "editionCreatedAtDecisionTime": { + "$ref": "#/components/schemas/Timestamp" + }, + "entityEditionId": { + "$ref": "#/components/schemas/EntityEditionId" + }, + "entityId": { + "$ref": "#/components/schemas/EntityId" + }, + "entityTypeIds": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VersionedUrl" + }, + "description": "The entity's direct types, parallel to\n[`entity_type_titles`](Self::entity_type_titles)." + }, + "entityTypeTitles": { + "type": "array", + "items": { + "type": "string" + } + }, + "label": { + "type": "string" + }, + "lastEditedBy": { + "$ref": "#/components/schemas/ActorEntityUuid" + }, + "properties": { + "$ref": "#/components/schemas/PropertyObject" + }, + "propertiesMetadata": { + "$ref": "#/components/schemas/PropertyObjectMetadata" + }, + "sourceEntity": { + "allOf": [ + { + "$ref": "#/components/schemas/EntityTableLinkEndpoint" + } + ] + }, + "targetEntity": { + "allOf": [ + { + "$ref": "#/components/schemas/EntityTableLinkEndpoint" + } + ] } } }, - "EntityQueryCursor": { - "type": "array", - "items": { - "type": "object" - } - }, - "EntityQuerySortingPath": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/EntityQuerySortingToken" - }, - { - "type": "string" - }, - { - "type": "number" - } - ] - } + "EntityTableSortKey": { + "type": "string", + "description": "Sort key of the entities table, closed over the materialized, indexable\ncolumns. Extending the table's sortable columns means adding a variant\nhere, never a free-form path.", + "enum": [ + "createdAtDecisionTime", + "editionCreatedAtDecisionTime", + "label", + "typeTitle", + "archived" + ] }, - "EntityQuerySortingRecord": { + "EntityTableSorting": { "type": "object", "required": [ - "path", - "ordering", - "nulls" + "key", + "ordering" ], "properties": { - "nulls": { - "$ref": "#/components/schemas/NullOrdering" + "key": { + "$ref": "#/components/schemas/EntityTableSortKey" }, "ordering": { "$ref": "#/components/schemas/Ordering" - }, - "path": { - "$ref": "#/components/schemas/EntityQuerySortingPath" } - } - }, - "EntityQuerySortingToken": { - "type": "string", - "enum": [ - "uuid", - "archived", - "label", - "editionCreatedAtTransactionTime", - "editionCreatedAtDecisionTime", - "createdAtTransactionTime", - "createdAtDecisionTime", - "typeTitle" - ] - }, - "EntityQueryToken": { - "type": "string", - "description": "A single token in an [`EntityQueryPath`].", - "enum": [ - "uuid", - "editionId", - "draftId", - "archived", - "webId", - "type", - "properties", - "label", - "createdById", - "editionCreatedById", - "createdAtTransactionTime", - "createdAtDecisionTime", - "provenance", - "editionProvenance", - "embedding", - "incomingLinks", - "outgoingLinks", - "leftEntity", - "rightEntity" - ] + }, + "additionalProperties": false }, - "EntityRecordId": { + "EntityTableSummary": { "type": "object", + "description": "The summary of the entities table.\n\nThe two halves have different scopes on purpose: [`count`] reflects the\npage's full filters, where the type maps span the whole scope so a filter\nUI can widen a narrowed selection.\n\n[`count`]: Self::count", "required": [ - "entityId", - "editionId" + "count", + "entityTypeIds", + "entityTypeTitles" ], "properties": { - "editionId": { - "$ref": "#/components/schemas/EntityEditionId" + "count": { + "type": "integer", + "description": "How many entities match the page's full filters.", + "minimum": 0 }, - "entityId": { - "$ref": "#/components/schemas/EntityId" + "entityTypeIds": { + "type": "object", + "description": "How many entities in the scope carry each (direct) type.", + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + }, + "entityTypeTitles": { + "type": "object", + "description": "The display titles of the types in [`entity_type_ids`].\n\n[`entity_type_ids`]: Self::entity_type_ids", + "additionalProperties": { + "type": "string" + } + } + } + }, + "EntityTableTypeScope": { + "oneOf": [ + { + "type": "object", + "description": "Only entities carrying one of the listed types.", + "required": [ + "entityTypeIds", + "type" + ], + "properties": { + "entityTypeIds": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VersionedUrl" + } + }, + "type": { + "type": "string", + "enum": [ + "include" + ] + } + } + }, + { + "type": "object", + "description": "The scope's visible-type universe, derived server-side from the\nsummary, except the types under the listed base URLs.\n\nEntities carrying an excluded type are left out entirely — of the\nrows, the count, and the type summary alike.", + "required": [ + "entityTypeBaseUrls", + "type" + ], + "properties": { + "entityTypeBaseUrls": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseUrl" + } + }, + "type": { + "type": "string", + "enum": [ + "exclude" + ] + } + } + } + ], + "description": "Which entity types the table draws rows from.\n\n[`Include`] selects versioned type ids, matching the versioned selections\na filter UI works with. [`Exclude`] cuts by base URL on purpose: an\nexclusion is meant to hide a type regardless of which version an entity\ncarries. The default is [`Exclude`] with an empty list: the whole\nvisible-type universe.\n\n[`Include`]: Self::Include\n[`Exclude`]: Self::Exclude", + "discriminator": { + "propertyName": "type" + } + }, + "EntityTableWebScope": { + "oneOf": [ + { + "type": "object", + "description": "Only the listed webs, where an empty list matches no rows at all.", + "required": [ + "webs", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "include" + ] + }, + "webs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebId" + } + } + } + }, + { + "type": "object", + "description": "Every web the actor may see except the listed ones.", + "required": [ + "webs", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "exclude" + ] + }, + "webs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebId" + } + } + } } + ], + "description": "Which webs the table draws rows from.\n\nThe default is [`Exclude`] with an empty list: every web the actor may\nsee.\n\n[`Exclude`]: Self::Exclude", + "discriminator": { + "propertyName": "type" } }, "EntityTemporalMetadata": { @@ -8235,6 +8868,92 @@ } } }, + "QueryEntitiesTableParams": { + "type": "object", + "description": "Parameters for [`EntityStore::query_entities_table`].\n\n[`EntityStore::query_entities_table`]: crate::entity::EntityStore::query_entities_table", + "required": [ + "filter", + "limit" + ], + "properties": { + "conversions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/QueryConversion" + }, + "description": "Converts the rows' property values at each conversion's path into its\ntarget data type, mirroring [`QueryEntitiesParams::conversions`].\n\n[`QueryEntitiesParams::conversions`]: crate::entity::QueryEntitiesParams::conversions" + }, + "cursor": { + "allOf": [ + { + "$ref": "#/components/schemas/EntityTableCursor" + } + ] + }, + "filter": { + "$ref": "#/components/schemas/EntityTableFilter" + }, + "includeEntityTypes": { + "allOf": [ + { + "$ref": "#/components/schemas/IncludeEntityTypeOption" + } + ] + }, + "includeSummary": { + "type": "boolean" + }, + "limit": { + "type": "integer", + "minimum": 0 + }, + "sort": { + "$ref": "#/components/schemas/EntityTableSorting" + } + }, + "additionalProperties": false + }, + "QueryEntitiesTableResponse": { + "type": "object", + "required": [ + "rows" + ], + "properties": { + "closedMultiEntityTypes": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ClosedMultiEntityTypeMap" + } + }, + "cursor": { + "allOf": [ + { + "$ref": "#/components/schemas/EntityTableCursor" + } + ] + }, + "definitions": { + "allOf": [ + { + "$ref": "#/components/schemas/EntityTypeResolveDefinitions" + } + ] + }, + "rows": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityTableRow" + } + }, + "summary": { + "allOf": [ + { + "$ref": "#/components/schemas/EntityTableSummary" + } + ] + } + } + }, "QueryEntitySubgraphRequest": { "oneOf": [ { diff --git a/libs/@local/graph/api/src/rest/entity/mod.rs b/libs/@local/graph/api/src/rest/entity/mod.rs index 4db265a1297..e1f35d705ab 100644 --- a/libs/@local/graph/api/src/rest/entity/mod.rs +++ b/libs/@local/graph/api/src/rest/entity/mod.rs @@ -18,11 +18,15 @@ use hash_graph_store::{ ClosedMultiEntityTypeMap, ClusterEntitiesParams, ClusterEntitiesResponse, CreateEntityParams, DiffEntityParams, DiffEntityResult, EntityCluster, EntityPermissions, EntityQueryCursor, EntityQuerySortingRecord, EntityQuerySortingToken, EntityQueryToken, - EntityStore, EntityTypesError, EntityValidationReport, EntityValidationType, + EntityStore, EntityTableCursor, EntityTableFilter, EntityTableLinkEndpoint, + EntityTablePropertyFilter, EntityTablePropertyValue, EntityTableRow, EntityTableSortKey, + EntityTableSorting, EntityTableSummary, EntityTableTypeScope, EntityTableWebScope, + EntityTypesError, EntityValidationReport, EntityValidationType, HasPermissionForEntitiesParams, LinkDataStateError, LinkDataValidationReport, LinkError, LinkTargetError, LinkValidationReport, LinkedEntityError, MetadataValidationReport, PatchEntityParams, PropertyMetadataValidationReport, QueryConversion, - QueryEntitiesResponse, SearchEntitiesFilter, SearchEntitiesParams, SearchEntitiesResponse, + QueryEntitiesResponse, QueryEntitiesTableParams, QueryEntitiesTableResponse, + SearchEntitiesFilter, SearchEntitiesParams, SearchEntitiesResponse, SummarizeEntitiesParams, SummarizeEntitiesResponse, UnexpectedEntityType, UpdateEntityEmbeddingsParams, ValidateEntityComponents, ValidateEntityParams, }, @@ -76,7 +80,7 @@ use type_system::{ }; use self::query::{ - QueryEntitySubgraphResponse, query_entities, query_entity_subgraph, + QueryEntitySubgraphResponse, query_entities, query_entities_table, query_entity_subgraph, request::{QueryEntitiesRequest, QueryEntitySubgraphRequest}, summarize_entities, }; @@ -97,6 +101,7 @@ use crate::rest::{ self::query::query_entities, self::query::query_entity_subgraph, self::query::summarize_entities, + self::query::query_entities_table, search_entities, patch_entity, update_entity_embeddings, @@ -113,6 +118,19 @@ use crate::rest::{ ValidateEntityParams, SummarizeEntitiesParams, SummarizeEntitiesResponse, + QueryEntitiesTableParams, + QueryEntitiesTableResponse, + EntityTableCursor, + EntityTableFilter, + EntityTableLinkEndpoint, + EntityTablePropertyFilter, + EntityTablePropertyValue, + EntityTableWebScope, + EntityTableTypeScope, + EntityTableRow, + EntityTableSortKey, + EntityTableSorting, + EntityTableSummary, EntityValidationType, ValidateEntityComponents, Embedding, @@ -245,7 +263,8 @@ impl EntityResource { Router::new() .route("/", post(query_entities::)) .route("/subgraph", post(query_entity_subgraph::)) - .route("/summarize", post(summarize_entities::)), + .route("/summarize", post(summarize_entities::)) + .route("/table", post(query_entities_table::)), ), ) } diff --git a/libs/@local/graph/api/src/rest/entity/query/mod.rs b/libs/@local/graph/api/src/rest/entity/query/mod.rs index 9be5deeffcb..9cd4143a7c9 100644 --- a/libs/@local/graph/api/src/rest/entity/query/mod.rs +++ b/libs/@local/graph/api/src/rest/entity/query/mod.rs @@ -8,7 +8,8 @@ use error_stack::{Report, ResultExt as _}; use hash_graph_store::{ entity::{ ClosedMultiEntityTypeMap, EntityPermissions, EntityQueryCursor, EntityStore as _, - QueryEntitiesResponse, SummarizeEntitiesParams, SummarizeEntitiesResponse, + QueryEntitiesResponse, QueryEntitiesTableParams, QueryEntitiesTableResponse, + SummarizeEntitiesParams, SummarizeEntitiesResponse, }, entity_type::EntityTypeResolveDefinitions, pool::StorePool, @@ -25,6 +26,7 @@ pub use self::request::{ use crate::rest::{ ApiConfig, AuthenticatedUserHeader, OpenApiQuery, QueryLogger, json::Json, + resolve_limit, status::{BoxedResponse, report_to_response}, utoipa_typedef::subgraph::Subgraph, }; @@ -228,3 +230,51 @@ where } response } + +#[utoipa::path( + post, + path = "/entities/query/table", + request_body = QueryEntitiesTableParams, + tag = "Entity", + params( + ("X-Authenticated-User-Actor-Id" = ActorEntityUuid, Header, description = "The ID of the actor which is used to authorize the request"), + ), + responses( + ( + status = 200, + content_type = "application/json", + body = QueryEntitiesTableResponse, + ), + (status = 422, content_type = "text/plain", description = "Provided query is invalid"), + (status = 500, description = "Store error occurred"), + ) +)] +pub(super) async fn query_entities_table( + AuthenticatedUserHeader(actor_id): AuthenticatedUserHeader, + store_pool: Extension>, + Extension(api_config): Extension, + temporal_client: Extension>>, + Json(request): Json, +) -> Result, BoxedResponse> +where + S: StorePool + Send + Sync, +{ + let mut params = QueryEntitiesTableParams::deserialize(&request) + .map_err(Report::from) + .attach(hash_status::StatusCode::InvalidArgument) + .map_err(report_to_response)?; + params.limit = resolve_limit(Some(params.limit), api_config.query_entity_limit) + .attach(hash_status::StatusCode::InvalidArgument) + .map_err(report_to_response)?; + + let mut store = store_pool + .acquire(temporal_client.0) + .await + .map_err(report_to_response)?; + + store + .query_entities_table(actor_id, params) + .await + .map(Json) + .map_err(report_to_response) +} diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/mod.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/mod.rs index 12ea3e00dfc..5a36120b4a8 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/mod.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod provenance; mod query; mod read; mod summary; +mod table; use alloc::borrow::Cow; use core::{any::Any, borrow::Borrow as _, fmt, mem}; @@ -27,10 +28,10 @@ use hash_graph_store::{ EntityQueryPath, EntityQuerySorting, EntityStore, EntityTypeRetrieval, EntityTypesError, EntityValidationReport, EntityValidationType, HasPermissionForEntitiesParams, PatchEntityParams, QueryConversion, QueryEntitiesParams, QueryEntitiesResponse, - QueryEntitySubgraphParams, QueryEntitySubgraphResponse, SearchEntitiesFilter, - SearchEntitiesParams, SearchEntitiesResponse, SummarizeEntitiesParams, - SummarizeEntitiesResponse, UpdateEntityEmbeddingsParams, ValidateEntityComponents, - ValidateEntityParams, + QueryEntitiesTableParams, QueryEntitiesTableResponse, QueryEntitySubgraphParams, + QueryEntitySubgraphResponse, SearchEntitiesFilter, SearchEntitiesParams, + SearchEntitiesResponse, SummarizeEntitiesParams, SummarizeEntitiesResponse, + UpdateEntityEmbeddingsParams, ValidateEntityComponents, ValidateEntityParams, }, entity_type::{EntityTypeStore as _, IncludeEntityTypeOption}, error::{ @@ -105,7 +106,7 @@ use crate::store::{ knowledge::entity::{ provenance::{SqlEntityEditionProvenance, SqlEntityProvenance}, read::EntityEdgeTraversalData, - summary::{Deduplication, EntitySummaryQuery}, + summary::{Deduplication, EntitySummaryQuery, EntitySummaryRequest}, }, query::{ Distinctness, PostgresRecord as _, PostgresSorting as _, SelectCompiler, @@ -527,16 +528,19 @@ where metadata.data_type_id = Some(target_data_type_id.clone()); } + /// Applies the `conversions` to a property object and its metadata in + /// place. #[tracing::instrument(level = "info", skip_all)] - async fn convert_entity( + pub(crate) async fn convert_properties( &self, provider: &P, - entity: &mut Entity, + properties: &mut PropertyObject, + metadata: &mut PropertyObjectMetadata, conversions: &[QueryConversion<'_>], ) -> Result<(), Report> { let mut property = PropertyWithMetadata::Object(PropertyObjectWithMetadata::from_parts( - mem::take(&mut entity.properties), - Some(mem::take(&mut entity.metadata.properties)), + mem::take(properties), + Some(mem::take(metadata)), )?); for conversion in conversions { self.convert_entity_properties( @@ -550,12 +554,28 @@ where let PropertyWithMetadata::Object(property) = property else { unreachable!("The property was just converted to an object"); }; - let (properties, metadata) = property.into_parts(); - entity.properties = properties; - entity.metadata.properties = metadata; + let (converted_properties, converted_metadata) = property.into_parts(); + *properties = converted_properties; + *metadata = converted_metadata; Ok(()) } + #[tracing::instrument(level = "info", skip_all)] + async fn convert_entity( + &self, + provider: &P, + entity: &mut Entity, + conversions: &[QueryConversion<'_>], + ) -> Result<(), Report> { + self.convert_properties( + provider, + &mut entity.properties, + &mut entity.metadata.properties, + conversions, + ) + .await + } + #[tracing::instrument(level = "info", skip_all)] #[expect(clippy::too_many_lines)] async fn read_entities_impl( @@ -1949,6 +1969,28 @@ where } #[tracing::instrument(level = "info", skip_all)] + async fn query_entities_table( + &mut self, + actor_id: ActorEntityUuid, + params: QueryEntitiesTableParams, + ) -> Result> { + // The summary defines the type universe the page query runs on. Reading + // both in one `REPEATABLE READ, READ ONLY` transaction gives them a + // shared snapshot, so the universe is exact for the page it fences. + let transaction = self + .begin_read_only_transaction() + .await + .change_context(QueryError)?; + + let response = transaction + .query_entities_table_impl(actor_id, params) + .await?; + + transaction.commit().await.change_context(QueryError)?; + + Ok(response) + } + async fn summarize_entities( &self, actor_id: ActorEntityUuid, @@ -2003,7 +2045,9 @@ where .add_filter(filter_to_use) .change_context(QueryError)?; - let Some(summary_query) = EntitySummaryQuery::new(&mut compiler, ¶ms) else { + let Some(summary_query) = + EntitySummaryQuery::new(&mut compiler, EntitySummaryRequest::from(¶ms)) + else { return Ok(SummarizeEntitiesResponse::default()); }; let (statement, parameters) = compiler.compile(); diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs index 72240f2e087..2b791fbcd16 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs @@ -23,6 +23,30 @@ use uuid::Uuid; use crate::store::postgres::query::SelectCompiler; +/// Which summary dimensions to aggregate. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct EntitySummaryRequest { + pub count: bool, + pub web_ids: bool, + pub created_by_ids: bool, + pub edition_created_by_ids: bool, + pub type_ids: bool, + pub type_titles: bool, +} + +impl From<&SummarizeEntitiesParams<'_>> for EntitySummaryRequest { + fn from(params: &SummarizeEntitiesParams<'_>) -> Self { + Self { + count: params.include_count, + web_ids: params.include_web_ids, + created_by_ids: params.include_created_by_ids, + edition_created_by_ids: params.include_edition_created_by_ids, + type_ids: params.include_type_ids, + type_titles: params.include_type_titles, + } + } +} + /// Aggregated `include_*` summaries of an entity query. /// /// Each map is populated only when its flag was requested. `type_ids` and `type_titles` @@ -105,7 +129,7 @@ struct TypeColumns { } impl EntitySummaryQuery { - /// Adds the selections required for the summaries requested in `params` to the + /// Adds the selections required for the summaries requested in `request` to the /// `compiler`, or returns [`None`] when no summary is requested. /// /// Must run *before* limit, sorting, or record selections are added to the compiler: @@ -113,14 +137,14 @@ impl EntitySummaryQuery { /// and a limit would truncate the aggregates. pub(crate) fn new( compiler: &mut SelectCompiler<'_, '_, Entity>, - params: &SummarizeEntitiesParams<'_>, + request: EntitySummaryRequest, ) -> Option { - if !(params.include_count - || params.include_web_ids - || params.include_created_by_ids - || params.include_edition_created_by_ids - || params.include_type_ids - || params.include_type_titles) + if !(request.count + || request.web_ids + || request.created_by_ids + || request.edition_created_by_ids + || request.type_ids + || request.type_titles) { return None; } @@ -128,29 +152,27 @@ impl EntitySummaryQuery { Some(Self { edition_id_column: compiler.add_selection_path(&EntityQueryPath::EditionId), web_id_column: compiler.add_selection_path(&EntityQueryPath::WebId), - created_by_column: params - .include_created_by_ids + created_by_column: request + .created_by_ids .then(|| compiler.add_selection_path(&EntityQueryPath::CreatedById)), - edition_created_by_column: params - .include_edition_created_by_ids + edition_created_by_column: request + .edition_created_by_ids .then(|| compiler.add_selection_path(&EntityQueryPath::EditionCreatedById)), - type_columns: (params.include_type_ids || params.include_type_titles).then(|| { - TypeColumns { - versioned_urls: compiler.add_selection_path(&EntityQueryPath::EntityTypeEdge { - edge_kind: SharedEdgeKind::IsOfType, - path: EntityTypeQueryPath::VersionedUrl, - inheritance_depth: None, - }), - direct_types: compiler.add_selection_path(&EntityQueryPath::DirectTypeCount), - type_titles: compiler.add_selection_path(&EntityQueryPath::EntityTypeEdge { - edge_kind: SharedEdgeKind::IsOfType, - path: EntityTypeQueryPath::Title, - inheritance_depth: None, - }), - } + type_columns: (request.type_ids || request.type_titles).then(|| TypeColumns { + versioned_urls: compiler.add_selection_path(&EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::VersionedUrl, + inheritance_depth: None, + }), + direct_types: compiler.add_selection_path(&EntityQueryPath::DirectTypeCount), + type_titles: compiler.add_selection_path(&EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::Title, + inheritance_depth: None, + }), }), - include_count: params.include_count, - include_web_ids: params.include_web_ids, + include_count: request.count, + include_web_ids: request.web_ids, }) } @@ -184,42 +206,70 @@ impl EntitySummaryQuery { } let hit_columns = hit_columns.join(", "); + // With both count and types requested, the count rides along in the + // type branch. When the type branch is the only other consumer of + // `hits`, this keeps the CTE at a single reference — the inlined, + // parallel aggregate plan — where a dedicated count branch would force + // materialization. With further dimensions the fold is merely free. + let fold_count_into_types = self.count_folds_into_types(); + + let fold_column = if fold_count_into_types { + ", NULL::int8 AS first_type_matches" + } else { + "" + }; + let mut branches = Vec::new(); - if self.include_count { + if self.include_count && !fold_count_into_types { branches.push(format!( "SELECT {}::int4 AS dimension, NULL::uuid AS dimension_id, NULL::text AS \ - dimension_type, count(*) AS matches, NULL::text AS dimension_title FROM hits", + dimension_type, count(*) AS matches, NULL::text AS dimension_title{fold_column} \ + FROM hits", Dimension::Count as i32 )); } if self.include_web_ids { branches.push(format!( - "SELECT {}::int4, web_id, NULL::text, count(*), NULL::text FROM hits GROUP BY \ - web_id", + "SELECT {}::int4, web_id, NULL::text, count(*), NULL::text{fold_column} FROM hits \ + GROUP BY web_id", Dimension::WebIds as i32 )); } if self.created_by_column.is_some() { branches.push(format!( - "SELECT {}::int4, created_by, NULL::text, count(*), NULL::text FROM hits GROUP BY \ - created_by", + "SELECT {}::int4, created_by, NULL::text, count(*), NULL::text{fold_column} FROM \ + hits GROUP BY created_by", Dimension::CreatedByIds as i32 )); } if self.edition_created_by_column.is_some() { branches.push(format!( - "SELECT {}::int4, edition_created_by, NULL::text, count(*), NULL::text FROM hits \ - GROUP BY edition_created_by", + "SELECT {}::int4, edition_created_by, NULL::text, count(*), \ + NULL::text{fold_column} FROM hits GROUP BY edition_created_by", Dimension::EditionCreatedByIds as i32 )); } if self.type_columns.is_some() { - branches.push(format!( - "SELECT {}::int4, NULL::uuid, t.type_id, count(*), min(t.title) FROM (SELECT \ - unnest(versioned_urls[1:direct_types]) AS type_id, \ - unnest(type_titles[1:direct_types]) AS title FROM hits) AS t GROUP BY t.type_id", - Dimension::TypeIds as i32 - )); + if fold_count_into_types { + // Every hit has exactly one first type, so summing this column + // over all type groups recovers the exact hit count. + branches.push(format!( + "SELECT {}::int4, NULL::uuid, t.type_id, count(*), min(t.title), count(*) \ + FILTER (WHERE t.ord = 1) FROM (SELECT unnest(versioned_urls[1:direct_types]) \ + AS type_id, unnest(type_titles[1:direct_types]) AS title, \ + generate_subscripts(versioned_urls[1:direct_types], 1) AS ord FROM hits) AS \ + t GROUP BY t.type_id", + Dimension::TypeIds as i32 + )); + } else { + branches.push(format!( + "SELECT {}::int4, NULL::uuid, t.type_id, count(*), min(t.title) FROM (SELECT \ + unnest(versioned_urls[1:direct_types]) AS type_id, \ + unnest(type_titles[1:direct_types]) AS title FROM hits) AS t GROUP BY \ + t.type_id", + Dimension::TypeIds as i32 + )); + } } format!( @@ -284,6 +334,13 @@ impl EntitySummaryQuery { let type_id = row .try_get::<_, VersionedUrl>(2) .change_context(QueryError)?; + if self.count_folds_into_types() + && let Some(count) = &mut summaries.count + { + *count += + usize::try_from(row.try_get::<_, i64>(5).change_context(QueryError)?) + .change_context(QueryError)?; + } if let Some(type_ids) = &mut summaries.type_ids { type_ids.insert(type_id.clone(), matches); } @@ -305,6 +362,16 @@ impl EntitySummaryQuery { Ok(summaries) } + /// Whether the hit count is aggregated inside the type branch instead of + /// its own `hits` pass. + /// + /// With the type branch as the only other consumer, a dedicated count + /// branch would be a second reference to the `hits` CTE, which forces + /// PostgreSQL to materialize it and abandons the parallel aggregate plan. + const fn count_folds_into_types(&self) -> bool { + self.include_count && self.type_columns.is_some() + } + fn column_count(&self) -> usize { [ Some(self.edition_id_column), @@ -361,6 +428,7 @@ mod tests { include_web_ids: true, }; + // Count and types together fold the count into the type branch. pretty_assertions::assert_eq!( trim_whitespace(&summary_query.statement("SELECT 1", Deduplication::Required)), trim_whitespace( @@ -372,26 +440,52 @@ mod tests { c5 AS direct_types, c6 AS type_titles FROM ( SELECT 1 ) AS raw (c0, c1, c2, c3, c4, c5, c6)) - SELECT 0::int4 AS dimension, NULL::uuid AS dimension_id, - NULL::text AS dimension_type, count(*) AS matches, - NULL::text AS dimension_title FROM hits - UNION ALL - SELECT 1::int4, web_id, NULL::text, count(*), NULL::text FROM hits GROUP BY web_id + SELECT 1::int4, web_id, NULL::text, count(*), NULL::text, + NULL::int8 AS first_type_matches FROM hits GROUP BY web_id UNION ALL - SELECT 2::int4, created_by, NULL::text, count(*), NULL::text FROM hits + SELECT 2::int4, created_by, NULL::text, count(*), NULL::text, + NULL::int8 AS first_type_matches FROM hits GROUP BY created_by UNION ALL - SELECT 3::int4, edition_created_by, NULL::text, count(*), NULL::text FROM hits + SELECT 3::int4, edition_created_by, NULL::text, count(*), NULL::text, + NULL::int8 AS first_type_matches FROM hits GROUP BY edition_created_by UNION ALL - SELECT 4::int4, NULL::uuid, t.type_id, count(*), min(t.title) FROM (SELECT + SELECT 4::int4, NULL::uuid, t.type_id, count(*), min(t.title), + count(*) FILTER (WHERE t.ord = 1) FROM (SELECT unnest(versioned_urls[1:direct_types]) AS type_id, - unnest(type_titles[1:direct_types]) AS title FROM hits) AS t + unnest(type_titles[1:direct_types]) AS title, + generate_subscripts(versioned_urls[1:direct_types], 1) AS ord FROM hits) AS t GROUP BY t.type_id" ), ); } + #[test] + fn statement_count_without_types_keeps_the_count_branch() { + let summary_query = EntitySummaryQuery { + edition_id_column: 0, + web_id_column: 1, + created_by_column: None, + edition_created_by_column: None, + type_columns: None, + include_count: true, + include_web_ids: false, + }; + + pretty_assertions::assert_eq!( + trim_whitespace(&summary_query.statement("SELECT 1", Deduplication::Required)), + trim_whitespace( + "WITH hits AS (SELECT DISTINCT ON (c0) + c1 AS web_id + FROM ( SELECT 1 ) AS raw (c0, c1)) + SELECT 0::int4 AS dimension, NULL::uuid AS dimension_id, + NULL::text AS dimension_type, count(*) AS matches, + NULL::text AS dimension_title FROM hits" + ), + ); + } + #[test] fn statement_count_only() { let summary_query = EntitySummaryQuery { diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs new file mode 100644 index 00000000000..7609a91bc4b --- /dev/null +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs @@ -0,0 +1,1362 @@ +//! The dedicated entities-table read: summary, count, and page in one +//! transaction. +//! +//! The table endpoint answers the entities table views with flat rows built from +//! materialized columns. When no explicit type filter is given, the visible-type +//! universe is derived from the summary and sent into the page query as an +//! include list — an indexable, estimable type clause the planner can work with, +//! where the bare scope filter alone provokes pathological plans. The count +//! rides along in the summary scan when the page carries no narrowing filters, +//! and runs as its own statement otherwise. Follow-up pages continue on the +//! first page's database state through the cursor. + +use alloc::borrow::Cow; +use std::collections::HashSet; + +use error_stack::{Report, ResultExt as _}; +use futures::TryStreamExt as _; +use hash_graph_authorization::policies::{MergePolicies, PolicyComponents, action::ActionName}; +use hash_graph_store::{ + entity::{ + EntityQueryCursor, EntityQueryPath, EntityQuerySorting, EntityQuerySortingRecord, + EntityTableCursor, EntityTableLinkEndpoint, EntityTablePropertyFilter, + EntityTablePropertyValue, EntityTableRow, EntityTableSortKey, EntityTableSorting, + EntityTableSummary, EntityTableTypeScope, EntityTableWebScope, QueryEntitiesTableParams, + QueryEntitiesTableResponse, + }, + entity_type::{EntityTypeQueryPath, EntityTypeStore as _, IncludeEntityTypeOption}, + error::QueryError, + filter::{ + Filter, FilterExpression, FilterExpressionList, JsonPath, Parameter, ParameterList, + PathToken, protection::transform_filter, + }, + query::CursorField, + subgraph::{ + edges::{EdgeDirection, KnowledgeGraphEdgeKind, SharedEdgeKind}, + temporal_axes::{ + PinnedTemporalAxis, QueryTemporalAxes, QueryTemporalAxesUnresolved, + VariableTemporalAxis, + }, + }, +}; +use hash_graph_temporal_versioning::{ + ClosedTemporalBound, DecisionTime, LeftClosedTemporalInterval, LimitedTemporalBound, + TemporalBound, TemporalTagged as _, Timestamp, TransactionTime, +}; +use serde::Deserialize as _; +use tokio_postgres::{GenericClient as _, Row}; +use tracing::Instrument as _; +use type_system::{ + knowledge::{ + Entity, + entity::id::{EntityEditionId, EntityId}, + property::metadata::PropertyObjectMetadata, + }, + ontology::{VersionedUrl, entity_type::EntityTypeUuid}, + principal::{actor::ActorEntityUuid, actor_group::WebId}, +}; + +use crate::store::{ + AsClient, PostgresStore, + postgres::{ + InTransaction, + knowledge::entity::summary::{ + Deduplication, EntitySummaries, EntitySummaryQuery, EntitySummaryRequest, + }, + query::{PostgresSorting as _, SelectCompiler, StatementShape}, + }, + validation::StoreProvider, +}; + +/// Column indices of the row selections in the compiled page statement. +struct RowIndices { + web_id: usize, + entity_uuid: usize, + draft_id: usize, + edition_id: usize, + label: usize, + type_versioned_urls: usize, + type_titles: usize, + direct_type_count: usize, + created_at_transaction_time: usize, + created_at_decision_time: usize, + decision_time: usize, + created_by: usize, + edition_created_by: usize, + archived: usize, + properties: usize, + property_metadata: usize, + source_entity: LinkEndpointIndices, + target_entity: LinkEndpointIndices, +} + +/// Column indices of one endpoint's selections in the compiled page statement. +/// +/// Every column reads `NULL` on non-link rows, which [`Self::decode`] folds +/// into [`None`]. +struct LinkEndpointIndices { + web_id: usize, + entity_uuid: usize, + edition_id: usize, + label: usize, + type_versioned_urls: usize, + direct_type_count: usize, +} + +/// The [`EntityQueryPath`]s selecting one endpoint of a link row. +/// +/// [`SelectCompiler::add_selection_path`] borrows its paths for the compiled +/// statement's lifetime, and [`EntityQueryPath::EntityEdge`] allocates — the +/// paths need an owner that outlives the compiler. +struct LinkEndpointPaths { + web_id: EntityQueryPath<'static>, + entity_uuid: EntityQueryPath<'static>, + edition_id: EntityQueryPath<'static>, + label: EntityQueryPath<'static>, + type_versioned_urls: EntityQueryPath<'static>, + direct_type_count: EntityQueryPath<'static>, +} + +impl LinkEndpointPaths { + fn new(edge_kind: KnowledgeGraphEdgeKind) -> Self { + let endpoint_path = |path: EntityQueryPath<'static>| EntityQueryPath::EntityEdge { + edge_kind, + path: Box::new(path), + direction: EdgeDirection::Outgoing, + }; + + Self { + web_id: endpoint_path(EntityQueryPath::WebId), + entity_uuid: endpoint_path(EntityQueryPath::Uuid), + edition_id: endpoint_path(EntityQueryPath::EditionId), + label: endpoint_path(EntityQueryPath::FirstLabel), + type_versioned_urls: endpoint_path(EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::VersionedUrl, + inheritance_depth: None, + }), + direct_type_count: endpoint_path(EntityQueryPath::DirectTypeCount), + } + } +} + +/// The allocating [`EntityQueryPath`]s among a table row's selections. +struct RowPaths { + source_entity: LinkEndpointPaths, + target_entity: LinkEndpointPaths, +} + +impl RowPaths { + fn new() -> Self { + Self { + source_entity: LinkEndpointPaths::new(KnowledgeGraphEdgeKind::HasLeftEntity), + target_entity: LinkEndpointPaths::new(KnowledgeGraphEdgeKind::HasRightEntity), + } + } +} + +impl LinkEndpointIndices { + fn install<'p>( + compiler: &mut SelectCompiler<'p, '_, Entity>, + paths: &'p LinkEndpointPaths, + ) -> Self { + Self { + web_id: compiler.add_selection_path(&paths.web_id), + entity_uuid: compiler.add_selection_path(&paths.entity_uuid), + edition_id: compiler.add_selection_path(&paths.edition_id), + label: compiler.add_selection_path(&paths.label), + type_versioned_urls: compiler.add_selection_path(&paths.type_versioned_urls), + direct_type_count: compiler.add_selection_path(&paths.direct_type_count), + } + } + + fn decode(&self, row: &Row) -> Option<(EntityTableLinkEndpoint, EntityEditionId)> { + let web_id = row.get::<_, Option<_>>(self.web_id)?; + let entity_uuid = row.get::<_, Option<_>>(self.entity_uuid)?; + let edition_id = row.get::<_, Option>(self.edition_id)?; + // The edition cache is maintained in the same transaction as the + // editions, so a matched edition always has its cache row and the + // cache columns read non-null, like the row's own decode. + let direct_type_count = usize::try_from(row.get::<_, i32>(self.direct_type_count)) + .expect("the direct type count should be non-negative"); + + Some(( + EntityTableLinkEndpoint { + entity_id: EntityId { + web_id, + entity_uuid, + draft_id: None, + }, + label: row.get(self.label), + entity_type_ids: row + .get::<_, Vec>(self.type_versioned_urls) + .into_iter() + .take(direct_type_count) + .collect(), + }, + edition_id, + )) + } +} + +impl RowIndices { + fn install<'p>(compiler: &mut SelectCompiler<'p, '_, Entity>, paths: &'p RowPaths) -> Self { + Self { + web_id: compiler.add_selection_path(&EntityQueryPath::WebId), + entity_uuid: compiler.add_selection_path(&EntityQueryPath::Uuid), + draft_id: compiler.add_selection_path(&EntityQueryPath::DraftId), + edition_id: compiler.add_selection_path(&EntityQueryPath::EditionId), + label: compiler.add_selection_path(&EntityQueryPath::FirstLabel), + type_versioned_urls: compiler.add_selection_path(&EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::VersionedUrl, + inheritance_depth: None, + }), + type_titles: compiler.add_selection_path(&EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::Title, + inheritance_depth: None, + }), + direct_type_count: compiler.add_selection_path(&EntityQueryPath::DirectTypeCount), + created_at_transaction_time: compiler + .add_selection_path(&EntityQueryPath::CreatedAtTransactionTime), + created_at_decision_time: compiler + .add_selection_path(&EntityQueryPath::CreatedAtDecisionTime), + decision_time: compiler.add_selection_path(&EntityQueryPath::DecisionTime), + created_by: compiler.add_selection_path(&EntityQueryPath::CreatedById), + edition_created_by: compiler.add_selection_path(&EntityQueryPath::EditionCreatedById), + archived: compiler.add_selection_path(&EntityQueryPath::Archived), + properties: compiler.add_selection_path(&EntityQueryPath::Properties(None)), + property_metadata: compiler + .add_selection_path(&EntityQueryPath::PropertyMetadata(None)), + source_entity: LinkEndpointIndices::install(compiler, &paths.source_entity), + target_entity: LinkEndpointIndices::install(compiler, &paths.target_entity), + } + } + + fn decode(&self, row: &Row) -> (EntityTableRow, LinkEndpointEditions) { + let direct_type_count = usize::try_from(row.get::<_, i32>(self.direct_type_count)) + .expect("the direct type count should be non-negative"); + + let decision_time = + row.get::<_, LeftClosedTemporalInterval>(self.decision_time); + let (ClosedTemporalBound::Inclusive(edition_created_at_decision_time), _) = + decision_time.into_bounds(); + + let source_entity = self.source_entity.decode(row); + let target_entity = self.target_entity.decode(row); + let editions = LinkEndpointEditions { + source: source_entity.as_ref().map(|(_, edition_id)| *edition_id), + target: target_entity.as_ref().map(|(_, edition_id)| *edition_id), + }; + + let row = EntityTableRow { + entity_id: EntityId { + web_id: row.get(self.web_id), + entity_uuid: row.get(self.entity_uuid), + draft_id: row.get(self.draft_id), + }, + entity_edition_id: row.get(self.edition_id), + label: row.get(self.label), + entity_type_ids: row + .get::<_, Vec>(self.type_versioned_urls) + .into_iter() + .take(direct_type_count) + .collect(), + entity_type_titles: row + .get::<_, Vec>(self.type_titles) + .into_iter() + .take(direct_type_count) + .collect(), + created_at_transaction_time: row.get(self.created_at_transaction_time), + created_at_decision_time: row.get(self.created_at_decision_time), + edition_created_at_decision_time, + created_by: row.get(self.created_by), + last_edited_by: row.get(self.edition_created_by), + archived: row.get(self.archived), + properties: row.get(self.properties), + properties_metadata: row + .get::<_, Option>(self.property_metadata) + .map(|value| { + PropertyObjectMetadata::deserialize(value) + .expect("the stored property metadata should be valid") + }) + .unwrap_or_default(), + source_entity: source_entity.map(|(endpoint, _)| endpoint), + target_entity: target_entity.map(|(endpoint, _)| endpoint), + }; + + (row, editions) + } +} + +/// The edition ids of a row's link endpoints. An endpoint is only shown once +/// its edition passes the knowledge-edge permission check. +struct LinkEndpointEditions { + source: Option, + target: Option, +} + +fn sorting_records( + sort: EntityTableSorting, + include_drafts: bool, +) -> Vec> { + let key_path = match sort.key { + EntityTableSortKey::CreatedAtDecisionTime => EntityQueryPath::CreatedAtDecisionTime, + EntityTableSortKey::EditionCreatedAtDecisionTime => EntityQueryPath::DecisionTime, + EntityTableSortKey::Label => EntityQueryPath::FirstLabel, + EntityTableSortKey::TypeTitle => EntityQueryPath::FirstTypeTitle, + EntityTableSortKey::Archived => EntityQueryPath::Archived, + }; + + let mut records = vec![ + EntityQuerySortingRecord { + path: key_path, + ordering: sort.ordering, + nulls: None, + }, + EntityQuerySortingRecord { + path: EntityQueryPath::Uuid, + ordering: sort.ordering, + nulls: None, + }, + ]; + if include_drafts { + records.push(EntityQuerySortingRecord { + path: EntityQueryPath::DraftId, + ordering: sort.ordering, + nulls: None, + }); + } + records +} + +impl PostgresStore +where + C: AsClient, +{ + #[tracing::instrument(level = "info", skip(self, params))] + pub(crate) async fn query_entities_table_impl( + &self, + actor_id: ActorEntityUuid, + params: QueryEntitiesTableParams, + ) -> Result> { + let policy_components = PolicyComponents::builder(self) + .with_actor(actor_id) + .with_action(ActionName::ViewEntity, MergePolicies::Yes) + .await + .change_context(QueryError)?; + + // The sort and draft visibility shape the keyset, so a continuation + // reads them from its token instead of trusting the re-sent request. + let (transaction_time, decision_time, mut type_universe, sort, include_drafts, position) = + match ¶ms.cursor { + None => { + let now: Timestamp<()> = Timestamp::now(); + ( + now.cast(), + now.cast(), + None, + params.sort, + params.filter.include_drafts, + None, + ) + } + Some(cursor) => ( + cursor.transaction_time, + cursor.decision_time, + cursor.type_universe.clone(), + cursor.sort, + cursor.include_drafts, + Some(cursor.position.clone()), + ), + }; + let temporal_axes = instant_axes(transaction_time, decision_time); + + // TODO(BE-707): assemble the view-entity policy filter through a shared + // entry point instead of repeating the extraction on every read path. + let policy_filter = Filter::::for_policies( + policy_components.extract_filter_policies(ActionName::ViewEntity), + policy_components.actor_id(), + policy_components.optimization_data(ActionName::ViewEntity), + ); + + let type_include_ids = match ¶ms.filter.types { + EntityTableTypeScope::Include { entity_type_ids } => Some(entity_type_ids.clone()), + EntityTableTypeScope::Exclude { .. } => None, + }; + + // Sensitive properties get the same two-sided protection the generic + // read path applies: the responses mask them, and the user's filters + // are rewritten so they cannot be probed through row presence or the + // count. + let should_apply_protection = + !self.settings.filter_protection.is_empty() && !policy_components.is_instance_admin(); + + let scope_filter = scope_filter(¶ms); + let mut property_filter = property_filters_filter(¶ms.filter.property_filters); + if let Some(filter) = &mut property_filter { + // Aligns the compared values with the property columns' JSONB + // representation, like the generic read path does for its filters. + filter + .convert_parameters(&StoreProvider::new(self, &policy_components)) + .await + .change_context(QueryError)?; + } + if should_apply_protection { + property_filter = property_filter.map(|filter| { + transform_filter( + filter, + &self.settings.filter_protection, + 0, + policy_components.actor_id(), + ) + }); + } + + let needs_universe = type_include_ids.is_none() && type_universe.is_none(); + // Without narrowing filters the page's full filters equal the scope + // (the include clause is result-neutral and the universe's base-URL + // exclusions live inside the scope filter), so the count can ride + // along in the summary scan instead of paying a second full scan. + let count_matches_scope = + type_include_ids.is_none() && params.filter.property_filters.is_empty(); + let scope_summaries = if params.include_summary || needs_universe { + Some( + self.scope_summary( + &policy_filter, + &scope_filter, + &temporal_axes, + include_drafts, + params.include_summary && count_matches_scope, + ) + .await?, + ) + } else { + None + }; + if needs_universe { + type_universe = scope_summaries.as_ref().map(|summaries| { + let mut universe = summaries + .type_ids + .as_ref() + .expect("the type summary should always be requested") + .keys() + .cloned() + .collect::>(); + // The universe is sent as query parameters and travels inside the + // cursor — keep its order deterministic. + universe.sort_unstable(); + universe + }); + } + + let type_filter = type_include_ids + .as_ref() + .or(type_universe.as_ref()) + .map(|entity_type_ids| type_include_filter(entity_type_ids)); + + // The count reflects the page query's full filters, where the type + // summary spans the scope so the filter UI can widen a selection. + let summary = + if let Some(scope_summaries) = scope_summaries.filter(|_| params.include_summary) { + Some(EntityTableSummary { + count: match scope_summaries.count { + Some(count) => count, + None => { + self.table_count( + [ + Some(&policy_filter), + Some(&scope_filter), + type_filter.as_ref(), + property_filter.as_ref(), + ], + &temporal_axes, + include_drafts, + ) + .await? + } + }, + entity_type_ids: scope_summaries + .type_ids + .expect("the type summary should always be requested"), + entity_type_titles: scope_summaries + .type_titles + .expect("the type summary should always be requested"), + }) + } else { + None + }; + + let row_paths = RowPaths::new(); + let mut compiler = SelectCompiler::new(Some(&temporal_axes), include_drafts); + + // The rows carry raw property objects, so the page masks them the + // same way the generic read path masks its entities. + let property_protection_filter; + if should_apply_protection { + property_protection_filter = self + .settings + .filter_protection + .to_property_protection_filter(policy_components.actor_id()); + compiler.with_property_masking(&property_protection_filter); + } + + compiler + .add_filter(&policy_filter) + .change_context(QueryError)?; + compiler + .add_filter(&scope_filter) + .change_context(QueryError)?; + if let Some(property_filter) = &property_filter { + compiler + .add_filter(property_filter) + .change_context(QueryError)?; + } + if let Some(type_filter) = &type_filter { + compiler + .add_filter(type_filter) + .change_context(QueryError)?; + } + + compiler.set_limit(params.limit); + // The table vouches for the keys-first preconditions: its hydration + // joins match at most one row per key and the distinct key pins all + // row-multiplying columns. Embeddings cannot occur here. + compiler.set_statement_shape(StatementShape::KeysFirst); + + let sorting = EntityQuerySorting { + paths: sorting_records(sort, include_drafts), + cursor: position, + }; + let cursor_parameters = sorting.encode().change_context(QueryError)?; + let cursor_indices = sorting + .compile(&mut compiler, cursor_parameters.as_ref(), &temporal_axes) + .change_context(QueryError)?; + + let row_indices = RowIndices::install(&mut compiler, &row_paths); + + let (statement, parameters) = compiler.compile(); + + let rows = self + .as_client() + .query(&statement, parameters) + .instrument(tracing::info_span!( + "SELECT", + otel.kind = "client", + db.system = "postgresql", + peer.service = "Postgres", + db.query.text = statement, + )) + .await + .change_context(QueryError)?; + + let cursor = (rows.len() == params.limit) + .then(|| rows.last()) + .flatten() + .map(|row| EntityTableCursor { + transaction_time, + decision_time, + type_universe: type_include_ids + .is_none() + .then(|| type_universe.clone()) + .flatten(), + sort, + include_drafts, + position: EntityQueryCursor { + values: cursor_indices + .iter() + .map(|&index| { + row.get::<_, Option>(index) + .unwrap_or(CursorField::Json( + type_system::knowledge::PropertyValue::Null, + )) + .into_owned() + }) + .collect(), + }, + }); + + let (mut rows, row_editions): (Vec<_>, Vec<_>) = + rows.iter().map(|row| row_indices.decode(row)).unzip(); + + // Link endpoints are hydrated through joins the policy filter does not + // reach, so they pass the same permission check the subgraph traversal + // applies to its edges before they are shown. + let endpoint_editions = row_editions + .iter() + .flat_map(|editions| [editions.source, editions.target]) + .flatten() + .collect::>(); + if !endpoint_editions.is_empty() + && let Some(permitted) = self + .filter_knowledge_edges( + &endpoint_editions, + &temporal_axes, + &StoreProvider::new(self, &policy_components), + ) + .await? + { + for (row, editions) in rows.iter_mut().zip(&row_editions) { + if editions + .source + .is_some_and(|edition_id| !permitted.contains(&edition_id)) + { + row.source_entity = None; + } + if editions + .target + .is_some_and(|edition_id| !permitted.contains(&edition_id)) + { + row.target_entity = None; + } + } + } + + if !params.conversions.is_empty() { + let provider = StoreProvider::new(self, &policy_components); + for row in &mut rows { + self.convert_properties( + &provider, + &mut row.properties, + &mut row.properties_metadata, + ¶ms.conversions, + ) + .await + .change_context(QueryError)?; + } + } + + let closed_multi_entity_types = if params.include_entity_types.is_some() { + // Link endpoints resolve as their own type combinations, so their + // chips can be rendered from the same map as the rows. + let endpoint_types = |endpoint: &Option| { + endpoint + .as_ref() + .map(|endpoint| endpoint.entity_type_ids.clone()) + }; + Some( + self.get_closed_multi_entity_types( + actor_id, + rows.iter() + .map(|row| row.entity_type_ids.clone()) + .chain( + rows.iter() + .filter_map(|row| endpoint_types(&row.source_entity)), + ) + .chain( + rows.iter() + .filter_map(|row| endpoint_types(&row.target_entity)), + ), + QueryTemporalAxesUnresolved::live_only(), + None, + ) + .await? + .entity_types, + ) + } else { + None + }; + let definitions = match params.include_entity_types { + Some( + IncludeEntityTypeOption::Resolved + | IncludeEntityTypeOption::ResolvedWithDataTypeChildren, + ) => { + let entity_type_uuids = rows + .iter() + .flat_map(|row| { + row.entity_type_ids + .iter() + .chain( + row.source_entity + .iter() + .flat_map(|endpoint| endpoint.entity_type_ids.iter()), + ) + .chain( + row.target_entity + .iter() + .flat_map(|endpoint| endpoint.entity_type_ids.iter()), + ) + .map(EntityTypeUuid::from_url) + }) + .collect::>() + .into_iter() + .collect::>(); + Some( + self.get_entity_type_resolve_definitions( + actor_id, + &entity_type_uuids, + params.include_entity_types + == Some(IncludeEntityTypeOption::ResolvedWithDataTypeChildren), + ) + .await?, + ) + } + None | Some(IncludeEntityTypeOption::Closed) => None, + }; + + Ok(QueryEntitiesTableResponse { + rows, + closed_multi_entity_types, + definitions, + cursor, + summary, + }) + } + + /// Runs the type summary over the scope, before any type filter, carrying + /// the scope count along when requested. + async fn scope_summary( + &self, + policy_filter: &Filter<'_, Entity>, + scope_filter: &Filter<'_, Entity>, + temporal_axes: &QueryTemporalAxes, + include_drafts: bool, + include_count: bool, + ) -> Result> { + let mut compiler = SelectCompiler::new(Some(temporal_axes), include_drafts); + compiler + .add_filter(policy_filter) + .change_context(QueryError)?; + compiler + .add_filter(scope_filter) + .change_context(QueryError)?; + + let summary_query = EntitySummaryQuery::new( + &mut compiler, + EntitySummaryRequest { + count: include_count, + type_ids: true, + type_titles: true, + ..EntitySummaryRequest::default() + }, + ) + .expect("the type summaries should always be requested"); + + let (statement, parameters) = compiler.compile(); + // The table's temporal axes are a point in time, so an edition can only + // match once — but a to-many filter join could still fan out. + let dedup = if compiler.has_to_many_join() { + Deduplication::Required + } else { + Deduplication::Skip + }; + let statement = summary_query.statement(&statement, dedup); + + // The planner estimates the summary's set-returning functions a + // thousandfold too high, which pushes the statement over the JIT + // thresholds and burns hundreds of milliseconds compiling it on + // every execution. The transaction's other statements are limited + // keyset reads that never reach those thresholds, so the local + // setting staying on for them costs nothing. + self.as_client() + .execute("SET LOCAL jit = off", &[]) + .await + .change_context(QueryError)?; + + let rows = self + .as_client() + .query_raw(&statement, parameters.iter().copied()) + .instrument(tracing::info_span!( + "SELECT", + otel.kind = "client", + db.system = "postgresql", + peer.service = "Postgres", + db.query.text = statement, + )) + .await + .change_context(QueryError)? + .try_collect::>() + .await + .change_context(QueryError)?; + + summary_query.decode(rows) + } + + /// Counts the rows matching the page query's full filters. + async fn table_count<'f>( + &self, + filters: [Option<&'f Filter<'f, Entity>>; 4], + temporal_axes: &QueryTemporalAxes, + include_drafts: bool, + ) -> Result> { + let mut compiler = SelectCompiler::new(Some(temporal_axes), include_drafts); + for filter in filters.into_iter().flatten() { + compiler.add_filter(filter).change_context(QueryError)?; + } + + let summary_query = EntitySummaryQuery::new( + &mut compiler, + EntitySummaryRequest { + count: true, + ..EntitySummaryRequest::default() + }, + ) + .expect("the count summary should always be requested"); + + let (statement, parameters) = compiler.compile(); + let dedup = if compiler.has_to_many_join() { + Deduplication::Required + } else { + Deduplication::Skip + }; + let statement = summary_query.statement(&statement, dedup); + + let rows = self + .as_client() + .query_raw(&statement, parameters.iter().copied()) + .instrument(tracing::info_span!( + "SELECT", + otel.kind = "client", + db.system = "postgresql", + peer.service = "Postgres", + db.query.text = statement, + )) + .await + .change_context(QueryError)? + .try_collect::>() + .await + .change_context(QueryError)?; + + summary_query.decode(rows)?.count.ok_or_else(|| { + Report::new(QueryError).attach("the count summary returned no count row") + }) + } +} + +/// The scope shared by the summary and the page query: webs, archival, and +/// the universe's type exclusions. +fn scope_filter<'f>(params: &'f QueryEntitiesTableParams) -> Filter<'f, Entity> { + let webs_in = |webs: &'f [WebId]| { + Filter::In( + FilterExpression::Path { + path: EntityQueryPath::WebId, + }, + FilterExpressionList::ParameterList { + parameters: ParameterList::WebIds(webs), + }, + ) + }; + + let mut clauses = Vec::new(); + + match ¶ms.filter.webs { + EntityTableWebScope::Include { webs } => { + clauses.push(webs_in(webs)); + } + EntityTableWebScope::Exclude { webs } => { + if !webs.is_empty() { + clauses.push(Filter::Not(Box::new(webs_in(webs)))); + } + } + } + + if let EntityTableTypeScope::Exclude { + entity_type_base_urls, + } = ¶ms.filter.types + { + // Excluding here rather than in the page's type clause drops the + // excluded entities from the summary and the count as well, and it + // catches multi-type entities that also carry a universe type. + clauses.extend(entity_type_base_urls.iter().map(|base_url| { + Filter::NotEqual( + FilterExpression::Path { + path: EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::BaseUrl, + inheritance_depth: None, + }, + }, + FilterExpression::Parameter { + parameter: Parameter::Text(base_url.to_string().into()), + convert: None, + }, + ) + })); + } + + if !params.filter.include_archived { + clauses.push(Filter::NotEqual( + FilterExpression::Path { + path: EntityQueryPath::Archived, + }, + FilterExpression::Parameter { + parameter: Parameter::Boolean(true), + convert: None, + }, + )); + } + + Filter::All(clauses) +} + +/// Assembles the table's snapshot axes from its two instants. +/// +/// The table's one-row-per-entity shape — the summary counting entities +/// rather than editions, the keyset paging without duplicates, and the +/// hydration joins matching at most one row per key — assumes an edition can +/// match the axes only once. Instants guarantee that by construction, where +/// an interval axis would break it. +fn instant_axes( + transaction_time: Timestamp, + decision_time: Timestamp, +) -> QueryTemporalAxes { + QueryTemporalAxes::DecisionTime { + pinned: PinnedTemporalAxis::new(transaction_time), + variable: VariableTemporalAxis::new( + TemporalBound::Inclusive(decision_time), + LimitedTemporalBound::Inclusive(decision_time), + ), + } +} + +/// Translates the table's property conditions into one conjunctive filter, or +/// [`None`] when there are none. +fn property_filters_filter<'f>( + property_filters: &[EntityTablePropertyFilter], +) -> Option> { + if property_filters.is_empty() { + return None; + } + + Some(Filter::All( + property_filters + .iter() + .map(property_condition_filter) + .collect(), + )) +} + +fn property_condition_filter<'f>( + property_filter: &EntityTablePropertyFilter, +) -> Filter<'f, Entity> { + let path = || { + EntityQueryPath::Properties(Some(JsonPath::from_path_tokens(vec![PathToken::Field( + Cow::Owned(property_filter.property().to_string()), + )]))) + }; + let path_expression = || FilterExpression::Path { path: path() }; + let boolean = |value: bool| FilterExpression::Parameter { + parameter: Parameter::Boolean(value), + convert: None, + }; + let text = |value: &str| FilterExpression::Parameter { + parameter: Parameter::Text(Cow::Owned(value.to_owned())), + convert: None, + }; + let number = |value: &hash_codec::numeric::Real| FilterExpression::Parameter { + parameter: Parameter::Decimal(value.clone()), + convert: None, + }; + let value = |value: &EntityTablePropertyValue| match value { + EntityTablePropertyValue::Number(real) => number(real), + EntityTablePropertyValue::String(string) => text(string), + }; + + match property_filter { + EntityTablePropertyFilter::HasAnyValue { .. } => Filter::Exists { path: path() }, + EntityTablePropertyFilter::IsEmpty { .. } => { + Filter::Not(Box::new(Filter::Exists { path: path() })) + } + EntityTablePropertyFilter::IsTrue { .. } => Filter::Equal(path_expression(), boolean(true)), + EntityTablePropertyFilter::IsFalse { .. } => { + Filter::Equal(path_expression(), boolean(false)) + } + EntityTablePropertyFilter::Equals { + value: compared, .. + } => Filter::Equal(path_expression(), value(compared)), + EntityTablePropertyFilter::NotEquals { + value: compared, .. + } => Filter::NotEqual(path_expression(), value(compared)), + EntityTablePropertyFilter::GreaterThan { + value: compared, .. + } => Filter::Greater(path_expression(), number(compared)), + EntityTablePropertyFilter::GreaterThanOrEqual { + value: compared, .. + } => Filter::GreaterOrEqual(path_expression(), number(compared)), + EntityTablePropertyFilter::LessThan { + value: compared, .. + } => Filter::Less(path_expression(), number(compared)), + EntityTablePropertyFilter::LessThanOrEqual { + value: compared, .. + } => Filter::LessOrEqual(path_expression(), number(compared)), + EntityTablePropertyFilter::ContainsSegment { + value: compared, .. + } => Filter::ContainsSegment(path_expression(), text(compared)), + EntityTablePropertyFilter::StartsWith { + value: compared, .. + } => Filter::StartsWith(path_expression(), text(compared)), + EntityTablePropertyFilter::EndsWith { + value: compared, .. + } => Filter::EndsWith(path_expression(), text(compared)), + } +} + +fn type_include_filter<'f>(entity_type_ids: &[VersionedUrl]) -> Filter<'f, Entity> { + Filter::Any( + entity_type_ids + .iter() + .map(|entity_type_id| { + Filter::Equal( + FilterExpression::Path { + path: EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::VersionedUrl, + inheritance_depth: None, + }, + }, + FilterExpression::Parameter { + parameter: Parameter::Text(entity_type_id.to_string().into()), + convert: None, + }, + ) + }) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use core::str::FromStr as _; + + use hash_codec::numeric::Real; + use hash_graph_store::entity::EntityTableFilter; + use type_system::{ontology::BaseUrl, principal::actor_group::WebId}; + use uuid::Uuid; + + use super::*; + use crate::store::postgres::query::test_helper::trim_whitespace; + + fn params(webs: Option>) -> QueryEntitiesTableParams { + QueryEntitiesTableParams { + filter: EntityTableFilter { + webs: webs.map_or_else(EntityTableWebScope::default, |webs| { + EntityTableWebScope::Include { webs } + }), + types: EntityTableTypeScope::default(), + include_archived: false, + include_drafts: false, + property_filters: Vec::new(), + }, + cursor: None, + limit: 500, + sort: EntityTableSorting::default(), + conversions: Vec::new(), + include_summary: false, + include_entity_types: None, + } + } + + fn sample_timestamp() -> Timestamp { + serde_json::from_value(serde_json::json!("2025-01-01T00:00:00Z")) + .expect("the timestamp should deserialize") + } + + #[test] + fn cursor_token_roundtrip() { + let cursor = EntityTableCursor { + transaction_time: sample_timestamp(), + decision_time: sample_timestamp(), + type_universe: Some(vec![ + VersionedUrl::from_str("https://example.com/types/entity-type/person/v/1") + .expect("the URL should be a valid versioned URL"), + ]), + sort: EntityTableSorting { + key: EntityTableSortKey::Label, + ordering: hash_graph_store::query::Ordering::Ascending, + }, + include_drafts: false, + position: EntityQueryCursor { + values: vec![ + CursorField::Uuid(Uuid::nil()), + CursorField::String("cursor value".into()), + ], + }, + }; + + let token = serde_json::to_value(&cursor).expect("the cursor should serialize"); + assert!(token.is_string(), "the wire shape is an opaque token"); + + let decoded: EntityTableCursor = + serde_json::from_value(token.clone()).expect("the token should decode"); + + pretty_assertions::assert_eq!( + serde_json::to_value(&decoded).expect("the cursor should serialize"), + token, + ); + + assert_eq!(decoded.sort, cursor.sort, "the token pins the sort"); + assert!(!decoded.include_drafts); + } + + // The NOT clauses are compiled next to the derived universe's GIN-friendly + // include clause in the full page statement (`table_page_statement`) — on + // their own they would be unindexable, which the type scope's shape rules + // out. + #[test] + fn table_exclusion_scopes_statement() { + let temporal_axes = QueryTemporalAxesUnresolved::all().resolve(); + let mut params = params(None); + params.filter.webs = EntityTableWebScope::Exclude { + webs: vec![WebId::new(Uuid::nil())], + }; + params.filter.types = EntityTableTypeScope::Exclude { + entity_type_base_urls: vec![ + BaseUrl::new("https://example.com/types/entity-type/noise/".to_owned()) + .expect("the URL should be a valid base URL"), + ], + }; + params.filter.include_archived = true; + + let scope = scope_filter(¶ms); + + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), false); + compiler + .add_filter(&scope) + .expect("the scope filter should compile"); + compiler.add_selection_path(&EntityQueryPath::Uuid); + + let (statement, _parameters) = compiler.compile(); + + pretty_assertions::assert_eq!( + trim_whitespace(&statement), + trim_whitespace( + r#"SELECT "entity_temporal_metadata_0_0_0"."entity_uuid" + FROM "entity_temporal_metadata" AS "entity_temporal_metadata_0_0_0" + INNER JOIN "entity_edition_cache" AS "entity_edition_cache_0_1_0" ON "entity_edition_cache_0_1_0"."entity_edition_id" = "entity_temporal_metadata_0_0_0"."entity_edition_id" + WHERE ("entity_temporal_metadata_0_0_0"."draft_id" IS NULL) + AND ("entity_temporal_metadata_0_0_0"."transaction_time" @> $1::TIMESTAMPTZ) + AND ("entity_temporal_metadata_0_0_0"."decision_time" && $2) + AND ((NOT("entity_temporal_metadata_0_0_0"."web_id" = ANY($3))) + AND (NOT("entity_edition_cache_0_1_0"."base_urls" @> ARRAY[$4]::text[])))"#, + ), + ); + } + + #[test] + fn table_property_filters_statement() { + let property = |suffix: &str| { + BaseUrl::new(format!("https://example.com/types/property-type/{suffix}/")) + .expect("the URL should be a valid base URL") + }; + // Every operator once, pinning each condition's SQL shape. + let property_filters = vec![ + EntityTablePropertyFilter::HasAnyValue { + property: property("a"), + }, + EntityTablePropertyFilter::IsEmpty { + property: property("b"), + }, + EntityTablePropertyFilter::IsTrue { + property: property("c"), + }, + EntityTablePropertyFilter::IsFalse { + property: property("d"), + }, + EntityTablePropertyFilter::Equals { + property: property("e"), + value: EntityTablePropertyValue::String("x".to_owned()), + }, + EntityTablePropertyFilter::NotEquals { + property: property("f"), + value: EntityTablePropertyValue::Number(Real::from(1)), + }, + EntityTablePropertyFilter::GreaterThan { + property: property("g"), + value: Real::from(2), + }, + EntityTablePropertyFilter::GreaterThanOrEqual { + property: property("h"), + value: Real::from(3), + }, + EntityTablePropertyFilter::LessThan { + property: property("i"), + value: Real::from(4), + }, + EntityTablePropertyFilter::LessThanOrEqual { + property: property("j"), + value: Real::from(5), + }, + EntityTablePropertyFilter::ContainsSegment { + property: property("k"), + value: "y".to_owned(), + }, + EntityTablePropertyFilter::StartsWith { + property: property("l"), + value: "z".to_owned(), + }, + EntityTablePropertyFilter::EndsWith { + property: property("m"), + value: "w".to_owned(), + }, + ]; + + let filter = + property_filters_filter(&property_filters).expect("the filters should build a filter"); + + let temporal_axes = QueryTemporalAxesUnresolved::all().resolve(); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), false); + compiler + .add_filter(&filter) + .expect("the property filter should compile"); + compiler.add_selection_path(&EntityQueryPath::Uuid); + + let (statement, _parameters) = compiler.compile(); + + pretty_assertions::assert_eq!( + trim_whitespace(&statement), + trim_whitespace( + r#"SELECT "entity_temporal_metadata_0_0_0"."entity_uuid" + FROM "entity_temporal_metadata" AS "entity_temporal_metadata_0_0_0" + INNER JOIN "entity_editions" AS "entity_editions_0_1_0" ON "entity_editions_0_1_0"."entity_edition_id" = "entity_temporal_metadata_0_0_0"."entity_edition_id" + WHERE ("entity_temporal_metadata_0_0_0"."draft_id" IS NULL) + AND ("entity_temporal_metadata_0_0_0"."transaction_time" @> $2::TIMESTAMPTZ) + AND ("entity_temporal_metadata_0_0_0"."decision_time" && $3) + AND ((jsonb_path_query_first("entity_editions_0_1_0"."properties", (($1::text)::jsonpath)) IS NOT NULL) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($4::text)::jsonpath)) IS NULL) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($5::text)::jsonpath)) = $6) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($7::text)::jsonpath)) = $8) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($9::text)::jsonpath)) = $10) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($11::text)::jsonpath)) != $12) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($13::text)::jsonpath)) > $14) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($15::text)::jsonpath)) >= $16) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($17::text)::jsonpath)) < $18) + AND (jsonb_path_query_first("entity_editions_0_1_0"."properties", (($19::text)::jsonpath)) <= $20) + AND (strpos(((jsonb_path_query_first("entity_editions_0_1_0"."properties", (($21::text)::jsonpath))) #>> '{}'::text[]), $22) > 0) + AND (starts_with(((jsonb_path_query_first("entity_editions_0_1_0"."properties", (($23::text)::jsonpath))) #>> '{}'::text[]), $24)) + AND (right(((jsonb_path_query_first("entity_editions_0_1_0"."properties", (($25::text)::jsonpath))) #>> '{}'::text[]), length($26)) = $26))"#, + ), + ); + } + + #[test] + fn sort_keys_compile_to_their_order_by_columns() { + let order_by_lines = [ + (EntityTableSortKey::CreatedAtDecisionTime, false), + (EntityTableSortKey::EditionCreatedAtDecisionTime, false), + (EntityTableSortKey::Label, false), + (EntityTableSortKey::TypeTitle, false), + (EntityTableSortKey::Archived, false), + (EntityTableSortKey::CreatedAtDecisionTime, true), + ] + .map(|(key, include_drafts)| { + let temporal_axes = QueryTemporalAxesUnresolved::all().resolve(); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), include_drafts); + compiler.set_limit(10); + compiler.set_statement_shape(StatementShape::KeysFirst); + + let sorting = EntityQuerySorting { + paths: sorting_records( + EntityTableSorting { + key, + ordering: hash_graph_store::query::Ordering::Descending, + }, + include_drafts, + ), + cursor: None, + }; + let cursor_parameters = sorting.encode().expect("the sorting should encode"); + sorting + .compile(&mut compiler, cursor_parameters.as_ref(), &temporal_axes) + .expect("the sorting should compile"); + + let (statement, _parameters) = compiler.compile(); + let keyset_order = statement + .split("ORDER BY") + .nth(1) + .expect("the statement should have an ORDER BY") + .split("LIMIT") + .next() + .expect("the ORDER BY should precede the LIMIT") + .trim() + .to_owned(); + + keyset_order + }); + + pretty_assertions::assert_eq!( + order_by_lines, + [ + r#""created_at_decision_time" DESC, "entity_uuid" DESC"#, + r#""decision_time" DESC, "entity_uuid" DESC"#, + r#"("labels")[1] DESC, "entity_uuid" DESC"#, + r#"("type_titles")[1] DESC, "entity_uuid" DESC"#, + r#""archived" DESC, "entity_uuid" DESC"#, + r#""created_at_decision_time" DESC, "entity_uuid" DESC, "draft_id" DESC"#, + ] + ); + } + + #[test] + fn table_page_statement() { + let temporal_axes = QueryTemporalAxesUnresolved::all().resolve(); + let params = params(Some(vec![WebId::new(Uuid::nil())])); + + let scope = scope_filter(¶ms); + let types = type_include_filter(&[VersionedUrl::from_str( + "https://example.com/types/entity-type/person/v/1", + ) + .expect("the URL should be a valid versioned URL")]); + + let row_paths = RowPaths::new(); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), false); + compiler + .add_filter(&scope) + .expect("the scope filter should compile"); + compiler + .add_filter(&types) + .expect("the type filter should compile"); + compiler.set_limit(params.limit); + compiler.set_statement_shape(StatementShape::KeysFirst); + + let sorting = EntityQuerySorting { + paths: sorting_records(params.sort, params.filter.include_drafts), + cursor: None, + }; + let cursor_parameters = sorting.encode().expect("the sorting should encode"); + sorting + .compile(&mut compiler, cursor_parameters.as_ref(), &temporal_axes) + .expect("the sorting should compile"); + + RowIndices::install(&mut compiler, &row_paths); + + let (statement, _parameters) = compiler.compile(); + + pretty_assertions::assert_eq!( + trim_whitespace(&statement), + trim_whitespace( + r#"WITH "roots" AS (SELECT + "entity_ids_2_1_0"."created_at_decision_time", "entity_temporal_metadata_0_0_0"."entity_uuid", "entity_temporal_metadata_0_0_0"."web_id", "entity_temporal_metadata_0_0_0"."draft_id", "entity_temporal_metadata_0_0_0"."entity_edition_id", "entity_ids_2_1_0"."created_at_transaction_time", "entity_temporal_metadata_0_0_0"."decision_time", "entity_ids_2_1_0"."created_by_id" + FROM "entity_temporal_metadata" AS "entity_temporal_metadata_0_0_0" + INNER JOIN "entity_editions" AS "entity_editions_0_1_0" ON "entity_editions_0_1_0"."entity_edition_id" = "entity_temporal_metadata_0_0_0"."entity_edition_id" + INNER JOIN "entity_edition_cache" AS "entity_edition_cache_1_1_0" ON "entity_edition_cache_1_1_0"."entity_edition_id" = "entity_temporal_metadata_0_0_0"."entity_edition_id" + INNER JOIN "entity_ids" AS "entity_ids_2_1_0" ON "entity_ids_2_1_0"."web_id" = "entity_temporal_metadata_0_0_0"."web_id" AND "entity_ids_2_1_0"."entity_uuid" = "entity_temporal_metadata_0_0_0"."entity_uuid" + WHERE ("entity_temporal_metadata_0_0_0"."draft_id" IS NULL) + AND ("entity_temporal_metadata_0_0_0"."transaction_time" @> $1::TIMESTAMPTZ) + AND ("entity_temporal_metadata_0_0_0"."decision_time" && $2) + AND (("entity_temporal_metadata_0_0_0"."web_id" = ANY($3)) + AND ("entity_editions_0_1_0"."archived" != $4)) + AND (("entity_edition_cache_1_1_0"."versioned_urls" @> ARRAY[$5]::text[]))), + "limited" AS (SELECT * + FROM "roots" + ORDER BY "created_at_decision_time" DESC, "entity_uuid" DESC LIMIT 500) + SELECT DISTINCT ON("limited"."created_at_decision_time", "limited"."entity_uuid") "limited"."created_at_decision_time", "limited"."entity_uuid", "limited"."web_id", "limited"."draft_id", "limited"."entity_edition_id", ("entity_edition_cache_2_1_0"."labels")[1], "entity_edition_cache_2_1_0"."versioned_urls", "entity_edition_cache_2_1_0"."type_titles", "entity_edition_cache_2_1_0"."direct_types", "limited"."created_at_transaction_time", "limited"."decision_time", "limited"."created_by_id", "entity_editions_2_1_0"."created_by_id", "entity_editions_2_1_0"."archived", "entity_editions_2_1_0"."properties", "entity_editions_2_1_0"."property_metadata", "entity_has_left_entity_2_1_0"."left_web_id", "entity_has_left_entity_2_1_0"."left_entity_uuid", "entity_temporal_metadata_2_2_0"."entity_edition_id", ("entity_edition_cache_2_3_0"."labels")[1], "entity_edition_cache_2_3_0"."versioned_urls", "entity_edition_cache_2_3_0"."direct_types", "entity_has_right_entity_2_1_0"."right_web_id", "entity_has_right_entity_2_1_0"."right_entity_uuid", "entity_temporal_metadata_2_2_1"."entity_edition_id", ("entity_edition_cache_2_3_1"."labels")[1], "entity_edition_cache_2_3_2"."versioned_urls", "entity_edition_cache_2_3_3"."direct_types" + FROM "limited" + INNER JOIN "entity_edition_cache" AS "entity_edition_cache_2_1_0" ON "entity_edition_cache_2_1_0"."entity_edition_id" = "limited"."entity_edition_id" + INNER JOIN "entity_editions" AS "entity_editions_2_1_0" ON "entity_editions_2_1_0"."entity_edition_id" = "limited"."entity_edition_id" + LEFT OUTER JOIN "entity_has_left_entity" AS "entity_has_left_entity_2_1_0" ON "entity_has_left_entity_2_1_0"."web_id" = "limited"."web_id" AND "entity_has_left_entity_2_1_0"."entity_uuid" = "limited"."entity_uuid" + LEFT OUTER JOIN "entity_temporal_metadata" AS "entity_temporal_metadata_2_2_0" ON "entity_temporal_metadata_2_2_0"."web_id" = "entity_has_left_entity_2_1_0"."left_web_id" AND "entity_temporal_metadata_2_2_0"."entity_uuid" = "entity_has_left_entity_2_1_0"."left_entity_uuid" AND "entity_temporal_metadata_2_2_0"."draft_id" IS NULL AND "entity_temporal_metadata_2_2_0"."transaction_time" @> $1::TIMESTAMPTZ AND "entity_temporal_metadata_2_2_0"."decision_time" && $2 + LEFT OUTER JOIN "entity_edition_cache" AS "entity_edition_cache_2_3_0" ON "entity_edition_cache_2_3_0"."entity_edition_id" = "entity_temporal_metadata_2_2_0"."entity_edition_id" + LEFT OUTER JOIN "entity_has_right_entity" AS "entity_has_right_entity_2_1_0" ON "entity_has_right_entity_2_1_0"."web_id" = "limited"."web_id" AND "entity_has_right_entity_2_1_0"."entity_uuid" = "limited"."entity_uuid" + LEFT OUTER JOIN "entity_temporal_metadata" AS "entity_temporal_metadata_2_2_1" ON "entity_temporal_metadata_2_2_1"."web_id" = "entity_has_right_entity_2_1_0"."right_web_id" AND "entity_temporal_metadata_2_2_1"."entity_uuid" = "entity_has_right_entity_2_1_0"."right_entity_uuid" AND "entity_temporal_metadata_2_2_1"."draft_id" IS NULL AND "entity_temporal_metadata_2_2_1"."transaction_time" @> $1::TIMESTAMPTZ AND "entity_temporal_metadata_2_2_1"."decision_time" && $2 + LEFT OUTER JOIN "entity_temporal_metadata" AS "entity_temporal_metadata_2_2_2" ON "entity_temporal_metadata_2_2_2"."web_id" = "entity_has_right_entity_2_1_0"."right_web_id" AND "entity_temporal_metadata_2_2_2"."entity_uuid" = "entity_has_right_entity_2_1_0"."right_entity_uuid" AND "entity_temporal_metadata_2_2_2"."draft_id" IS NULL AND "entity_temporal_metadata_2_2_2"."transaction_time" @> $1::TIMESTAMPTZ AND "entity_temporal_metadata_2_2_2"."decision_time" && $2 + LEFT OUTER JOIN "entity_edition_cache" AS "entity_edition_cache_2_3_1" ON "entity_edition_cache_2_3_1"."entity_edition_id" = "entity_temporal_metadata_2_2_2"."entity_edition_id" + LEFT OUTER JOIN "entity_temporal_metadata" AS "entity_temporal_metadata_2_2_3" ON "entity_temporal_metadata_2_2_3"."web_id" = "entity_has_right_entity_2_1_0"."right_web_id" AND "entity_temporal_metadata_2_2_3"."entity_uuid" = "entity_has_right_entity_2_1_0"."right_entity_uuid" AND "entity_temporal_metadata_2_2_3"."draft_id" IS NULL AND "entity_temporal_metadata_2_2_3"."transaction_time" @> $1::TIMESTAMPTZ AND "entity_temporal_metadata_2_2_3"."decision_time" && $2 + LEFT OUTER JOIN "entity_edition_cache" AS "entity_edition_cache_2_3_2" ON "entity_edition_cache_2_3_2"."entity_edition_id" = "entity_temporal_metadata_2_2_3"."entity_edition_id" + LEFT OUTER JOIN "entity_temporal_metadata" AS "entity_temporal_metadata_2_2_4" ON "entity_temporal_metadata_2_2_4"."web_id" = "entity_has_right_entity_2_1_0"."right_web_id" AND "entity_temporal_metadata_2_2_4"."entity_uuid" = "entity_has_right_entity_2_1_0"."right_entity_uuid" AND "entity_temporal_metadata_2_2_4"."draft_id" IS NULL AND "entity_temporal_metadata_2_2_4"."transaction_time" @> $1::TIMESTAMPTZ AND "entity_temporal_metadata_2_2_4"."decision_time" && $2 + LEFT OUTER JOIN "entity_edition_cache" AS "entity_edition_cache_2_3_3" ON "entity_edition_cache_2_3_3"."entity_edition_id" = "entity_temporal_metadata_2_2_4"."entity_edition_id" + ORDER BY "limited"."created_at_decision_time" DESC, "limited"."entity_uuid" DESC LIMIT 500"#, + ), + ); + } +} diff --git a/libs/@local/graph/postgres-store/src/store/postgres/query/ast/mod.rs b/libs/@local/graph/postgres-store/src/store/postgres/query/ast/mod.rs index 5b2bfba07ca..37b0ef0c975 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/query/ast/mod.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/query/ast/mod.rs @@ -28,7 +28,7 @@ pub use self::{ UnaryExpression, UnaryOperator, VariadicExpression, VariadicOperator, WindowDefinition, }, identifier::Identifier, - non_empty::{EmptyVec, NonEmptyVec}, + non_empty::{EmptyVecError, NonEmptyVec}, set_quantifier::SetQuantifier, statement::{ OnConflict, SelectClause, SelectQuantifier, SelectStatement, SetOperator, SimpleSelect, diff --git a/libs/@local/graph/postgres-store/src/store/postgres/query/ast/non_empty.rs b/libs/@local/graph/postgres-store/src/store/postgres/query/ast/non_empty.rs index ac36e0c98b2..6723e7678d6 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/query/ast/non_empty.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/query/ast/non_empty.rs @@ -21,9 +21,9 @@ pub struct NonEmptyVec(Vec); #[derive(Debug, PartialEq, Eq, derive_more::Display)] #[display("the vector must not be empty")] -pub struct EmptyVec; +pub struct EmptyVecError; -impl Error for EmptyVec {} +impl Error for EmptyVecError {} impl NonEmptyVec { pub fn push(&mut self, value: T) { @@ -61,11 +61,11 @@ impl From for NonEmptyVec { } impl TryFrom> for NonEmptyVec { - type Error = EmptyVec; + type Error = EmptyVecError; fn try_from(values: Vec) -> Result { if values.is_empty() { - return Err(EmptyVec); + return Err(EmptyVecError); } Ok(Self(values)) } @@ -130,7 +130,7 @@ mod tests { assert_eq!( NonEmptyVec::::try_from(Vec::new()) .expect_err("an empty vector should be rejected"), - EmptyVec + EmptyVecError ); let mut values = NonEmptyVec::from(1); diff --git a/libs/@local/graph/postgres-store/src/store/postgres/query/mod.rs b/libs/@local/graph/postgres-store/src/store/postgres/query/mod.rs index 98055245677..1c2f537f262 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/query/mod.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/query/mod.rs @@ -31,7 +31,7 @@ use type_system::knowledge::{Entity, PropertyValue}; pub use self::{ ast::{ BinaryExpression, BinaryOperator, ColumnName, ColumnReference, CommonTableExpression, - Constant, EmptyVec, EqualityOperator, Expression, FromItem, FromItemFunctionBuilder, + Constant, EmptyVecError, EqualityOperator, Expression, FromItem, FromItemFunctionBuilder, FromItemJoinBuilder, FromItemSubqueryBuilder, FromItemTableBuilder, Function, GroupByClause, GroupingElement, Identifier, JoinType, Materialization, NonEmptyVec, NonFinitePercentage, NullsOrder, OnConflict, OrderByClause, SamplePercentage, diff --git a/libs/@local/graph/sdk/typescript/src/entity.ts b/libs/@local/graph/sdk/typescript/src/entity.ts index ac1405e9109..a644aa3142b 100644 --- a/libs/@local/graph/sdk/typescript/src/entity.ts +++ b/libs/@local/graph/sdk/typescript/src/entity.ts @@ -55,6 +55,7 @@ import type { ClosedEntityType, ClosedMultiEntityType, Entity, + EntityEditionId, EntityId, EntityMetadata, EntityUuid, @@ -89,10 +90,17 @@ import type { CreateEntityParams as GraphApiCreateEntityParams, DiffEntityParams, Entity as GraphApiEntity, + EntityTableFilter as EntityTableFilterGraphApi, + EntityTableLinkEndpoint as EntityTableLinkEndpointGraphApi, + EntityTablePropertyFilter as EntityTablePropertyFilterGraphApi, + EntityTableRow as EntityTableRowGraphApi, GraphApi, PatchEntityParams as GraphApiPatchEntityParams, QueryEntitiesRequest as QueryEntitiesRequestGraphApi, QueryEntitiesResponse as QueryEntitiesResponseGraphApi, + QueryEntitiesTableParams as QueryEntitiesTableParamsGraphApi, + QueryEntitiesTableResponse as QueryEntitiesTableResponseGraphApi, + QueryEntitiesTableResponseSummary as QueryEntitiesTableResponseSummaryGraphApi, QueryEntitySubgraphRequest as QueryEntitySubgraphRequestGraphApi, QueryEntitySubgraphResponse as QueryEntitySubgraphResponseGraphApi, ValidateEntityParams, @@ -318,6 +326,83 @@ export type SummarizeEntitiesResponse = DistributiveOmit< typeTitles?: Record; }; +export type EntityTablePropertyFilter = DistributiveOmit< + EntityTablePropertyFilterGraphApi, + "property" +> & { + property: BaseUrl; +}; + +export type EntityTableWebScope = + | { type: "include"; webs: WebId[] } + | { type: "exclude"; webs: WebId[] }; + +export type EntityTableTypeScope = + | { type: "include"; entityTypeIds: VersionedUrl[] } + | { type: "exclude"; entityTypeBaseUrls: BaseUrl[] }; + +export type QueryEntitiesTableParams = DistributiveOmit< + QueryEntitiesTableParamsGraphApi, + "filter" | "conversions" +> & { + filter: Omit< + EntityTableFilterGraphApi, + "types" | "webs" | "propertyFilters" + > & { + webs?: EntityTableWebScope; + types?: EntityTableTypeScope; + propertyFilters?: EntityTablePropertyFilter[]; + }; + conversions?: ConversionRequest[]; +}; + +export type EntityTableLinkEndpoint = DistributiveOmit< + EntityTableLinkEndpointGraphApi, + "entityId" | "entityTypeIds" +> & { + entityId: EntityId; + entityTypeIds: [VersionedUrl, ...VersionedUrl[]]; +}; + +export type EntityTableRow = DistributiveOmit< + EntityTableRowGraphApi, + | "entityId" + | "entityEditionId" + | "entityTypeIds" + | "createdBy" + | "lastEditedBy" + | "propertiesMetadata" + | "sourceEntity" + | "targetEntity" +> & { + entityId: EntityId; + entityEditionId: EntityEditionId; + entityTypeIds: [VersionedUrl, ...VersionedUrl[]]; + createdBy: ActorEntityUuid; + lastEditedBy: ActorEntityUuid; + propertiesMetadata: PropertyObjectMetadata; + sourceEntity?: EntityTableLinkEndpoint; + targetEntity?: EntityTableLinkEndpoint; +}; + +export type EntityTableSummary = DistributiveOmit< + QueryEntitiesTableResponseSummaryGraphApi, + "entityTypeIds" | "entityTypeTitles" +> & { + entityTypeIds: Record; + entityTypeTitles: Record; +}; + +export type QueryEntitiesTableResponse = DistributiveOmit< + QueryEntitiesTableResponseGraphApi, + "rows" | "summary" | "closedMultiEntityTypes" | "definitions" +> & { + rows: EntityTableRow[]; + summary?: EntityTableSummary; + closedMultiEntityTypes?: ClosedMultiEntityTypesRootMap; + definitions?: EntityTypeResolveDefinitions; +}; + export type SerializedQueryEntitySubgraphResponse = DistributiveOmit< QueryEntitySubgraphResponse, "subgraph" @@ -1559,8 +1644,8 @@ export const summarizeEntities = async ( }, authentication: AuthenticationContext, params: SummarizeEntitiesParams, -): Promise => { - return context.graphApi +): Promise => + context.graphApi .summarizeEntities(authentication.actorId, params) .then(({ data: response }) => ({ ...response, @@ -1576,7 +1661,31 @@ export const summarizeEntities = async ( | Record | undefined, })); -}; + +export const queryEntitiesTable = async ( + context: { + graphApi: GraphApi; + }, + authentication: AuthenticationContext, + params: QueryEntitiesTableParams, +): Promise => + context.graphApi + .queryEntitiesTable(authentication.actorId, params) + .then(({ data: response }) => ({ + ...response, + rows: response.rows as QueryEntitiesTableResponse["rows"], + summary: response.summary as QueryEntitiesTableResponse["summary"], + closedMultiEntityTypes: response.closedMultiEntityTypes + ? mapGraphApiClosedMultiEntityTypeMapToClosedMultiEntityTypeMap( + response.closedMultiEntityTypes, + ) + : undefined, + definitions: response.definitions + ? mapGraphApiEntityTypeResolveDefinitionsToEntityTypeResolveDefinitions( + response.definitions, + ) + : undefined, + })); export const serializeQueryEntitiesResponse = < PropertyMap extends TypeIdsAndPropertiesForEntity = diff --git a/libs/@local/graph/store/Cargo.toml b/libs/@local/graph/store/Cargo.toml index c8169997f42..d2086f5c69e 100644 --- a/libs/@local/graph/store/Cargo.toml +++ b/libs/@local/graph/store/Cargo.toml @@ -26,6 +26,7 @@ hash-graph-temporal-versioning = { workspace = true } type-system = { workspace = true } # Private third-party dependencies +base64 = { workspace = true } bytes = { workspace = true, optional = true } derive-where = { workspace = true } derive_more = { workspace = true, features = ["display", "error"] } diff --git a/libs/@local/graph/store/src/entity/mod.rs b/libs/@local/graph/store/src/entity/mod.rs index 3e23f996a9c..a10f01b06bc 100644 --- a/libs/@local/graph/store/src/entity/mod.rs +++ b/libs/@local/graph/store/src/entity/mod.rs @@ -14,6 +14,12 @@ pub use self::{ SummarizeEntitiesResponse, UpdateEntityEmbeddingsParams, ValidateEntityComponents, ValidateEntityError, ValidateEntityParams, }, + table::{ + EntityTableCursor, EntityTableFilter, EntityTableLinkEndpoint, EntityTablePropertyFilter, + EntityTablePropertyValue, EntityTableRow, EntityTableSortKey, EntityTableSorting, + EntityTableSummary, EntityTableTypeScope, EntityTableWebScope, QueryEntitiesTableParams, + QueryEntitiesTableResponse, + }, validation_report::{ EmptyEntityTypes, EntityRetrieval, EntityTypeRetrieval, EntityTypesError, EntityValidationReport, LinkDataStateError, LinkDataValidationReport, LinkError, @@ -25,6 +31,7 @@ pub use self::{ mod query; mod store; +mod table; mod validation_report; use type_system::knowledge::Entity; diff --git a/libs/@local/graph/store/src/entity/store.rs b/libs/@local/graph/store/src/entity/store.rs index 64b297c68ec..d696721c528 100644 --- a/libs/@local/graph/store/src/entity/store.rs +++ b/libs/@local/graph/store/src/entity/store.rs @@ -35,7 +35,10 @@ use utoipa::{ }; use crate::{ - entity::{EntityQueryCursor, EntityQuerySorting, EntityValidationReport}, + entity::{ + EntityQueryCursor, EntityQuerySorting, EntityValidationReport, QueryEntitiesTableParams, + QueryEntitiesTableResponse, + }, entity_type::{EntityTypeResolveDefinitions, IncludeEntityTypeOption}, error::{ CheckPermissionError, ClusterError, DeletionError, InsertionError, QueryError, UpdateError, @@ -872,6 +875,28 @@ pub trait EntityStore { params: SummarizeEntitiesParams<'_>, ) -> impl Future>> + Send; + /// Queries one page of the entities table. + /// + /// Runs the summary and the page read in one transaction: when the + /// filter's [`types`] scope is [`Exclude`], the visible-type universe is + /// derived from the summary and applied to the page query. Follow-up + /// pages continue on the first page's database state through the + /// [`EntityTableCursor`]. + /// + /// # Errors + /// + /// - if the cursor position cannot be encoded into query parameters + /// - if the request to the database fails + /// + /// [`types`]: crate::entity::EntityTableFilter::types + /// [`Exclude`]: crate::entity::EntityTableTypeScope::Exclude + /// [`EntityTableCursor`]: crate::entity::EntityTableCursor + fn query_entities_table( + &mut self, + actor_id: ActorEntityUuid, + params: QueryEntitiesTableParams, + ) -> impl Future>> + Send; + fn get_entity_by_id( &self, actor_id: ActorEntityUuid, diff --git a/libs/@local/graph/store/src/entity/table.rs b/libs/@local/graph/store/src/entity/table.rs new file mode 100644 index 00000000000..3d82feb1e72 --- /dev/null +++ b/libs/@local/graph/store/src/entity/table.rs @@ -0,0 +1,752 @@ +//! Types for the dedicated entities-table query. +//! +//! The table endpoint serves the entities table views: flat rows built from +//! materialized columns instead of a subgraph, with the summary and the page +//! read in one transaction. Follow-up pages are pinned to the first page's +//! database state through the [`EntityTableCursor`]. + +use std::collections::HashMap; + +use base64::Engine as _; +use hash_codec::numeric::Real; +use hash_graph_temporal_versioning::{DecisionTime, Timestamp, TransactionTime}; +use serde::{Deserialize, Serialize, de, ser}; +use type_system::{ + knowledge::{ + entity::id::{EntityEditionId, EntityId}, + property::{PropertyObject, metadata::PropertyObjectMetadata}, + }, + ontology::{VersionedUrl, id::BaseUrl}, + principal::{actor::ActorEntityUuid, actor_group::WebId}, +}; +#[cfg(feature = "utoipa")] +use utoipa::{ToSchema, openapi}; + +use crate::{ + entity::{ClosedMultiEntityTypeMap, EntityQueryCursor, QueryConversion}, + entity_type::{EntityTypeResolveDefinitions, IncludeEntityTypeOption}, + query::Ordering, +}; + +/// Sort key of the entities table, closed over the materialized, indexable +/// columns. Extending the table's sortable columns means adding a variant +/// here, never a free-form path. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub enum EntityTableSortKey { + /// When the entity was first created, in decision time. + CreatedAtDecisionTime, + /// When the current edition became effective — the "last edited" column. + EditionCreatedAtDecisionTime, + /// The entity's display label. + Label, + /// The title of the entity's first direct type. + TypeTitle, + /// Whether the entity is archived. + Archived, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EntityTableSorting { + pub key: EntityTableSortKey, + pub ordering: Ordering, +} + +impl Default for EntityTableSorting { + fn default() -> Self { + Self { + key: EntityTableSortKey::CreatedAtDecisionTime, + ordering: Ordering::Descending, + } + } +} + +/// A property value a table filter compares against. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(untagged)] +pub enum EntityTablePropertyValue { + Number(Real), + String(String), +} + +/// A filter on one of the table's property columns, mirroring the operators the table's filter +/// UI offers. +/// +/// The `type` tag selects the operator, `property` names the column, and each operator carries +/// exactly the value fields it needs, so a value-less operator with a value (or the reverse) is +/// unrepresentable. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum EntityTablePropertyFilter { + HasAnyValue { + property: BaseUrl, + }, + IsEmpty { + property: BaseUrl, + }, + IsTrue { + property: BaseUrl, + }, + IsFalse { + property: BaseUrl, + }, + Equals { + property: BaseUrl, + value: EntityTablePropertyValue, + }, + NotEquals { + property: BaseUrl, + value: EntityTablePropertyValue, + }, + GreaterThan { + property: BaseUrl, + value: Real, + }, + GreaterThanOrEqual { + property: BaseUrl, + value: Real, + }, + LessThan { + property: BaseUrl, + value: Real, + }, + LessThanOrEqual { + property: BaseUrl, + value: Real, + }, + /// Matches anywhere inside the property's text. + ContainsSegment { + property: BaseUrl, + value: String, + }, + StartsWith { + property: BaseUrl, + value: String, + }, + EndsWith { + property: BaseUrl, + value: String, + }, +} + +impl EntityTablePropertyFilter { + /// The property column the filter applies to. + #[must_use] + pub const fn property(&self) -> &BaseUrl { + match self { + Self::HasAnyValue { property } + | Self::IsEmpty { property } + | Self::IsTrue { property } + | Self::IsFalse { property } + | Self::Equals { property, .. } + | Self::NotEquals { property, .. } + | Self::GreaterThan { property, .. } + | Self::GreaterThanOrEqual { property, .. } + | Self::LessThan { property, .. } + | Self::LessThanOrEqual { property, .. } + | Self::ContainsSegment { property, .. } + | Self::StartsWith { property, .. } + | Self::EndsWith { property, .. } => property, + } + } +} + +/// Which webs the table draws rows from. +/// +/// The default is [`Exclude`] with an empty list: every web the actor may +/// see. +/// +/// [`Exclude`]: Self::Exclude +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum EntityTableWebScope { + /// Only the listed webs, where an empty list matches no rows at all. + Include { webs: Vec }, + /// Every web the actor may see except the listed ones. + Exclude { webs: Vec }, +} + +impl Default for EntityTableWebScope { + fn default() -> Self { + Self::Exclude { webs: Vec::new() } + } +} + +/// Which entity types the table draws rows from. +/// +/// [`Include`] selects versioned type ids, matching the versioned selections +/// a filter UI works with. [`Exclude`] cuts by base URL on purpose: an +/// exclusion is meant to hide a type regardless of which version an entity +/// carries. The default is [`Exclude`] with an empty list: the whole +/// visible-type universe. +/// +/// [`Include`]: Self::Include +/// [`Exclude`]: Self::Exclude +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum EntityTableTypeScope { + /// Only entities carrying one of the listed types. + Include { + // utoipa does not read `rename_all_fields`, so the fields carry their + // renames themselves to keep the spec aligned with the wire. + #[serde(rename = "entityTypeIds")] + entity_type_ids: Vec, + }, + /// The scope's visible-type universe, derived server-side from the + /// summary, except the types under the listed base URLs. + /// + /// Entities carrying an excluded type are left out entirely — of the + /// rows, the count, and the type summary alike. + Exclude { + #[serde(rename = "entityTypeBaseUrls")] + entity_type_base_urls: Vec, + }, +} + +impl Default for EntityTableTypeScope { + fn default() -> Self { + Self::Exclude { + entity_type_base_urls: Vec::new(), + } + } +} + +/// The scope of the entities table. +#[derive(Debug, Default, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EntityTableFilter { + #[serde(default)] + pub webs: EntityTableWebScope, + #[serde(default)] + pub types: EntityTableTypeScope, + #[serde(default)] + pub include_archived: bool, + #[serde(default)] + pub include_drafts: bool, + /// Conditions on property columns, all of which a row has to satisfy. + #[serde(default)] + pub property_filters: Vec, +} + +/// Continuation of an entities-table page sequence. +/// +/// It carries everything that determines the keyset's shape and the page +/// sequence's database state — the snapshot instants, the type universe, the +/// sort, the draft visibility, and the keyset position. A continuation reads +/// all of these from the token and ignores the request's own sort and draft +/// settings, so a re-sent request cannot drift from the sequence the token +/// was handed out for. On the wire it is a Base64-encoded token the client +/// treats as opaque. +/// +/// The snapshot is two bare instants rather than temporal axes: the table +/// only ever reads a current-instant snapshot, and a timestamp cannot express +/// the interval axes that would break its one-row-per-entity shape. +#[derive(Debug, Clone)] +pub struct EntityTableCursor { + /// The transaction-time instant the sequence's snapshot is pinned at. + pub transaction_time: Timestamp, + /// The decision-time instant the sequence reads at. + pub decision_time: Timestamp, + /// The type universe derived from the first page's summary. `None` when + /// the page ran on an explicit type filter, which the client re-sends + /// instead. + pub type_universe: Option>, + pub sort: EntityTableSorting, + pub include_drafts: bool, + pub position: EntityQueryCursor<'static>, +} + +impl Serialize for EntityTableCursor { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct Payload<'a> { + transaction_time: Timestamp, + decision_time: Timestamp, + type_universe: &'a Option>, + sort: &'a EntityTableSorting, + include_drafts: bool, + position: &'a EntityQueryCursor<'static>, + } + + let Self { + transaction_time, + decision_time, + type_universe, + sort, + include_drafts, + position, + } = self; + + let bytes = serde_json::to_vec(&Payload { + transaction_time: *transaction_time, + decision_time: *decision_time, + type_universe, + sort, + include_drafts: *include_drafts, + position, + }) + .map_err(ser::Error::custom)?; + + serializer.serialize_str(&base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes)) + } +} + +impl<'de> Deserialize<'de> for EntityTableCursor { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase", deny_unknown_fields)] + struct EntityTableCursorPayload<'a> { + transaction_time: Timestamp, + decision_time: Timestamp, + type_universe: Option>, + sort: EntityTableSorting, + include_drafts: bool, + #[serde(borrow)] + position: EntityQueryCursor<'a>, + } + + let token = String::deserialize(deserializer)?; + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(&token) + .map_err(de::Error::custom)?; + let EntityTableCursorPayload { + transaction_time, + decision_time, + type_universe, + sort, + include_drafts, + position, + } = serde_json::from_slice(&bytes).map_err(de::Error::custom)?; + + Ok(Self { + transaction_time, + decision_time, + type_universe, + sort, + include_drafts, + position: position.into_owned(), + }) + } +} + +#[cfg(feature = "utoipa")] +impl ToSchema<'_> for EntityTableCursor { + fn schema() -> (&'static str, openapi::RefOr) { + ( + "EntityTableCursor", + openapi::Schema::Object( + openapi::schema::ObjectBuilder::new() + .schema_type(openapi::SchemaType::String) + .description(Some("An opaque continuation token for the entities table")) + .build(), + ) + .into(), + ) + } +} + +/// Parameters for [`EntityStore::query_entities_table`]. +/// +/// [`EntityStore::query_entities_table`]: crate::entity::EntityStore::query_entities_table +#[derive(Debug, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct QueryEntitiesTableParams { + pub filter: EntityTableFilter, + /// Continuation from a previous page. Left out on the first page, which + /// reads a fresh snapshot at the current instant. The sort and draft + /// visibility of a continuation come from the token, not from the + /// request. + #[serde(default)] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub cursor: Option, + pub limit: usize, + #[serde(default)] + pub sort: EntityTableSorting, + /// Converts the rows' property values at each conversion's path into its + /// target data type, mirroring [`QueryEntitiesParams::conversions`]. + /// + /// [`QueryEntitiesParams::conversions`]: crate::entity::QueryEntitiesParams::conversions + #[serde(default)] + pub conversions: Vec>, + #[serde(default)] + pub include_summary: bool, + /// Resolves the closed schemas of the rows' types into + /// [`closed_multi_entity_types`] and [`definitions`] alongside the page. + /// + /// [`closed_multi_entity_types`]: QueryEntitiesTableResponse::closed_multi_entity_types + /// [`definitions`]: QueryEntitiesTableResponse::definitions + #[serde(default)] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub include_entity_types: Option, +} + +/// One endpoint of a link row: its identity, display label, and direct types. +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct EntityTableLinkEndpoint { + pub entity_id: EntityId, + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub label: Option, + /// The endpoint's direct types. + pub entity_type_ids: Vec, +} + +/// One row of the entities table, built entirely from materialized columns +/// plus the raw property object. +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct EntityTableRow { + pub entity_id: EntityId, + pub entity_edition_id: EntityEditionId, + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub label: Option, + /// The entity's direct types, parallel to + /// [`entity_type_titles`](Self::entity_type_titles). + pub entity_type_ids: Vec, + pub entity_type_titles: Vec, + pub created_at_transaction_time: Timestamp, + pub created_at_decision_time: Timestamp, + /// When the current edition became effective — the "last edited" column. + pub edition_created_at_decision_time: Timestamp, + pub created_by: ActorEntityUuid, + pub last_edited_by: ActorEntityUuid, + pub archived: bool, + pub properties: PropertyObject, + /// The value-level metadata of [`properties`](Self::properties), including + /// each value's resolved data type id. + pub properties_metadata: PropertyObjectMetadata, + /// The link's source, set on link rows only. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub source_entity: Option, + /// The link's target, set on link rows only. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub target_entity: Option, +} + +/// The summary of the entities table. +/// +/// The two halves have different scopes on purpose: [`count`] reflects the +/// page's full filters, where the type maps span the whole scope so a filter +/// UI can widen a narrowed selection. +/// +/// [`count`]: Self::count +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct EntityTableSummary { + /// How many entities match the page's full filters. + pub count: usize, + /// How many entities in the scope carry each (direct) type. + pub entity_type_ids: HashMap, + /// The display titles of the types in [`entity_type_ids`]. + /// + /// [`entity_type_ids`]: Self::entity_type_ids + pub entity_type_titles: HashMap, +} + +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "camelCase")] +pub struct QueryEntitiesTableResponse { + pub rows: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub closed_multi_entity_types: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub definitions: Option, + /// Continuation for the next page. `None` means this page was known to be + /// the last, where a page that exactly fills the limit still hands one out + /// and the follow-up page comes back empty. + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub cursor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(nullable = false))] + pub summary: Option, +} + +#[cfg(test)] +mod tests { + use core::str::FromStr as _; + + use serde_json::json; + use type_system::knowledge::entity::id::{EntityEditionId, EntityUuid}; + use uuid::Uuid; + + use super::*; + use crate::query::CursorField; + + fn sample_cursor() -> EntityTableCursor { + EntityTableCursor { + transaction_time: serde_json::from_value(json!("2025-01-01T00:00:00Z")) + .expect("the timestamp should deserialize"), + decision_time: serde_json::from_value(json!("2025-01-01T00:00:00Z")) + .expect("the timestamp should deserialize"), + type_universe: Some(vec![ + VersionedUrl::from_str("https://example.com/types/entity-type/person/v/1") + .expect("the URL should be a valid versioned URL"), + ]), + sort: EntityTableSorting::default(), + include_drafts: false, + position: EntityQueryCursor { + values: vec![CursorField::String("position".into())], + }, + } + } + + #[test] + fn garbage_cursor_tokens_are_rejected() { + for (token, case) in [ + (json!("not base64 !!!"), "invalid base64"), + ( + json!(base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(b"not json")), + "valid base64, invalid payload", + ), + ( + json!( + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(br#"{"stray": true}"#) + ), + "valid json, unknown payload shape", + ), + ] { + assert!( + serde_json::from_value::(token).is_err(), + "{case} must be rejected at the serde boundary", + ); + } + } + + fn params_json() -> serde_json::Value { + json!({ + "filter": {}, + "limit": 500, + }) + } + + #[test] + fn params_first_page_deserializes_with_defaults() { + let params: QueryEntitiesTableParams = + serde_json::from_value(params_json()).expect("the params should deserialize"); + + assert!(params.cursor.is_none()); + assert_eq!(params.limit, 500); + assert_eq!(params.sort, EntityTableSorting::default()); + assert!(!params.include_summary); + assert_eq!(params.filter.webs, EntityTableWebScope::default()); + assert_eq!(params.filter.types, EntityTableTypeScope::default()); + assert!(!params.filter.include_archived); + assert!(!params.filter.include_drafts); + } + + #[test] + fn scopes_deserialize_from_their_tags() { + let mut json = params_json(); + json["filter"] = json!({ + "webs": { + "type": "exclude", + "webs": ["00000000-0000-0000-0000-000000000000"], + }, + "types": { + "type": "exclude", + "entityTypeBaseUrls": ["https://example.com/types/entity-type/noise/"], + }, + }); + + let params: QueryEntitiesTableParams = + serde_json::from_value(json).expect("the params should deserialize"); + + assert!(matches!( + params.filter.webs, + EntityTableWebScope::Exclude { webs } if webs.len() == 1, + )); + assert!(matches!( + params.filter.types, + EntityTableTypeScope::Exclude { entity_type_base_urls } + if entity_type_base_urls.len() == 1, + )); + + let mut json = params_json(); + json["filter"] = json!({ + "types": { + "type": "include", + "entityTypeIds": ["https://example.com/types/entity-type/person/v/1"], + }, + }); + + let params: QueryEntitiesTableParams = + serde_json::from_value(json).expect("the params should deserialize"); + + assert!(matches!( + params.filter.types, + EntityTableTypeScope::Include { entity_type_ids } if entity_type_ids.len() == 1, + )); + } + + #[test] + fn params_continuation_deserializes() { + let token = serde_json::to_value(sample_cursor()).expect("the cursor should serialize"); + + let mut json = params_json(); + json["cursor"] = token; + + let params: QueryEntitiesTableParams = + serde_json::from_value(json).expect("the params should deserialize"); + + let cursor = params.cursor.expect("the params should carry the cursor"); + assert_eq!( + cursor.type_universe, + sample_cursor().type_universe, + "the cursor round-trips through the params", + ); + } + + #[test] + fn property_filters_deserialize() { + let mut json = params_json(); + json["filter"] = json!({ + "propertyFilters": [ + { + "type": "equals", + "property": "https://example.com/types/property-type/name/", + "value": "Alice", + }, + { + "type": "greaterThan", + "property": "https://example.com/types/property-type/age/", + "value": 30, + }, + { + "type": "hasAnyValue", + "property": "https://example.com/types/property-type/hobby/", + }, + ], + }); + + let params: QueryEntitiesTableParams = + serde_json::from_value(json).expect("the params should deserialize"); + + assert_eq!(params.filter.property_filters.len(), 3); + assert!(matches!( + ¶ms.filter.property_filters[0], + EntityTablePropertyFilter::Equals { + value: EntityTablePropertyValue::String(value), + .. + } if value == "Alice", + )); + assert!(matches!( + ¶ms.filter.property_filters[1], + EntityTablePropertyFilter::GreaterThan { .. }, + )); + assert!(matches!( + ¶ms.filter.property_filters[2], + EntityTablePropertyFilter::HasAnyValue { .. }, + )); + } + + #[test] + fn params_reject_unknown_fields() { + let mut json = params_json(); + json["page"] = json!({ "temporalAxes": {} }); + assert!( + serde_json::from_value::(json).is_err(), + "an unknown field must be rejected", + ); + } + + #[test] + fn response_wire_shape() { + fn timestamp(value: &str) -> Timestamp + where + for<'de> Timestamp: Deserialize<'de>, + { + serde_json::from_value(json!(value)).expect("the timestamp should deserialize") + } + + let response = QueryEntitiesTableResponse { + rows: vec![EntityTableRow { + entity_id: EntityId { + web_id: WebId::new(Uuid::nil()), + entity_uuid: EntityUuid::new(Uuid::nil()), + draft_id: None, + }, + entity_edition_id: EntityEditionId::new(Uuid::nil()), + label: None, + entity_type_ids: vec![ + VersionedUrl::from_str("https://example.com/types/entity-type/person/v/1") + .expect("the URL should be a valid versioned URL"), + ], + entity_type_titles: vec!["Person".to_owned()], + created_at_transaction_time: timestamp("2025-01-01T00:00:00Z"), + created_at_decision_time: timestamp("2025-01-01T00:00:00Z"), + edition_created_at_decision_time: timestamp("2025-01-02T00:00:00Z"), + created_by: ActorEntityUuid::new(Uuid::nil()), + last_edited_by: ActorEntityUuid::new(Uuid::nil()), + archived: false, + properties: serde_json::from_value(json!({})) + .expect("the empty object should be a valid property object"), + properties_metadata: PropertyObjectMetadata::default(), + source_entity: None, + target_entity: None, + }], + closed_multi_entity_types: None, + definitions: None, + cursor: Some(sample_cursor()), + summary: None, + }; + + let json = serde_json::to_value(&response).expect("the response should serialize"); + + assert!( + json["cursor"].is_string(), + "the cursor serializes as an opaque token", + ); + assert!( + json.get("summary").is_none(), + "an unrequested summary is omitted", + ); + + let row = &json["rows"][0]; + for key in [ + "entityId", + "entityEditionId", + "entityTypeIds", + "entityTypeTitles", + "createdAtTransactionTime", + "createdAtDecisionTime", + "editionCreatedAtDecisionTime", + "createdBy", + "lastEditedBy", + "archived", + "properties", + ] { + assert!(row.get(key).is_some(), "row is missing the `{key}` key"); + } + assert!( + row.get("label").is_none(), + "a missing label is omitted from the row", + ); + } +} diff --git a/libs/@local/graph/store/src/query/ordering.rs b/libs/@local/graph/store/src/query/ordering.rs index b9a3b5b4e40..8f44461b630 100644 --- a/libs/@local/graph/store/src/query/ordering.rs +++ b/libs/@local/graph/store/src/query/ordering.rs @@ -1,6 +1,6 @@ use type_system::ontology::VersionedUrl; -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, serde::Deserialize)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] #[serde(rename_all = "camelCase")] pub enum Ordering { diff --git a/libs/@local/graph/type-fetcher/src/store.rs b/libs/@local/graph/type-fetcher/src/store.rs index 1b51c6081d6..7be7aa00f9a 100644 --- a/libs/@local/graph/type-fetcher/src/store.rs +++ b/libs/@local/graph/type-fetcher/src/store.rs @@ -37,10 +37,10 @@ use hash_graph_store::{ entity::{ ClusterEntitiesParams, ClusterEntitiesResponse, CreateEntityParams, DeleteEntitiesParams, DeletionSummary, EntityStore, EntityValidationReport, HasPermissionForEntitiesParams, - PatchEntityParams, QueryEntitiesParams, QueryEntitiesResponse, QueryEntitySubgraphParams, - QueryEntitySubgraphResponse, SearchEntitiesParams, SearchEntitiesResponse, - SummarizeEntitiesParams, SummarizeEntitiesResponse, UpdateEntityEmbeddingsParams, - ValidateEntityParams, + PatchEntityParams, QueryEntitiesParams, QueryEntitiesResponse, QueryEntitiesTableParams, + QueryEntitiesTableResponse, QueryEntitySubgraphParams, QueryEntitySubgraphResponse, + SearchEntitiesParams, SearchEntitiesResponse, SummarizeEntitiesParams, + SummarizeEntitiesResponse, UpdateEntityEmbeddingsParams, ValidateEntityParams, }, entity_type::{ ArchiveEntityTypeParams, CommonQueryEntityTypesParams, CountEntityTypesParams, @@ -1654,6 +1654,14 @@ where self.store.summarize_entities(actor_id, params).await } + async fn query_entities_table( + &mut self, + actor_id: ActorEntityUuid, + params: QueryEntitiesTableParams, + ) -> Result> { + self.store.query_entities_table(actor_id, params).await + } + async fn patch_entity( &mut self, actor_id: ActorEntityUuid, diff --git a/libs/@local/hash-isomorphic-utils/src/graphql/scalar-mapping.ts b/libs/@local/hash-isomorphic-utils/src/graphql/scalar-mapping.ts index c00aa5a57a9..f503aaea9c2 100644 --- a/libs/@local/hash-isomorphic-utils/src/graphql/scalar-mapping.ts +++ b/libs/@local/hash-isomorphic-utils/src/graphql/scalar-mapping.ts @@ -75,6 +75,10 @@ export const scalars = { SummarizeEntitiesParams: "@local/hash-graph-client#SummarizeEntitiesParams", SummarizeEntitiesResponse: "@local/hash-graph-sdk/entity#SummarizeEntitiesResponse", + QueryEntitiesTableParams: + "@local/hash-graph-sdk/entity#QueryEntitiesTableParams", + QueryEntitiesTableResponse: + "@local/hash-graph-sdk/entity#QueryEntitiesTableResponse", QueryEntitiesRequest: "@local/hash-graph-sdk/entity#QueryEntitiesRequest", QueryEntitiesResponse: "@local/hash-graph-sdk/entity#SerializedQueryEntitiesResponse", diff --git a/libs/@local/hash-isomorphic-utils/src/graphql/type-defs/knowledge/entity.typedef.ts b/libs/@local/hash-isomorphic-utils/src/graphql/type-defs/knowledge/entity.typedef.ts index f73c2ce7786..75bfa79e7b4 100644 --- a/libs/@local/hash-isomorphic-utils/src/graphql/type-defs/knowledge/entity.typedef.ts +++ b/libs/@local/hash-isomorphic-utils/src/graphql/type-defs/knowledge/entity.typedef.ts @@ -5,6 +5,8 @@ export const entityTypedef = gql` scalar ClosedMultiEntityTypesDefinitions scalar SummarizeEntitiesParams scalar SummarizeEntitiesResponse + scalar QueryEntitiesTableParams + scalar QueryEntitiesTableResponse scalar CreatedByIdsMap scalar EntityId scalar EntityMetadata @@ -116,6 +118,10 @@ export const entityTypedef = gql` request: SummarizeEntitiesParams! ): SummarizeEntitiesResponse! + queryEntitiesTable( + request: QueryEntitiesTableParams! + ): QueryEntitiesTableResponse! + queryEntities(request: QueryEntitiesRequest!): QueryEntitiesResponse! searchEntities(request: SearchEntitiesRequest!): SearchEntitiesResponse! diff --git a/tests/graph/integration/postgres/email_filter_protection.rs b/tests/graph/integration/postgres/email_filter_protection.rs index 8425a47bb56..f989d10f7bc 100644 --- a/tests/graph/integration/postgres/email_filter_protection.rs +++ b/tests/graph/integration/postgres/email_filter_protection.rs @@ -20,7 +20,9 @@ use hash_graph_postgres_store::store::PostgresStoreSettings; use hash_graph_store::{ entity::{ CreateEntityParams, EntityQueryPath, EntityQuerySorting, EntityQuerySortingRecord, - EntityStore as _, QueryEntitiesParams, QueryEntitySubgraphParams, SummarizeEntitiesParams, + EntityStore as _, EntityTableFilter, EntityTablePropertyFilter, EntityTablePropertyValue, + EntityTableSorting, EntityTableTypeScope, EntityTableWebScope, QueryEntitiesParams, + QueryEntitiesTableParams, QueryEntitySubgraphParams, SummarizeEntitiesParams, }, entity_type::EntityTypeQueryPath, filter::{ @@ -3222,3 +3224,121 @@ async fn subgraph_traversal_masks_linked_user_email() { "Shortname should still be visible for traversed User entity" ); } + +// ============================================================================= +// Entities-Table Endpoint Protection Tests +// ============================================================================= +// +// The table endpoint compiles its property filters, its count, and its rows +// separately from the generic read path, so the protection is verified on it +// as well: the filter rewrite (WHERE and count) and the response masking. + +fn table_params_with(property_filters: Vec) -> QueryEntitiesTableParams { + QueryEntitiesTableParams { + filter: EntityTableFilter { + webs: EntityTableWebScope::default(), + types: EntityTableTypeScope::default(), + include_archived: false, + include_drafts: false, + property_filters, + }, + cursor: None, + limit: 100, + sort: EntityTableSorting::default(), + conversions: Vec::new(), + include_summary: true, + include_entity_types: None, + } +} + +fn table_email_filter(email: &str) -> EntityTablePropertyFilter { + EntityTablePropertyFilter::Equals { + property: BaseUrl::new(EMAIL_PROPERTY_BASE_URL.to_owned()) + .expect("the URL should be a valid base URL"), + value: EntityTablePropertyValue::String(email.to_owned()), + } +} + +/// email = X on the table endpoint → the User row is NOT returned and the +/// count does not reveal it, where the Invitation with the same email is. +#[tokio::test] +async fn table_email_eq_excludes_user() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = seed_with_email_types(&mut database).await; + api.create_user("shared@example.com", "alice").await; + let invitation = api.create_invitation("shared@example.com", "bob").await; + + let response = api + .query_entities_table( + api.account_id, + table_params_with(vec![table_email_filter("shared@example.com")]), + ) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 1, "only the invitation should match"); + assert_eq!( + response.rows[0].entity_id, + invitation.metadata.record_id.entity_id, + ); + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 1, "the count should not reveal the user"); +} + +/// `startsWith` probing on the table endpoint cannot enumerate a user's email +/// through row presence or the count. +#[tokio::test] +async fn table_email_starts_with_excludes_user() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = seed_with_user_only(&mut database).await; + api.create_user("secret@example.com", "alice").await; + + let response = api + .query_entities_table( + api.account_id, + table_params_with(vec![EntityTablePropertyFilter::StartsWith { + property: BaseUrl::new(EMAIL_PROPERTY_BASE_URL.to_owned()) + .expect("the URL should be a valid base URL"), + value: "secret".to_owned(), + }]), + ) + .await + .expect("the table query should succeed"); + + assert!(response.rows.is_empty(), "no row should reveal the user"); + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 0, "the count should not reveal the user"); +} + +/// The table's rows mask another user's email like the generic read path. +#[tokio::test] +async fn table_masks_user_email_in_rows() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = seed_with_email_types(&mut database).await; + api.create_user("hidden@example.com", "alice").await; + api.create_invitation("visible@example.com", "bob").await; + + let response = api + .query_entities_table(api.account_id, table_params_with(Vec::new())) + .await + .expect("the table query should succeed"); + + let email_url = BaseUrl::new(EMAIL_PROPERTY_BASE_URL.to_owned()) + .expect("the URL should be a valid base URL"); + assert_eq!(response.rows.len(), 2); + for row in &response.rows { + let is_user = row + .entity_type_ids + .iter() + .any(|url| url.base_url.as_str() == USER_ENTITY_TYPE_BASE_URL); + assert_eq!( + row.properties.properties().contains_key(&email_url), + !is_user, + "the user's email should be masked and the invitation's visible", + ); + } +} diff --git a/tests/graph/integration/postgres/lib.rs b/tests/graph/integration/postgres/lib.rs index c92eb3082a9..c7f28f866a6 100644 --- a/tests/graph/integration/postgres/lib.rs +++ b/tests/graph/integration/postgres/lib.rs @@ -20,6 +20,7 @@ mod property_metadata; mod property_type; mod read_only; mod sorting; +mod table; mod transaction; use std::collections::{HashMap, HashSet}; @@ -50,10 +51,10 @@ use hash_graph_store::{ entity::{ CreateEntityParams, DeleteEntitiesParams, DeletionSummary, EntityStore, EntityValidationReport, HasPermissionForEntitiesParams, PatchEntityParams, - QueryEntitiesParams, QueryEntitiesResponse, QueryEntitySubgraphParams, - QueryEntitySubgraphResponse, SearchEntitiesParams, SearchEntitiesResponse, - SummarizeEntitiesParams, SummarizeEntitiesResponse, UpdateEntityEmbeddingsParams, - ValidateEntityParams, + QueryEntitiesParams, QueryEntitiesResponse, QueryEntitiesTableParams, + QueryEntitiesTableResponse, QueryEntitySubgraphParams, QueryEntitySubgraphResponse, + SearchEntitiesParams, SearchEntitiesResponse, SummarizeEntitiesParams, + SummarizeEntitiesResponse, UpdateEntityEmbeddingsParams, ValidateEntityParams, }, entity_type::{ ArchiveEntityTypeParams, CountEntityTypesParams, CreateEntityTypeParams, EntityTypeStore, @@ -852,6 +853,14 @@ impl EntityStore for DatabaseApi<'_> { self.store.summarize_entities(actor_id, params).await } + async fn query_entities_table( + &mut self, + actor_id: ActorEntityUuid, + params: QueryEntitiesTableParams, + ) -> Result> { + self.store.query_entities_table(actor_id, params).await + } + async fn get_entity_by_id( &self, actor_id: ActorEntityUuid, diff --git a/tests/graph/integration/postgres/table.rs b/tests/graph/integration/postgres/table.rs new file mode 100644 index 00000000000..41dea5f561a --- /dev/null +++ b/tests/graph/integration/postgres/table.rs @@ -0,0 +1,1248 @@ +use std::collections::{HashMap, HashSet}; + +use hash_graph_authorization::policies::{ + Effect, + action::ActionName, + resource::{EntityResourceConstraint, ResourceConstraint}, + store::PolicyCreationParams, +}; +use hash_graph_store::{ + entity::{ + CreateEntityParams, EntityStore as _, EntityTableFilter, EntityTableRow, + EntityTableSortKey, EntityTableSorting, QueryEntitiesTableParams, + QueryEntitiesTableResponse, + }, + query::Ordering, +}; +use hash_graph_test_data::{data_type, entity, entity_type, property_type}; +use pretty_assertions::assert_eq; +use type_system::{ + knowledge::{ + entity::{LinkData, id::EntityUuid, provenance::ProvidedEntityEditionProvenance}, + property::{PropertyObject, PropertyObjectWithMetadata, metadata::PropertyProvenance}, + }, + ontology::{ + VersionedUrl, + id::{BaseUrl, OntologyTypeVersion}, + }, + principal::{actor::ActorType, actor_group::WebId}, + provenance::{OriginProvenance, OriginType}, +}; + +use crate::{DatabaseApi, DatabaseTestWrapper}; + +fn person_entity_type() -> VersionedUrl { + VersionedUrl { + base_url: BaseUrl::new( + "https://blockprotocol.org/@alice/types/entity-type/person/".to_owned(), + ) + .expect("the URL should be a valid base URL"), + version: OntologyTypeVersion { + major: 1, + pre_release: None, + }, + } +} + +fn page_entity_type() -> VersionedUrl { + VersionedUrl { + base_url: BaseUrl::new( + "https://blockprotocol.org/@alice/types/entity-type/page/".to_owned(), + ) + .expect("the URL should be a valid base URL"), + version: OntologyTypeVersion { + major: 1, + pre_release: None, + }, + } +} + +async fn insert(database: &mut DatabaseTestWrapper) -> DatabaseApi<'_> { + let mut api = database + .seed( + [ + data_type::VALUE_V1, + data_type::TEXT_V1, + data_type::NUMBER_V1, + ], + [ + property_type::NAME_V1, + property_type::AGE_V1, + property_type::TEXT_V1, + property_type::FAVORITE_SONG_V1, + property_type::FAVORITE_FILM_V1, + property_type::HOBBY_V1, + property_type::INTERESTS_V1, + ], + [ + entity_type::PERSON_V1, + entity_type::PAGE_V1, + entity_type::LINK_V1, + entity_type::link::FRIEND_OF_V1, + entity_type::link::ACQUAINTANCE_OF_V1, + ], + ) + .await + .expect("the database should seed"); + + let person = person_entity_type(); + let page = page_entity_type(); + let entities_properties = [ + (entity::PERSON_ALICE_V1, &person), + (entity::PERSON_BOB_V1, &person), + (entity::PERSON_CHARLES_V1, &person), + (entity::PAGE_V1, &page), + (entity::PAGE_V2, &page), + ]; + + for (idx, (entity, type_id)) in entities_properties.into_iter().enumerate() { + let properties: PropertyObject = + serde_json::from_str(entity).expect("the entity fixture should parse"); + // Deterministic but properly versioned — the opaque cursor round-trips + // entity uuids through serde, which rejects non-RFC4122 ones. + let entity_uuid = uuid::Builder::from_u128(idx as u128) + .with_variant(uuid::Variant::RFC4122) + .with_version(uuid::Version::Random) + .into_uuid(); + api.create_entity( + api.account_id, + CreateEntityParams { + web_id: WebId::new(api.account_id), + entity_uuid: Some(EntityUuid::new(entity_uuid)), + decision_time: None, + entity_type_ids: HashSet::from([type_id.clone()]), + properties: PropertyObjectWithMetadata::from_parts(properties, None) + .expect("the property object should build"), + confidence: None, + link_data: None, + draft: false, + policies: Vec::new(), + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the entity should be created"); + } + + api +} + +#[tokio::test] +async fn link_rows_carry_their_endpoints() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let friend_of_type = VersionedUrl { + base_url: BaseUrl::new( + "https://blockprotocol.org/@alice/types/entity-type/friend-of/".to_owned(), + ) + .expect("the URL should be a valid base URL"), + version: OntologyTypeVersion { + major: 1, + pre_release: None, + }, + }; + + // Alice and Bob were seeded with the deterministic uuids 0 and 1. + let endpoint_uuid = |index: u128| { + EntityUuid::new( + uuid::Builder::from_u128(index) + .with_variant(uuid::Variant::RFC4122) + .with_version(uuid::Version::Random) + .into_uuid(), + ) + }; + let web_id = WebId::new(api.account_id); + let alice_id = type_system::knowledge::entity::id::EntityId { + web_id, + entity_uuid: endpoint_uuid(0), + draft_id: None, + }; + let bob_id = type_system::knowledge::entity::id::EntityId { + web_id, + entity_uuid: endpoint_uuid(1), + draft_id: None, + }; + + api.create_entity( + actor_id, + CreateEntityParams { + web_id, + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([friend_of_type.clone()]), + properties: PropertyObjectWithMetadata::from_parts(PropertyObject::empty(), None) + .expect("the property object should build"), + link_data: Some(LinkData { + left_entity_id: alice_id, + right_entity_id: bob_id, + left_entity_confidence: None, + left_entity_provenance: PropertyProvenance::default(), + right_entity_confidence: None, + right_entity_provenance: PropertyProvenance::default(), + }), + draft: false, + policies: Vec::new(), + confidence: None, + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the link should be created"); + + let rows = collect_all_pages(&mut api, 10, EntityTableSorting::default()).await; + assert_eq!(rows.len(), 6); + + let link_rows = rows + .iter() + .filter(|row| row.source_entity.is_some()) + .collect::>(); + assert_eq!(link_rows.len(), 1, "exactly the link row has endpoints"); + + let link_row = link_rows[0]; + let source = link_row + .source_entity + .as_ref() + .expect("the link row should have a source"); + let target = link_row + .target_entity + .as_ref() + .expect("the link row should have a target"); + assert_eq!(source.entity_id, alice_id); + assert_eq!(target.entity_id, bob_id); + // The fixture types define no label property, so the hydrated labels are + // legitimately absent — the endpoint ids and types prove the join. + assert_eq!(source.entity_type_ids, vec![person_entity_type()]); + assert_eq!(target.entity_type_ids, vec![person_entity_type()]); + + for row in &rows { + if row.entity_id != link_row.entity_id { + assert!(row.source_entity.is_none()); + assert!(row.target_entity.is_none()); + } + } +} + +#[tokio::test] +async fn property_filters_narrow_the_page_but_not_the_type_summary() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(10); + params.include_summary = true; + params.filter.property_filters = vec![ + hash_graph_store::entity::EntityTablePropertyFilter::Equals { + property: BaseUrl::new( + "https://blockprotocol.org/@alice/types/property-type/name/".to_owned(), + ) + .expect("the URL should be a valid base URL"), + value: hash_graph_store::entity::EntityTablePropertyValue::String("Alice".to_owned()), + }, + ]; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 1, "only Alice matches the filter"); + + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!( + summary.count, 1, + "the count reflects the page query's filters" + ); + assert_eq!( + summary.entity_type_ids.len(), + 2, + "the type summary spans the scope so the filter UI can widen the selection", + ); +} + +#[tokio::test] +async fn label_sort_pages_alphabetically() { + use hash_graph_store::entity_type::{CreateEntityTypeParams, EntityTypeStore as _}; + use type_system::ontology::provenance::{OntologyOwnership, ProvidedOntologyEditionProvenance}; + + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + // The seeded types have no label property, leaving every row's label + // NULL — a labeled type makes the ordering observable. + api.create_entity_type( + actor_id, + CreateEntityTypeParams { + schema: serde_json::from_value(serde_json::json!({ + "$schema": "https://blockprotocol.org/types/modules/graph/0.3/schema/entity-type", + "kind": "entityType", + "$id": "http://localhost:3000/@alice/types/entity-type/labeled/v/1", + "type": "object", + "title": "Labeled", + "description": "An entity labeled by its name", + "properties": { + "https://blockprotocol.org/@alice/types/property-type/name/": { + "$ref": "https://blockprotocol.org/@alice/types/property-type/name/v/1" + } + }, + "labelProperty": "https://blockprotocol.org/@alice/types/property-type/name/", + })) + .expect("the entity type fixture should parse"), + ownership: OntologyOwnership::Local { + web_id: WebId::new(actor_id), + }, + conflict_behavior: hash_graph_store::query::ConflictBehavior::Fail, + provenance: ProvidedOntologyEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + }, + ) + .await + .expect("the entity type should be created"); + + let labeled_type = VersionedUrl { + base_url: BaseUrl::new( + "http://localhost:3000/@alice/types/entity-type/labeled/v/1" + .trim_end_matches("v/1") + .to_owned(), + ) + .expect("the URL should be a valid base URL"), + version: OntologyTypeVersion { + major: 1, + pre_release: None, + }, + }; + for name in ["Zebra", "Aardvark", "Mango"] { + api.create_entity( + actor_id, + CreateEntityParams { + web_id: WebId::new(actor_id), + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([labeled_type.clone()]), + properties: PropertyObjectWithMetadata::from_parts( + serde_json::from_value(serde_json::json!({ + "https://blockprotocol.org/@alice/types/property-type/name/": name, + })) + .expect("the properties should parse"), + None, + ) + .expect("the property object should build"), + confidence: None, + link_data: None, + draft: false, + policies: Vec::new(), + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the entity should be created"); + } + + let rows = collect_all_pages( + &mut api, + 2, + EntityTableSorting { + key: EntityTableSortKey::Label, + ordering: hash_graph_store::query::Ordering::Ascending, + }, + ) + .await; + + assert_eq!(rows.len(), 8); + let labels = rows.iter().map(|row| row.label.clone()).collect::>(); + assert_eq!( + labels[..3], + [ + Some("Aardvark".to_owned()), + Some("Mango".to_owned()), + Some("Zebra".to_owned()), + ], + "labeled rows come first, alphabetically", + ); + assert!( + labels[3..].iter().all(Option::is_none), + "unlabeled rows sort last", + ); +} + +fn page_params(limit: usize) -> QueryEntitiesTableParams { + QueryEntitiesTableParams { + filter: EntityTableFilter { + webs: hash_graph_store::entity::EntityTableWebScope::default(), + types: hash_graph_store::entity::EntityTableTypeScope::default(), + include_archived: false, + include_drafts: false, + property_filters: Vec::new(), + }, + cursor: None, + limit, + sort: hash_graph_store::entity::EntityTableSorting::default(), + conversions: Vec::new(), + include_summary: false, + include_entity_types: None, + } +} + +/// Pages through the whole table in chunks of `limit`, asserting no row is +/// seen twice, and returns all rows in page order. +async fn collect_all_pages( + api: &mut DatabaseApi<'_>, + limit: usize, + sort: EntityTableSorting, +) -> Vec { + collect_all_pages_with(api, limit, sort, false).await +} + +/// [`collect_all_pages`] with draft visibility. +async fn collect_all_pages_with( + api: &mut DatabaseApi<'_>, + limit: usize, + sort: EntityTableSorting, + include_drafts: bool, +) -> Vec { + let actor_id = api.account_id; + let mut cursor = None; + let mut seen = HashSet::new(); + let mut rows = Vec::new(); + + loop { + let mut params = page_params(limit); + params.sort = sort; + params.filter.include_drafts = include_drafts; + params.cursor = Option::take(&mut cursor); + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + let page_len = response.rows.len(); + assert!(page_len <= limit, "page exceeds its limit"); + + for row in response.rows { + assert!( + seen.insert(row.entity_id), + "duplicate row across pages: {:?}", + row.entity_id, + ); + rows.push(row); + } + + if page_len < limit { + assert!( + response.cursor.is_none(), + "a short page must not hand out a continuation", + ); + break; + } + let Some(new_cursor) = response.cursor else { + break; + }; + cursor = Some(new_cursor); + } + + rows +} + +#[tokio::test] +async fn first_page_summarizes_and_pages_stay_consistent() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(2); + params.include_summary = true; + params.include_entity_types = + Some(hash_graph_store::entity_type::IncludeEntityTypeOption::Resolved); + + let QueryEntitiesTableResponse { + rows, + cursor, + summary, + closed_multi_entity_types, + definitions, + } = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + let summary = summary.expect("the summary should be present when requested"); + assert_eq!(summary.count, 5); + assert_eq!( + summary.entity_type_ids, + HashMap::from([(person_entity_type(), 3), (page_entity_type(), 2)]), + ); + assert_eq!( + summary.entity_type_titles.keys().collect::>(), + summary.entity_type_ids.keys().collect::>(), + ); + + assert_eq!(rows.len(), 2); + assert!(cursor.is_some(), "a full page hands out a continuation"); + + let closed_multi_entity_types = closed_multi_entity_types.expect("entity types were requested"); + for row in &rows { + assert!( + closed_multi_entity_types.contains_key(&row.entity_type_ids[0]), + "each row's type is resolved alongside the page", + ); + } + assert!(definitions.is_some(), "resolved definitions were requested"); + + let all_rows = collect_all_pages(&mut api, 2, EntityTableSorting::default()).await; + assert_eq!(all_rows.len(), 5); + + // Default sort is created-at, newest first — verified over the keyset pages. + for pair in all_rows.windows(2) { + assert!( + pair[0].created_at_decision_time >= pair[1].created_at_decision_time, + "rows are not sorted by creation time, newest first", + ); + } + + for row in &all_rows { + assert_eq!(row.entity_type_ids.len(), 1); + assert_eq!(row.entity_type_titles.len(), 1); + assert!(!row.archived); + assert!(!row.properties.properties().is_empty()); + } +} + +#[tokio::test] +async fn explicit_type_filter_scopes_the_table() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(10); + params.filter.types = hash_graph_store::entity::EntityTableTypeScope::Include { + entity_type_ids: vec![person_entity_type()], + }; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 3); + assert!(response.cursor.is_none()); + assert!(response.summary.is_none()); + for row in &response.rows { + assert_eq!(row.entity_type_ids, vec![person_entity_type()]); + } +} + +#[tokio::test] +async fn conversions_convert_row_property_values() { + use std::collections::HashMap; + + use hash_graph_store::{ + data_type::{CreateDataTypeParams, DataTypeStore as _}, + query::ConflictBehavior, + }; + use type_system::ontology::{ + data_type::Conversions, + provenance::{OntologyOwnership, ProvidedOntologyEditionProvenance}, + }; + + let mut database = DatabaseTestWrapper::new().await; + let mut api = database + .seed( + [ + data_type::VALUE_V1, + data_type::NUMBER_V1, + data_type::LENGTH_V1, + data_type::METER_V1, + ], + [property_type::LENGTH_V1], + [entity_type::LINE_V1], + ) + .await + .expect("the database should seed"); + let actor_id = api.account_id; + + // The seed creates data types without conversions — the centimeter gets + // its meter conversion here explicitly. + api.create_data_type( + actor_id, + CreateDataTypeParams { + schema: serde_json::from_str(data_type::CENTIMETER_V1) + .expect("the data type fixture should parse"), + ownership: OntologyOwnership::Local { + web_id: WebId::new(actor_id), + }, + conflict_behavior: ConflictBehavior::Fail, + provenance: ProvidedOntologyEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + conversions: HashMap::from([( + BaseUrl::new("https://hash.ai/@h/types/data-type/meter/".to_owned()) + .expect("the URL should be a valid base URL"), + Conversions { + from: serde_json::from_value(serde_json::json!({ + "expression": ["*", "self", { "const": 100, "type": "number" }] + })) + .expect("the conversion should parse"), + to: serde_json::from_value(serde_json::json!({ + "expression": ["/", "self", { "const": 100, "type": "number" }] + })) + .expect("the conversion should parse"), + }, + )]), + }, + ) + .await + .expect("the data type should be created"); + + let line_type = VersionedUrl { + base_url: BaseUrl::new("http://localhost:3000/@alice/types/entity-type/line/".to_owned()) + .expect("the URL should be a valid base URL"), + version: OntologyTypeVersion { + major: 1, + pre_release: None, + }, + }; + let length_property = + BaseUrl::new("http://localhost:3000/@alice/types/property-type/length/".to_owned()) + .expect("the URL should be a valid base URL"); + + // A bare number is ambiguous between the length hierarchy's concrete + // types, so the property pins its data type explicitly. + let properties: PropertyObjectWithMetadata = serde_json::from_value(serde_json::json!({ + "value": { + "http://localhost:3000/@alice/types/property-type/length/": { + "value": 2, + "metadata": { + "dataTypeId": "https://hash.ai/@h/types/data-type/meter/v/1", + }, + }, + }, + })) + .expect("the properties should parse"); + api.create_entity( + actor_id, + CreateEntityParams { + web_id: WebId::new(actor_id), + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([line_type]), + properties, + confidence: None, + link_data: None, + draft: false, + policies: Vec::new(), + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the entity should be created"); + + let mut params = page_params(10); + params.conversions = serde_json::from_value(serde_json::json!([{ + "path": [length_property.to_string()], + "dataTypeId": "https://hash.ai/@h/types/data-type/centimeter/v/1", + }])) + .expect("the conversions should parse"); + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 1); + let row = &response.rows[0]; + + let type_system::knowledge::Property::Value(type_system::knowledge::PropertyValue::Number( + value, + )) = &row.properties.properties()[&length_property] + else { + panic!("length is a numeric value property"); + }; + assert_eq!( + *value, + hash_codec::numeric::Real::from(200), + "the meter value converts into centimeters", + ); + + let type_system::knowledge::property::metadata::PropertyMetadata::Value(value_metadata) = + &row.properties_metadata.value[&length_property] + else { + panic!("length is a value property"); + }; + assert_eq!( + value_metadata + .metadata + .data_type_id + .as_ref() + .map(ToString::to_string), + Some("https://hash.ai/@h/types/data-type/centimeter/v/1".to_owned()), + "the property metadata records the target data type", + ); +} + +#[tokio::test] +async fn exclude_webs_hides_the_excluded_web() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(10); + params.include_summary = true; + params.filter.webs = hash_graph_store::entity::EntityTableWebScope::Exclude { + webs: vec![WebId::new(actor_id)], + }; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 0, "the only seeded web is excluded"); + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 0); + assert!(summary.entity_type_ids.is_empty()); +} + +#[tokio::test] +async fn archived_entities_only_appear_when_requested() { + use hash_graph_store::entity::PatchEntityParams; + + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + // Bob was seeded with the deterministic uuid 1. + let bob_id = type_system::knowledge::entity::id::EntityId { + web_id: WebId::new(actor_id), + entity_uuid: EntityUuid::new( + uuid::Builder::from_u128(1) + .with_variant(uuid::Variant::RFC4122) + .with_version(uuid::Version::Random) + .into_uuid(), + ), + draft_id: None, + }; + api.patch_entity( + actor_id, + PatchEntityParams { + entity_id: bob_id, + decision_time: None, + entity_type_ids: HashSet::new(), + properties: Vec::new(), + draft: None, + archived: Some(true), + confidence: None, + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + }, + ) + .await + .expect("the entity should archive"); + + let mut params = page_params(10); + params.include_summary = true; + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + assert_eq!(response.rows.len(), 4, "the archived entity is hidden"); + assert_eq!( + response + .summary + .expect("the summary should be present when requested") + .count, + 4 + ); + assert!(response.rows.iter().all(|row| !row.archived)); + + let mut params = page_params(10); + params.include_summary = true; + params.filter.include_archived = true; + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + assert_eq!(response.rows.len(), 5, "the archived entity is included"); + assert_eq!( + response + .summary + .expect("the summary should be present when requested") + .count, + 5 + ); + assert_eq!( + response + .rows + .iter() + .filter(|row| row.archived && row.entity_id == bob_id) + .count(), + 1, + ); +} + +#[tokio::test] +async fn type_filter_narrows_the_count_but_not_the_type_summary() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(10); + params.include_summary = true; + params.filter.types = hash_graph_store::entity::EntityTableTypeScope::Include { + entity_type_ids: vec![person_entity_type()], + }; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 3); + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 3, "the count follows the type filter"); + assert_eq!( + summary.entity_type_ids.len(), + 2, + "the type summary spans the scope", + ); + assert_eq!(summary.entity_type_ids[&person_entity_type()], 3); + assert_eq!(summary.entity_type_ids[&page_entity_type()], 2); +} + +#[tokio::test] +async fn a_page_exactly_filling_the_limit_hands_out_a_final_empty_page() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + + // The seed creates exactly five entities, so the first page fills the + // limit and only the empty follow-up page reveals the end. + let rows = collect_all_pages(&mut api, 5, EntityTableSorting::default()).await; + assert_eq!(rows.len(), 5); +} + +#[tokio::test] +async fn a_continuation_reads_its_sort_from_the_token() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let ascending = EntityTableSorting { + key: EntityTableSortKey::CreatedAtDecisionTime, + ordering: Ordering::Ascending, + }; + + let reference = collect_all_pages(&mut api, 2, ascending).await; + + let mut params = page_params(2); + params.sort = ascending; + let first = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + let cursor = first + .cursor + .expect("a full page should hand out a continuation"); + + // The re-sent request contradicts the token with the opposite ordering. + // Were the continuation to honor it, the keyset comparison would flip and + // return the rows before the position instead of the ones after it. + let mut params = page_params(2); + params.cursor = Some(cursor); + params.sort = EntityTableSorting::default(); + let second = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!( + second + .rows + .iter() + .map(|row| row.entity_id) + .collect::>(), + reference[2..4] + .iter() + .map(|row| row.entity_id) + .collect::>(), + "the continuation should continue the token's sort, not the request's", + ); +} + +#[tokio::test] +async fn excluded_type_base_urls_leave_the_universe_and_the_summary() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(10); + params.include_summary = true; + params.filter.types = hash_graph_store::entity::EntityTableTypeScope::Exclude { + entity_type_base_urls: vec![page_entity_type().base_url], + }; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert_eq!(response.rows.len(), 3, "the two pages are excluded"); + assert!( + response + .rows + .iter() + .all(|row| row.entity_type_ids == vec![person_entity_type()]), + ); + + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 3); + assert_eq!( + summary.entity_type_ids.len(), + 1, + "excluded types leave the type summary as well", + ); + assert_eq!(summary.entity_type_ids[&person_entity_type()], 3); +} + +#[tokio::test] +async fn link_endpoints_hide_entities_the_actor_cannot_view() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + let web_id = WebId::new(api.account_id); + + let hidden_person = api + .create_entity( + actor_id, + CreateEntityParams { + web_id, + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([person_entity_type()]), + properties: PropertyObjectWithMetadata::from_parts( + serde_json::from_str(entity::PERSON_ALICE_V1) + .expect("the entity fixture should parse"), + None, + ) + .expect("the property object should build"), + confidence: None, + link_data: None, + draft: false, + policies: Vec::new(), + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the hidden person should be created"); + let hidden_id = hidden_person.metadata.record_id.entity_id; + + let friend_of_type = VersionedUrl { + base_url: BaseUrl::new( + "https://blockprotocol.org/@alice/types/entity-type/friend-of/".to_owned(), + ) + .expect("the URL should be a valid base URL"), + version: OntologyTypeVersion { + major: 1, + pre_release: None, + }, + }; + let alice_id = type_system::knowledge::entity::id::EntityId { + web_id, + entity_uuid: EntityUuid::new( + uuid::Builder::from_u128(0) + .with_variant(uuid::Variant::RFC4122) + .with_version(uuid::Version::Random) + .into_uuid(), + ), + draft_id: None, + }; + + api.create_entity( + actor_id, + CreateEntityParams { + web_id, + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([friend_of_type]), + properties: PropertyObjectWithMetadata::from_parts(PropertyObject::empty(), None) + .expect("the property object should build"), + link_data: Some(LinkData { + left_entity_id: alice_id, + right_entity_id: hidden_id, + left_entity_confidence: None, + left_entity_provenance: PropertyProvenance::default(), + right_entity_confidence: None, + right_entity_provenance: PropertyProvenance::default(), + }), + draft: false, + policies: Vec::new(), + confidence: None, + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the link should be created"); + + // The forbid policy comes after the link, because creating a link + // validates that the actor can see the endpoints it connects. + api.store + .insert_policies_into_database([&PolicyCreationParams { + name: Some("test-hidden-endpoint".to_owned()), + effect: Effect::Forbid, + principal: None, + actions: vec![ActionName::ViewEntity], + resource: Some(ResourceConstraint::Entity( + EntityResourceConstraint::Exact { + id: hidden_id.entity_uuid, + }, + )), + }]) + .await + .expect("the forbid policy should be inserted"); + + let rows = collect_all_pages(&mut api, 10, EntityTableSorting::default()).await; + + assert!( + rows.iter().all(|row| row.entity_id != hidden_id), + "the forbidden entity should not appear as its own row", + ); + + let link_row = rows + .iter() + .find(|row| row.source_entity.is_some() || row.target_entity.is_some()) + .expect("the link row should be visible"); + assert_eq!( + link_row + .source_entity + .as_ref() + .expect("the visible source should stay on the link row") + .entity_id, + alice_id, + ); + assert!( + link_row.target_entity.is_none(), + "the forbidden target should be hidden from the link row", + ); +} + +#[tokio::test] +async fn included_webs_scope_the_page_and_empty_include_matches_nothing() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut params = page_params(10); + params.include_summary = true; + params.filter.webs = hash_graph_store::entity::EntityTableWebScope::Include { + webs: vec![WebId::new(api.account_id)], + }; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + assert_eq!(response.rows.len(), 5, "the own web carries all rows"); + assert_eq!( + response + .summary + .expect("the summary should be present when requested") + .count, + 5, + ); + + let mut params = page_params(10); + params.include_summary = true; + params.filter.webs = hash_graph_store::entity::EntityTableWebScope::Include { webs: vec![] }; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + assert!( + response.rows.is_empty(), + "an empty include list should match no rows", + ); + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 0); + assert!(summary.entity_type_ids.is_empty()); +} + +#[tokio::test] +async fn drafts_are_hidden_unless_requested() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + api.create_entity( + actor_id, + CreateEntityParams { + web_id: WebId::new(api.account_id), + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([person_entity_type()]), + properties: PropertyObjectWithMetadata::from_parts( + serde_json::from_str(entity::PERSON_ALICE_V1) + .expect("the entity fixture should parse"), + None, + ) + .expect("the property object should build"), + confidence: None, + link_data: None, + draft: true, + policies: Vec::new(), + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the draft should be created"); + + let rows = collect_all_pages(&mut api, 10, EntityTableSorting::default()).await; + assert_eq!(rows.len(), 5, "drafts stay hidden by default"); + + // Paging with a small limit exercises the draft-aware keyset, whose + // cursor carries the draft id as a third sort column. + let rows = collect_all_pages_with(&mut api, 2, EntityTableSorting::default(), true).await; + assert_eq!(rows.len(), 6, "the draft shows up when requested"); + assert_eq!( + rows.iter() + .filter(|row| row.entity_id.draft_id.is_some()) + .count(), + 1, + ); +} + +#[tokio::test] +async fn multi_type_entities_count_once_and_pill_under_each_type() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + let mut properties: serde_json::Map = + serde_json::from_str(entity::PERSON_ALICE_V1).expect("the entity fixture should parse"); + properties.extend( + serde_json::from_str::>(entity::PAGE_V1) + .expect("the entity fixture should parse"), + ); + + api.create_entity( + actor_id, + CreateEntityParams { + web_id: WebId::new(api.account_id), + entity_uuid: None, + decision_time: None, + entity_type_ids: HashSet::from([person_entity_type(), page_entity_type()]), + properties: PropertyObjectWithMetadata::from_parts( + serde_json::from_value(serde_json::Value::Object(properties)) + .expect("the merged fixtures should be a valid property object"), + None, + ) + .expect("the property object should build"), + confidence: None, + link_data: None, + draft: false, + policies: Vec::new(), + provenance: ProvidedEntityEditionProvenance { + actor_type: ActorType::User, + origin: OriginProvenance::from_empty_type(OriginType::Api), + sources: Vec::new(), + }, + read_only: false, + }, + ) + .await + .expect("the multi-type entity should be created"); + + let mut params = page_params(10); + params.include_summary = true; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!( + summary.count, 6, + "a multi-type entity should count exactly once", + ); + assert_eq!( + summary.entity_type_ids, + HashMap::from([(person_entity_type(), 4), (page_entity_type(), 3)]), + "a multi-type entity should pill under each of its types", + ); + + let multi_row = response + .rows + .iter() + .find(|row| row.entity_type_ids.len() == 2) + .expect("the multi-type row should be on the page"); + assert_eq!( + multi_row.entity_type_ids.iter().collect::>(), + HashSet::from([&person_entity_type(), &page_entity_type()]), + ); + assert_eq!( + multi_row + .entity_type_titles + .iter() + .collect::>() + .len(), + 2, + "the titles should stay parallel to the two types", + ); +} From a589d0273bf148a90ab38c102d326c8c7f3b3bf4 Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Fri, 24 Jul 2026 14:39:44 +0200 Subject: [PATCH 02/18] BE-705: Clear clippy across all targets Eq derives for the property-filter types, a slice pattern instead of repeated indexing, expects for the long test bodies and the impl, and a reasoned expect for the summary request's independent include switches. --- .../src/store/postgres/knowledge/entity/summary.rs | 4 ++++ .../src/store/postgres/knowledge/entity/table.rs | 7 +++---- libs/@local/graph/store/src/entity/table.rs | 4 ++-- tests/graph/integration/postgres/table.rs | 8 +++++++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs index 2b791fbcd16..99d8560e25c 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/summary.rs @@ -24,6 +24,10 @@ use uuid::Uuid; use crate::store::postgres::query::SelectCompiler; /// Which summary dimensions to aggregate. +#[expect( + clippy::struct_excessive_bools, + reason = "the flags mirror the wire contract's independent include switches" +)] #[derive(Debug, Clone, Copy, Default)] pub(crate) struct EntitySummaryRequest { pub count: bool, diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs index 7609a91bc4b..5a88767dc94 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs @@ -335,6 +335,7 @@ impl PostgresStore where C: AsClient, { + #[expect(clippy::too_many_lines)] #[tracing::instrument(level = "info", skip(self, params))] pub(crate) async fn query_entities_table_impl( &self, @@ -1261,7 +1262,7 @@ mod tests { .expect("the sorting should compile"); let (statement, _parameters) = compiler.compile(); - let keyset_order = statement + statement .split("ORDER BY") .nth(1) .expect("the statement should have an ORDER BY") @@ -1269,9 +1270,7 @@ mod tests { .next() .expect("the ORDER BY should precede the LIMIT") .trim() - .to_owned(); - - keyset_order + .to_owned() }); pretty_assertions::assert_eq!( diff --git a/libs/@local/graph/store/src/entity/table.rs b/libs/@local/graph/store/src/entity/table.rs index 3d82feb1e72..75398a5d75b 100644 --- a/libs/@local/graph/store/src/entity/table.rs +++ b/libs/@local/graph/store/src/entity/table.rs @@ -65,7 +65,7 @@ impl Default for EntityTableSorting { } /// A property value a table filter compares against. -#[derive(Debug, Clone, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] #[serde(untagged)] pub enum EntityTablePropertyValue { @@ -79,7 +79,7 @@ pub enum EntityTablePropertyValue { /// The `type` tag selects the operator, `property` names the column, and each operator carries /// exactly the value fields it needs, so a value-less operator with a value (or the reverse) is /// unrepresentable. -#[derive(Debug, Clone, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] #[serde(tag = "type", rename_all = "camelCase")] pub enum EntityTablePropertyFilter { diff --git a/tests/graph/integration/postgres/table.rs b/tests/graph/integration/postgres/table.rs index 41dea5f561a..42535e19cf9 100644 --- a/tests/graph/integration/postgres/table.rs +++ b/tests/graph/integration/postgres/table.rs @@ -273,6 +273,7 @@ async fn property_filters_narrow_the_page_but_not_the_type_summary() { ); } +#[expect(clippy::too_many_lines)] #[tokio::test] async fn label_sort_pages_alphabetically() { use hash_graph_store::entity_type::{CreateEntityTypeParams, EntityTypeStore as _}; @@ -516,8 +517,11 @@ async fn first_page_summarizes_and_pages_stay_consistent() { // Default sort is created-at, newest first — verified over the keyset pages. for pair in all_rows.windows(2) { + let [newer, older] = pair else { + unreachable!("windows(2) should yield pairs"); + }; assert!( - pair[0].created_at_decision_time >= pair[1].created_at_decision_time, + newer.created_at_decision_time >= older.created_at_decision_time, "rows are not sorted by creation time, newest first", ); } @@ -554,6 +558,7 @@ async fn explicit_type_filter_scopes_the_table() { } } +#[expect(clippy::too_many_lines)] #[tokio::test] async fn conversions_convert_row_property_values() { use std::collections::HashMap; @@ -941,6 +946,7 @@ async fn excluded_type_base_urls_leave_the_universe_and_the_summary() { assert_eq!(summary.entity_type_ids[&person_entity_type()], 3); } +#[expect(clippy::too_many_lines)] #[tokio::test] async fn link_endpoints_hide_entities_the_actor_cannot_view() { let mut database = DatabaseTestWrapper::new().await; From 0e73f8ade04ea5635dcd6685191637540a40ee22 Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Fri, 24 Jul 2026 14:43:53 +0200 Subject: [PATCH 03/18] BE-705: Drop the frontend TODO pointing at this ticket The dedicated endpoint this PR adds is what the note asked for, and the table view moves onto it in the follow-up PR. --- .../shared/entities-visualizer/use-entities-visualizer-data.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx index 2bf7c0aab98..c29d3c000c4 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx @@ -136,8 +136,6 @@ export const useEntitiesVisualizerData = (params: { ], ); - // TODO(BE-705): a dedicated Graph endpoint will run the summary and the page - // query in one transaction, replacing this client-side coordination. const awaitingTypeUniverse = typeUniverse === null && !entityTypeBaseUrl && From e3c2e68d55ea9092cb16e6f5ddff315b91754733 Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Sat, 25 Jul 2026 14:20:02 +0200 Subject: [PATCH 04/18] BE-705: Separate the type selection from the excluded types and drop draft rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A selection used to replace the base-URL exclusions rather than narrow within them, so picking a type pill brought the noisy system types back into the rows, the count, and the pills. The exclusions move to their own filter field that always applies, and the selection is a plain list of versioned ids: absent for no narrowing, empty to match nothing. The visible-type universe stays a Graph-internal notion. Draft entities are no longer rows. Including them dropped the compiler's draft_id IS NULL from every temporal-metadata join, including the ones hydrating a link row's endpoints — an endpoint with drafts matched one row per edition, letting a product row mix columns from different editions while the outer DISTINCT ON collapsed the fan-out arbitrarily. Excluding drafts makes the one-row-per-key invariant structural. The derived universe is capped: past the limit it is dropped rather than grown into the cursor token and the statement's parameter list, which the scope filter makes safe. Cursor positions of the wrong arity are rejected instead of compiling into a silently weakened keyset. New tests pin the snapshot a continuation reads, the summary's policy scope, and that a selection cannot bring back an excluded type. --- libs/@local/graph/api/openapi/openapi.json | 73 ++---- libs/@local/graph/api/src/rest/entity/mod.rs | 19 +- .../store/postgres/knowledge/entity/table.rs | 215 ++++++++--------- .../@local/graph/sdk/typescript/src/entity.ts | 13 +- libs/@local/graph/store/src/entity/mod.rs | 4 +- libs/@local/graph/store/src/entity/store.rs | 11 +- libs/@local/graph/store/src/entity/table.rs | 217 +++++++++++------- .../postgres/email_filter_protection.rs | 8 +- tests/graph/integration/postgres/table.rs | 217 ++++++++++++++---- 9 files changed, 452 insertions(+), 325 deletions(-) diff --git a/libs/@local/graph/api/openapi/openapi.json b/libs/@local/graph/api/openapi/openapi.json index d6414a5b399..1bd5100688c 100644 --- a/libs/@local/graph/api/openapi/openapi.json +++ b/libs/@local/graph/api/openapi/openapi.json @@ -4990,10 +4990,22 @@ "type": "object", "description": "The scope of the entities table.", "properties": { - "includeArchived": { - "type": "boolean" + "entityTypeIds": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VersionedUrl" + }, + "description": "The types the rows are narrowed to, as the versioned selections a\nfilter UI works with.\n\nLeft out for no narrowing at all, where the table spans every type its\nscope holds. An empty list narrows to nothing and matches no rows.", + "nullable": true }, - "includeDrafts": { + "excludedTypeBaseUrls": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseUrl" + }, + "description": "Types the table never shows, whatever\n[`entity_type_ids`](Self::entity_type_ids) selects.\n\nEntities carrying one of these types are left out entirely — of the\nrows, the count, and the type summary alike — so a selection cannot\nbring them back. Base URLs rather than versioned ids on purpose: an\nexclusion is meant to hide a type regardless of which version an\nentity carries, and it covers inherited types as well as direct ones." + }, + "includeArchived": { "type": "boolean" }, "propertyFilters": { @@ -5003,9 +5015,6 @@ }, "description": "Conditions on property columns, all of which a row has to satisfy." }, - "types": { - "$ref": "#/components/schemas/EntityTableTypeScope" - }, "webs": { "$ref": "#/components/schemas/EntityTableWebScope" } @@ -5463,58 +5472,6 @@ } } }, - "EntityTableTypeScope": { - "oneOf": [ - { - "type": "object", - "description": "Only entities carrying one of the listed types.", - "required": [ - "entityTypeIds", - "type" - ], - "properties": { - "entityTypeIds": { - "type": "array", - "items": { - "$ref": "#/components/schemas/VersionedUrl" - } - }, - "type": { - "type": "string", - "enum": [ - "include" - ] - } - } - }, - { - "type": "object", - "description": "The scope's visible-type universe, derived server-side from the\nsummary, except the types under the listed base URLs.\n\nEntities carrying an excluded type are left out entirely — of the\nrows, the count, and the type summary alike.", - "required": [ - "entityTypeBaseUrls", - "type" - ], - "properties": { - "entityTypeBaseUrls": { - "type": "array", - "items": { - "$ref": "#/components/schemas/BaseUrl" - } - }, - "type": { - "type": "string", - "enum": [ - "exclude" - ] - } - } - } - ], - "description": "Which entity types the table draws rows from.\n\n[`Include`] selects versioned type ids, matching the versioned selections\na filter UI works with. [`Exclude`] cuts by base URL on purpose: an\nexclusion is meant to hide a type regardless of which version an entity\ncarries. The default is [`Exclude`] with an empty list: the whole\nvisible-type universe.\n\n[`Include`]: Self::Include\n[`Exclude`]: Self::Exclude", - "discriminator": { - "propertyName": "type" - } - }, "EntityTableWebScope": { "oneOf": [ { diff --git a/libs/@local/graph/api/src/rest/entity/mod.rs b/libs/@local/graph/api/src/rest/entity/mod.rs index e1f35d705ab..5e987141078 100644 --- a/libs/@local/graph/api/src/rest/entity/mod.rs +++ b/libs/@local/graph/api/src/rest/entity/mod.rs @@ -20,15 +20,15 @@ use hash_graph_store::{ EntityQueryCursor, EntityQuerySortingRecord, EntityQuerySortingToken, EntityQueryToken, EntityStore, EntityTableCursor, EntityTableFilter, EntityTableLinkEndpoint, EntityTablePropertyFilter, EntityTablePropertyValue, EntityTableRow, EntityTableSortKey, - EntityTableSorting, EntityTableSummary, EntityTableTypeScope, EntityTableWebScope, - EntityTypesError, EntityValidationReport, EntityValidationType, - HasPermissionForEntitiesParams, LinkDataStateError, LinkDataValidationReport, LinkError, - LinkTargetError, LinkValidationReport, LinkedEntityError, MetadataValidationReport, - PatchEntityParams, PropertyMetadataValidationReport, QueryConversion, - QueryEntitiesResponse, QueryEntitiesTableParams, QueryEntitiesTableResponse, - SearchEntitiesFilter, SearchEntitiesParams, SearchEntitiesResponse, - SummarizeEntitiesParams, SummarizeEntitiesResponse, UnexpectedEntityType, - UpdateEntityEmbeddingsParams, ValidateEntityComponents, ValidateEntityParams, + EntityTableSorting, EntityTableSummary, EntityTableWebScope, EntityTypesError, + EntityValidationReport, EntityValidationType, HasPermissionForEntitiesParams, + LinkDataStateError, LinkDataValidationReport, LinkError, LinkTargetError, + LinkValidationReport, LinkedEntityError, MetadataValidationReport, PatchEntityParams, + PropertyMetadataValidationReport, QueryConversion, QueryEntitiesResponse, + QueryEntitiesTableParams, QueryEntitiesTableResponse, SearchEntitiesFilter, + SearchEntitiesParams, SearchEntitiesResponse, SummarizeEntitiesParams, + SummarizeEntitiesResponse, UnexpectedEntityType, UpdateEntityEmbeddingsParams, + ValidateEntityComponents, ValidateEntityParams, }, filter::SemanticDistance, pool::StorePool, @@ -126,7 +126,6 @@ use crate::rest::{ EntityTablePropertyFilter, EntityTablePropertyValue, EntityTableWebScope, - EntityTableTypeScope, EntityTableRow, EntityTableSortKey, EntityTableSorting, diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs index 5a88767dc94..657b4356757 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs @@ -21,8 +21,8 @@ use hash_graph_store::{ EntityQueryCursor, EntityQueryPath, EntityQuerySorting, EntityQuerySortingRecord, EntityTableCursor, EntityTableLinkEndpoint, EntityTablePropertyFilter, EntityTablePropertyValue, EntityTableRow, EntityTableSortKey, EntityTableSorting, - EntityTableSummary, EntityTableTypeScope, EntityTableWebScope, QueryEntitiesTableParams, - QueryEntitiesTableResponse, + EntityTableSummary, EntityTableWebScope, QueryEntitiesTableParams, + QueryEntitiesTableResponse, TYPE_UNIVERSE_LIMIT, }, entity_type::{EntityTypeQueryPath, EntityTypeStore as _, IncludeEntityTypeOption}, error::QueryError, @@ -297,10 +297,9 @@ struct LinkEndpointEditions { target: Option, } -fn sorting_records( - sort: EntityTableSorting, - include_drafts: bool, -) -> Vec> { +/// The sort key plus the uuid tiebreaker — the keyset the cursor's position +/// carries. Drafts are never rows, so the entity uuid alone breaks ties. +fn sorting_records(sort: EntityTableSorting) -> Vec> { let key_path = match sort.key { EntityTableSortKey::CreatedAtDecisionTime => EntityQueryPath::CreatedAtDecisionTime, EntityTableSortKey::EditionCreatedAtDecisionTime => EntityQueryPath::DecisionTime, @@ -309,7 +308,7 @@ fn sorting_records( EntityTableSortKey::Archived => EntityQueryPath::Archived, }; - let mut records = vec![ + vec![ EntityQuerySortingRecord { path: key_path, ordering: sort.ordering, @@ -320,17 +319,17 @@ fn sorting_records( ordering: sort.ordering, nulls: None, }, - ]; - if include_drafts { - records.push(EntityQuerySortingRecord { - path: EntityQueryPath::DraftId, - ordering: sort.ordering, - nulls: None, - }); - } - records + ] } +/// Draft entities are never rows of the table. +/// +/// Excluding them keeps `draft_id IS NULL` on every temporal-metadata join the +/// compiler emits — including the ones hydrating a link row's endpoints, which +/// would otherwise match one row per draft edition and let a product row mix +/// columns from different editions. +const INCLUDE_DRAFTS: bool = false; + impl PostgresStore where C: AsClient, @@ -348,27 +347,22 @@ where .await .change_context(QueryError)?; - // The sort and draft visibility shape the keyset, so a continuation - // reads them from its token instead of trusting the re-sent request. - let (transaction_time, decision_time, mut type_universe, sort, include_drafts, position) = + // The sort shapes the keyset, so a continuation reads it from its + // token instead of trusting the re-sent request. The snapshot instants + // are minted here for a first page — a shade after the transaction's + // own snapshot, so a writer committing in between stays out of this + // page's aggregates but can still surface on a later page. + let (transaction_time, decision_time, mut type_universe, sort, position) = match ¶ms.cursor { None => { let now: Timestamp<()> = Timestamp::now(); - ( - now.cast(), - now.cast(), - None, - params.sort, - params.filter.include_drafts, - None, - ) + (now.cast(), now.cast(), None, params.sort, None) } Some(cursor) => ( cursor.transaction_time, cursor.decision_time, cursor.type_universe.clone(), cursor.sort, - cursor.include_drafts, Some(cursor.position.clone()), ), }; @@ -382,10 +376,7 @@ where policy_components.optimization_data(ActionName::ViewEntity), ); - let type_include_ids = match ¶ms.filter.types { - EntityTableTypeScope::Include { entity_type_ids } => Some(entity_type_ids.clone()), - EntityTableTypeScope::Exclude { .. } => None, - }; + let type_selection = params.filter.entity_type_ids.clone(); // Sensitive properties get the same two-sided protection the generic // read path applies: the responses mask them, and the user's filters @@ -415,20 +406,19 @@ where }); } - let needs_universe = type_include_ids.is_none() && type_universe.is_none(); + let needs_universe = type_selection.is_none() && type_universe.is_none(); // Without narrowing filters the page's full filters equal the scope - // (the include clause is result-neutral and the universe's base-URL - // exclusions live inside the scope filter), so the count can ride - // along in the summary scan instead of paying a second full scan. + // (the include clause is result-neutral and the excluded base URLs + // live inside the scope filter), so the count can ride along in the + // summary scan instead of paying a second full scan. let count_matches_scope = - type_include_ids.is_none() && params.filter.property_filters.is_empty(); + type_selection.is_none() && params.filter.property_filters.is_empty(); let scope_summaries = if params.include_summary || needs_universe { Some( self.scope_summary( &policy_filter, &scope_filter, &temporal_axes, - include_drafts, params.include_summary && count_matches_scope, ) .await?, @@ -437,22 +427,25 @@ where None }; if needs_universe { - type_universe = scope_summaries.as_ref().map(|summaries| { - let mut universe = summaries + type_universe = scope_summaries.as_ref().and_then(|summaries| { + let type_ids = summaries .type_ids .as_ref() - .expect("the type summary should always be requested") - .keys() - .cloned() - .collect::>(); - // The universe is sent as query parameters and travels inside the - // cursor — keep its order deterministic. - universe.sort_unstable(); - universe + .expect("the type summary should always be requested"); + // Past the limit the clause stops being worth its cost in + // parameters and cursor bytes, and the scope filter alone + // decides the rows either way. + (type_ids.len() <= TYPE_UNIVERSE_LIMIT).then(|| { + let mut universe = type_ids.keys().cloned().collect::>(); + // The universe is sent as query parameters and travels + // inside the cursor — keep its order deterministic. + universe.sort_unstable(); + universe + }) }); } - let type_filter = type_include_ids + let type_filter = type_selection .as_ref() .or(type_universe.as_ref()) .map(|entity_type_ids| type_include_filter(entity_type_ids)); @@ -473,7 +466,6 @@ where property_filter.as_ref(), ], &temporal_axes, - include_drafts, ) .await? } @@ -490,7 +482,7 @@ where }; let row_paths = RowPaths::new(); - let mut compiler = SelectCompiler::new(Some(&temporal_axes), include_drafts); + let mut compiler = SelectCompiler::new(Some(&temporal_axes), INCLUDE_DRAFTS); // The rows carry raw property objects, so the page masks them the // same way the generic read path masks its entities. @@ -527,7 +519,7 @@ where compiler.set_statement_shape(StatementShape::KeysFirst); let sorting = EntityQuerySorting { - paths: sorting_records(sort, include_drafts), + paths: sorting_records(sort), cursor: position, }; let cursor_parameters = sorting.encode().change_context(QueryError)?; @@ -558,12 +550,11 @@ where .map(|row| EntityTableCursor { transaction_time, decision_time, - type_universe: type_include_ids + type_universe: type_selection .is_none() .then(|| type_universe.clone()) .flatten(), sort, - include_drafts, position: EntityQueryCursor { values: cursor_indices .iter() @@ -705,17 +696,19 @@ where }) } - /// Runs the type summary over the scope, before any type filter, carrying - /// the scope count along when requested. + /// Runs the type summary over the scope, before any type selection, + /// carrying the scope count along when requested. + /// + /// The policy filter is part of the scope, so the counts and types only + /// ever span what the actor may view. async fn scope_summary( &self, policy_filter: &Filter<'_, Entity>, scope_filter: &Filter<'_, Entity>, temporal_axes: &QueryTemporalAxes, - include_drafts: bool, include_count: bool, ) -> Result> { - let mut compiler = SelectCompiler::new(Some(temporal_axes), include_drafts); + let mut compiler = SelectCompiler::new(Some(temporal_axes), INCLUDE_DRAFTS); compiler .add_filter(policy_filter) .change_context(QueryError)?; @@ -779,9 +772,8 @@ where &self, filters: [Option<&'f Filter<'f, Entity>>; 4], temporal_axes: &QueryTemporalAxes, - include_drafts: bool, ) -> Result> { - let mut compiler = SelectCompiler::new(Some(temporal_axes), include_drafts); + let mut compiler = SelectCompiler::new(Some(temporal_axes), INCLUDE_DRAFTS); for filter in filters.into_iter().flatten() { compiler.add_filter(filter).change_context(QueryError)?; } @@ -825,8 +817,8 @@ where } } -/// The scope shared by the summary and the page query: webs, archival, and -/// the universe's type exclusions. +/// The scope shared by the summary and the page query: webs, excluded types, +/// and archival. fn scope_filter<'f>(params: &'f QueryEntitiesTableParams) -> Filter<'f, Entity> { let webs_in = |webs: &'f [WebId]| { Filter::In( @@ -852,29 +844,31 @@ fn scope_filter<'f>(params: &'f QueryEntitiesTableParams) -> Filter<'f, Entity> } } - if let EntityTableTypeScope::Exclude { - entity_type_base_urls, - } = ¶ms.filter.types - { - // Excluding here rather than in the page's type clause drops the - // excluded entities from the summary and the count as well, and it - // catches multi-type entities that also carry a universe type. - clauses.extend(entity_type_base_urls.iter().map(|base_url| { - Filter::NotEqual( - FilterExpression::Path { - path: EntityQueryPath::EntityTypeEdge { - edge_kind: SharedEdgeKind::IsOfType, - path: EntityTypeQueryPath::BaseUrl, - inheritance_depth: None, + // Excluding here rather than in the page's type clause drops the excluded + // entities from the summary and the count as well, catches multi-type + // entities that also carry a selected type, and keeps the exclusions in + // force when the request narrows to a selection. + clauses.extend( + params + .filter + .excluded_type_base_urls + .iter() + .map(|base_url| { + Filter::NotEqual( + FilterExpression::Path { + path: EntityQueryPath::EntityTypeEdge { + edge_kind: SharedEdgeKind::IsOfType, + path: EntityTypeQueryPath::BaseUrl, + inheritance_depth: None, + }, }, - }, - FilterExpression::Parameter { - parameter: Parameter::Text(base_url.to_string().into()), - convert: None, - }, - ) - })); - } + FilterExpression::Parameter { + parameter: Parameter::Text(base_url.to_string().into()), + convert: None, + }, + ) + }), + ); if !params.filter.include_archived { clauses.push(Filter::NotEqual( @@ -1034,9 +1028,9 @@ mod tests { webs: webs.map_or_else(EntityTableWebScope::default, |webs| { EntityTableWebScope::Include { webs } }), - types: EntityTableTypeScope::default(), + entity_type_ids: None, + excluded_type_base_urls: Vec::new(), include_archived: false, - include_drafts: false, property_filters: Vec::new(), }, cursor: None, @@ -1066,11 +1060,10 @@ mod tests { key: EntityTableSortKey::Label, ordering: hash_graph_store::query::Ordering::Ascending, }, - include_drafts: false, position: EntityQueryCursor { values: vec![ - CursorField::Uuid(Uuid::nil()), CursorField::String("cursor value".into()), + CursorField::Uuid(Uuid::nil()), ], }, }; @@ -1087,7 +1080,6 @@ mod tests { ); assert_eq!(decoded.sort, cursor.sort, "the token pins the sort"); - assert!(!decoded.include_drafts); } // The NOT clauses are compiled next to the derived universe's GIN-friendly @@ -1101,17 +1093,15 @@ mod tests { params.filter.webs = EntityTableWebScope::Exclude { webs: vec![WebId::new(Uuid::nil())], }; - params.filter.types = EntityTableTypeScope::Exclude { - entity_type_base_urls: vec![ - BaseUrl::new("https://example.com/types/entity-type/noise/".to_owned()) - .expect("the URL should be a valid base URL"), - ], - }; + params.filter.excluded_type_base_urls = vec![ + BaseUrl::new("https://example.com/types/entity-type/noise/".to_owned()) + .expect("the URL should be a valid base URL"), + ]; params.filter.include_archived = true; let scope = scope_filter(¶ms); - let mut compiler = SelectCompiler::::new(Some(&temporal_axes), false); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), INCLUDE_DRAFTS); compiler .add_filter(&scope) .expect("the scope filter should compile"); @@ -1196,7 +1186,7 @@ mod tests { property_filters_filter(&property_filters).expect("the filters should build a filter"); let temporal_axes = QueryTemporalAxesUnresolved::all().resolve(); - let mut compiler = SelectCompiler::::new(Some(&temporal_axes), false); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), INCLUDE_DRAFTS); compiler .add_filter(&filter) .expect("the property filter should compile"); @@ -1233,27 +1223,23 @@ mod tests { #[test] fn sort_keys_compile_to_their_order_by_columns() { let order_by_lines = [ - (EntityTableSortKey::CreatedAtDecisionTime, false), - (EntityTableSortKey::EditionCreatedAtDecisionTime, false), - (EntityTableSortKey::Label, false), - (EntityTableSortKey::TypeTitle, false), - (EntityTableSortKey::Archived, false), - (EntityTableSortKey::CreatedAtDecisionTime, true), + EntityTableSortKey::CreatedAtDecisionTime, + EntityTableSortKey::EditionCreatedAtDecisionTime, + EntityTableSortKey::Label, + EntityTableSortKey::TypeTitle, + EntityTableSortKey::Archived, ] - .map(|(key, include_drafts)| { + .map(|key| { let temporal_axes = QueryTemporalAxesUnresolved::all().resolve(); - let mut compiler = SelectCompiler::::new(Some(&temporal_axes), include_drafts); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), INCLUDE_DRAFTS); compiler.set_limit(10); compiler.set_statement_shape(StatementShape::KeysFirst); let sorting = EntityQuerySorting { - paths: sorting_records( - EntityTableSorting { - key, - ordering: hash_graph_store::query::Ordering::Descending, - }, - include_drafts, - ), + paths: sorting_records(EntityTableSorting { + key, + ordering: hash_graph_store::query::Ordering::Descending, + }), cursor: None, }; let cursor_parameters = sorting.encode().expect("the sorting should encode"); @@ -1281,7 +1267,6 @@ mod tests { r#"("labels")[1] DESC, "entity_uuid" DESC"#, r#"("type_titles")[1] DESC, "entity_uuid" DESC"#, r#""archived" DESC, "entity_uuid" DESC"#, - r#""created_at_decision_time" DESC, "entity_uuid" DESC, "draft_id" DESC"#, ] ); } @@ -1298,7 +1283,7 @@ mod tests { .expect("the URL should be a valid versioned URL")]); let row_paths = RowPaths::new(); - let mut compiler = SelectCompiler::::new(Some(&temporal_axes), false); + let mut compiler = SelectCompiler::::new(Some(&temporal_axes), INCLUDE_DRAFTS); compiler .add_filter(&scope) .expect("the scope filter should compile"); @@ -1309,7 +1294,7 @@ mod tests { compiler.set_statement_shape(StatementShape::KeysFirst); let sorting = EntityQuerySorting { - paths: sorting_records(params.sort, params.filter.include_drafts), + paths: sorting_records(params.sort), cursor: None, }; let cursor_parameters = sorting.encode().expect("the sorting should encode"); diff --git a/libs/@local/graph/sdk/typescript/src/entity.ts b/libs/@local/graph/sdk/typescript/src/entity.ts index a644aa3142b..8748b1e05c7 100644 --- a/libs/@local/graph/sdk/typescript/src/entity.ts +++ b/libs/@local/graph/sdk/typescript/src/entity.ts @@ -337,20 +337,21 @@ export type EntityTableWebScope = | { type: "include"; webs: WebId[] } | { type: "exclude"; webs: WebId[] }; -export type EntityTableTypeScope = - | { type: "include"; entityTypeIds: VersionedUrl[] } - | { type: "exclude"; entityTypeBaseUrls: BaseUrl[] }; - export type QueryEntitiesTableParams = DistributiveOmit< QueryEntitiesTableParamsGraphApi, "filter" | "conversions" > & { filter: Omit< EntityTableFilterGraphApi, - "types" | "webs" | "propertyFilters" + "webs" | "entityTypeIds" | "excludedTypeBaseUrls" | "propertyFilters" > & { webs?: EntityTableWebScope; - types?: EntityTableTypeScope; + /** + * The types to narrow to. Left out for no narrowing, where an empty list + * matches no rows. + */ + entityTypeIds?: VersionedUrl[] | null; + excludedTypeBaseUrls?: BaseUrl[]; propertyFilters?: EntityTablePropertyFilter[]; }; conversions?: ConversionRequest[]; diff --git a/libs/@local/graph/store/src/entity/mod.rs b/libs/@local/graph/store/src/entity/mod.rs index a10f01b06bc..a73bdd05431 100644 --- a/libs/@local/graph/store/src/entity/mod.rs +++ b/libs/@local/graph/store/src/entity/mod.rs @@ -17,8 +17,8 @@ pub use self::{ table::{ EntityTableCursor, EntityTableFilter, EntityTableLinkEndpoint, EntityTablePropertyFilter, EntityTablePropertyValue, EntityTableRow, EntityTableSortKey, EntityTableSorting, - EntityTableSummary, EntityTableTypeScope, EntityTableWebScope, QueryEntitiesTableParams, - QueryEntitiesTableResponse, + EntityTableSummary, EntityTableWebScope, QueryEntitiesTableParams, + QueryEntitiesTableResponse, TYPE_UNIVERSE_LIMIT, }, validation_report::{ EmptyEntityTypes, EntityRetrieval, EntityTypeRetrieval, EntityTypesError, diff --git a/libs/@local/graph/store/src/entity/store.rs b/libs/@local/graph/store/src/entity/store.rs index d696721c528..f268100c866 100644 --- a/libs/@local/graph/store/src/entity/store.rs +++ b/libs/@local/graph/store/src/entity/store.rs @@ -877,10 +877,10 @@ pub trait EntityStore { /// Queries one page of the entities table. /// - /// Runs the summary and the page read in one transaction: when the - /// filter's [`types`] scope is [`Exclude`], the visible-type universe is - /// derived from the summary and applied to the page query. Follow-up - /// pages continue on the first page's database state through the + /// Runs the summary and the page read in one transaction: without a type + /// selection in [`entity_type_ids`], the visible-type universe is derived + /// from the summary and applied to the page query. Follow-up pages + /// continue on the first page's database state through the /// [`EntityTableCursor`]. /// /// # Errors @@ -888,8 +888,7 @@ pub trait EntityStore { /// - if the cursor position cannot be encoded into query parameters /// - if the request to the database fails /// - /// [`types`]: crate::entity::EntityTableFilter::types - /// [`Exclude`]: crate::entity::EntityTableTypeScope::Exclude + /// [`entity_type_ids`]: crate::entity::EntityTableFilter::entity_type_ids /// [`EntityTableCursor`]: crate::entity::EntityTableCursor fn query_entities_table( &mut self, diff --git a/libs/@local/graph/store/src/entity/table.rs b/libs/@local/graph/store/src/entity/table.rs index 75398a5d75b..9e1b94ccb3a 100644 --- a/libs/@local/graph/store/src/entity/table.rs +++ b/libs/@local/graph/store/src/entity/table.rs @@ -4,6 +4,10 @@ //! materialized columns instead of a subgraph, with the summary and the page //! read in one transaction. Follow-up pages are pinned to the first page's //! database state through the [`EntityTableCursor`]. +//! +//! Draft entities are never rows. Excluding them keeps every hydration join — +//! including the ones reaching a link row's endpoints — matched to at most one +//! edition per key, which is what lets the table read its rows in one pass. use std::collections::HashMap; @@ -178,46 +182,6 @@ impl Default for EntityTableWebScope { } } -/// Which entity types the table draws rows from. -/// -/// [`Include`] selects versioned type ids, matching the versioned selections -/// a filter UI works with. [`Exclude`] cuts by base URL on purpose: an -/// exclusion is meant to hide a type regardless of which version an entity -/// carries. The default is [`Exclude`] with an empty list: the whole -/// visible-type universe. -/// -/// [`Include`]: Self::Include -/// [`Exclude`]: Self::Exclude -#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum EntityTableTypeScope { - /// Only entities carrying one of the listed types. - Include { - // utoipa does not read `rename_all_fields`, so the fields carry their - // renames themselves to keep the spec aligned with the wire. - #[serde(rename = "entityTypeIds")] - entity_type_ids: Vec, - }, - /// The scope's visible-type universe, derived server-side from the - /// summary, except the types under the listed base URLs. - /// - /// Entities carrying an excluded type are left out entirely — of the - /// rows, the count, and the type summary alike. - Exclude { - #[serde(rename = "entityTypeBaseUrls")] - entity_type_base_urls: Vec, - }, -} - -impl Default for EntityTableTypeScope { - fn default() -> Self { - Self::Exclude { - entity_type_base_urls: Vec::new(), - } - } -} - /// The scope of the entities table. #[derive(Debug, Default, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] @@ -225,12 +189,25 @@ impl Default for EntityTableTypeScope { pub struct EntityTableFilter { #[serde(default)] pub webs: EntityTableWebScope, + /// The types the rows are narrowed to, as the versioned selections a + /// filter UI works with. + /// + /// Left out for no narrowing at all, where the table spans every type its + /// scope holds. An empty list narrows to nothing and matches no rows. #[serde(default)] - pub types: EntityTableTypeScope, + pub entity_type_ids: Option>, + /// Types the table never shows, whatever + /// [`entity_type_ids`](Self::entity_type_ids) selects. + /// + /// Entities carrying one of these types are left out entirely — of the + /// rows, the count, and the type summary alike — so a selection cannot + /// bring them back. Base URLs rather than versioned ids on purpose: an + /// exclusion is meant to hide a type regardless of which version an + /// entity carries, and it covers inherited types as well as direct ones. #[serde(default)] - pub include_archived: bool, + pub excluded_type_base_urls: Vec, #[serde(default)] - pub include_drafts: bool, + pub include_archived: bool, /// Conditions on property columns, all of which a row has to satisfy. #[serde(default)] pub property_filters: Vec, @@ -240,11 +217,10 @@ pub struct EntityTableFilter { /// /// It carries everything that determines the keyset's shape and the page /// sequence's database state — the snapshot instants, the type universe, the -/// sort, the draft visibility, and the keyset position. A continuation reads -/// all of these from the token and ignores the request's own sort and draft -/// settings, so a re-sent request cannot drift from the sequence the token -/// was handed out for. On the wire it is a Base64-encoded token the client -/// treats as opaque. +/// sort, and the keyset position. A continuation reads all of these from the +/// token and ignores the request's own sort, so a re-sent request cannot +/// drift from the sequence the token was handed out for. On the wire it is a +/// Base64-encoded token the client treats as opaque. /// /// The snapshot is two bare instants rather than temporal axes: the table /// only ever reads a current-instant snapshot, and a timestamp cannot express @@ -255,15 +231,22 @@ pub struct EntityTableCursor { pub transaction_time: Timestamp, /// The decision-time instant the sequence reads at. pub decision_time: Timestamp, - /// The type universe derived from the first page's summary. `None` when - /// the page ran on an explicit type filter, which the client re-sends - /// instead. + /// The type universe derived from the first page's summary. [`None`] when + /// the page ran on a type selection, which the client re-sends instead, or + /// when the universe exceeded [`TYPE_UNIVERSE_LIMIT`] and was dropped. pub type_universe: Option>, pub sort: EntityTableSorting, - pub include_drafts: bool, pub position: EntityQueryCursor<'static>, } +/// How many types the derived universe may carry before it is dropped. +/// +/// The universe is a planner aid — the scope filter alone decides which rows +/// match — so dropping it past a limit only costs the include clause's +/// estimability. Keeping it unbounded would grow both the cursor token and the +/// page statement's parameter list with every type a web accumulates. +pub const TYPE_UNIVERSE_LIMIT: usize = 512; + impl Serialize for EntityTableCursor { fn serialize(&self, serializer: S) -> Result where @@ -276,7 +259,6 @@ impl Serialize for EntityTableCursor { decision_time: Timestamp, type_universe: &'a Option>, sort: &'a EntityTableSorting, - include_drafts: bool, position: &'a EntityQueryCursor<'static>, } @@ -285,7 +267,6 @@ impl Serialize for EntityTableCursor { decision_time, type_universe, sort, - include_drafts, position, } = self; @@ -294,7 +275,6 @@ impl Serialize for EntityTableCursor { decision_time: *decision_time, type_universe, sort, - include_drafts: *include_drafts, position, }) .map_err(ser::Error::custom)?; @@ -315,7 +295,6 @@ impl<'de> Deserialize<'de> for EntityTableCursor { decision_time: Timestamp, type_universe: Option>, sort: EntityTableSorting, - include_drafts: bool, #[serde(borrow)] position: EntityQueryCursor<'a>, } @@ -329,16 +308,23 @@ impl<'de> Deserialize<'de> for EntityTableCursor { decision_time, type_universe, sort, - include_drafts, position, } = serde_json::from_slice(&bytes).map_err(de::Error::custom)?; + // The keyset's sort key plus the uuid tiebreaker — a position of any + // other arity would compile into a silently weakened keyset, where + // `EntityQuerySorting::compile` zips the two and truncates. + if position.values.len() != 2 { + return Err(de::Error::custom( + "the cursor position should carry the sort key and the uuid tiebreaker", + )); + } + Ok(Self { transaction_time, decision_time, type_universe, sort, - include_drafts, position: position.into_owned(), }) } @@ -510,13 +496,57 @@ mod tests { .expect("the URL should be a valid versioned URL"), ]), sort: EntityTableSorting::default(), - include_drafts: false, position: EntityQueryCursor { - values: vec![CursorField::String("position".into())], + values: vec![ + CursorField::String("position".into()), + CursorField::Uuid(Uuid::nil()), + ], }, } } + #[test] + fn a_cursor_position_of_the_wrong_arity_is_rejected() { + let token = |values: Vec>| { + let mut payload = serde_json::to_value(sample_cursor()) + .and_then(|token| { + serde_json::from_slice::( + &base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(token.as_str().expect("the token should be a string")) + .expect("the token should be base64"), + ) + }) + .expect("the cursor should round-trip"); + payload["position"] = serde_json::to_value(EntityQueryCursor { values }) + .expect("the position should serialize"); + json!( + base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_vec(&payload).expect("the payload should serialize"),) + ) + }; + + for (values, case) in [ + (vec![], "an empty position leaves no keyset condition"), + ( + vec![CursorField::String("position".into())], + "a short position drops the uuid tiebreaker", + ), + ( + vec![ + CursorField::String("position".into()), + CursorField::Uuid(Uuid::nil()), + CursorField::Uuid(Uuid::nil()), + ], + "a long position carries a column the sort does not", + ), + ] { + assert!( + serde_json::from_value::(token(values)).is_err(), + "{case} must be rejected at the serde boundary", + ); + } + } + #[test] fn garbage_cursor_tokens_are_rejected() { for (token, case) in [ @@ -556,23 +586,19 @@ mod tests { assert_eq!(params.sort, EntityTableSorting::default()); assert!(!params.include_summary); assert_eq!(params.filter.webs, EntityTableWebScope::default()); - assert_eq!(params.filter.types, EntityTableTypeScope::default()); + assert_eq!(params.filter.entity_type_ids, None); assert!(!params.filter.include_archived); - assert!(!params.filter.include_drafts); + assert!(params.filter.excluded_type_base_urls.is_empty()); } #[test] - fn scopes_deserialize_from_their_tags() { + fn the_web_scope_deserializes_from_its_tag() { let mut json = params_json(); json["filter"] = json!({ "webs": { "type": "exclude", "webs": ["00000000-0000-0000-0000-000000000000"], }, - "types": { - "type": "exclude", - "entityTypeBaseUrls": ["https://example.com/types/entity-type/noise/"], - }, }); let params: QueryEntitiesTableParams = @@ -582,27 +608,58 @@ mod tests { params.filter.webs, EntityTableWebScope::Exclude { webs } if webs.len() == 1, )); - assert!(matches!( - params.filter.types, - EntityTableTypeScope::Exclude { entity_type_base_urls } - if entity_type_base_urls.len() == 1, - )); + } + #[test] + fn a_type_selection_is_told_apart_from_no_selection() { + let selection = |value: serde_json::Value| { + let mut json = params_json(); + json["filter"] = json!({ "entityTypeIds": value }); + serde_json::from_value::(json) + .expect("the params should deserialize") + .filter + .entity_type_ids + }; + + assert_eq!( + selection(json!(null)), + None, + "an absent selection should narrow to nothing at all", + ); + assert_eq!( + selection(json!([])), + Some(Vec::new()), + "an empty selection should stay distinct from an absent one", + ); + assert_eq!( + selection(json!(["https://example.com/types/entity-type/person/v/1"])).as_deref(), + Some( + [ + VersionedUrl::from_str("https://example.com/types/entity-type/person/v/1") + .expect("the URL should be a valid versioned URL"), + ] + .as_slice() + ), + ); + } + + #[test] + fn a_selection_keeps_the_exclusions() { let mut json = params_json(); json["filter"] = json!({ - "types": { - "type": "include", - "entityTypeIds": ["https://example.com/types/entity-type/person/v/1"], - }, + "entityTypeIds": ["https://example.com/types/entity-type/person/v/1"], + "excludedTypeBaseUrls": ["https://example.com/types/entity-type/noise/"], }); let params: QueryEntitiesTableParams = serde_json::from_value(json).expect("the params should deserialize"); - assert!(matches!( - params.filter.types, - EntityTableTypeScope::Include { entity_type_ids } if entity_type_ids.len() == 1, - )); + assert!(params.filter.entity_type_ids.is_some()); + assert_eq!( + params.filter.excluded_type_base_urls.len(), + 1, + "a selection should not drop the exclusions", + ); } #[test] diff --git a/tests/graph/integration/postgres/email_filter_protection.rs b/tests/graph/integration/postgres/email_filter_protection.rs index f989d10f7bc..d0ee6651f9a 100644 --- a/tests/graph/integration/postgres/email_filter_protection.rs +++ b/tests/graph/integration/postgres/email_filter_protection.rs @@ -21,8 +21,8 @@ use hash_graph_store::{ entity::{ CreateEntityParams, EntityQueryPath, EntityQuerySorting, EntityQuerySortingRecord, EntityStore as _, EntityTableFilter, EntityTablePropertyFilter, EntityTablePropertyValue, - EntityTableSorting, EntityTableTypeScope, EntityTableWebScope, QueryEntitiesParams, - QueryEntitiesTableParams, QueryEntitySubgraphParams, SummarizeEntitiesParams, + EntityTableSorting, EntityTableWebScope, QueryEntitiesParams, QueryEntitiesTableParams, + QueryEntitySubgraphParams, SummarizeEntitiesParams, }, entity_type::EntityTypeQueryPath, filter::{ @@ -3237,9 +3237,9 @@ fn table_params_with(property_filters: Vec) -> QueryE QueryEntitiesTableParams { filter: EntityTableFilter { webs: EntityTableWebScope::default(), - types: EntityTableTypeScope::default(), + entity_type_ids: None, + excluded_type_base_urls: Vec::new(), include_archived: false, - include_drafts: false, property_filters, }, cursor: None, diff --git a/tests/graph/integration/postgres/table.rs b/tests/graph/integration/postgres/table.rs index 42535e19cf9..a49a2632dfb 100644 --- a/tests/graph/integration/postgres/table.rs +++ b/tests/graph/integration/postgres/table.rs @@ -392,9 +392,9 @@ fn page_params(limit: usize) -> QueryEntitiesTableParams { QueryEntitiesTableParams { filter: EntityTableFilter { webs: hash_graph_store::entity::EntityTableWebScope::default(), - types: hash_graph_store::entity::EntityTableTypeScope::default(), + entity_type_ids: None, + excluded_type_base_urls: Vec::new(), include_archived: false, - include_drafts: false, property_filters: Vec::new(), }, cursor: None, @@ -412,16 +412,6 @@ async fn collect_all_pages( api: &mut DatabaseApi<'_>, limit: usize, sort: EntityTableSorting, -) -> Vec { - collect_all_pages_with(api, limit, sort, false).await -} - -/// [`collect_all_pages`] with draft visibility. -async fn collect_all_pages_with( - api: &mut DatabaseApi<'_>, - limit: usize, - sort: EntityTableSorting, - include_drafts: bool, ) -> Vec { let actor_id = api.account_id; let mut cursor = None; @@ -431,7 +421,6 @@ async fn collect_all_pages_with( loop { let mut params = page_params(limit); params.sort = sort; - params.filter.include_drafts = include_drafts; params.cursor = Option::take(&mut cursor); let response = api @@ -459,7 +448,7 @@ async fn collect_all_pages_with( break; } let Some(new_cursor) = response.cursor else { - break; + panic!("a page filling the limit should hand out a continuation"); }; cursor = Some(new_cursor); } @@ -541,9 +530,7 @@ async fn explicit_type_filter_scopes_the_table() { let actor_id = api.account_id; let mut params = page_params(10); - params.filter.types = hash_graph_store::entity::EntityTableTypeScope::Include { - entity_type_ids: vec![person_entity_type()], - }; + params.filter.entity_type_ids = Some(vec![person_entity_type()]); let response = api .query_entities_table(actor_id, params) @@ -827,9 +814,7 @@ async fn type_filter_narrows_the_count_but_not_the_type_summary() { let mut params = page_params(10); params.include_summary = true; - params.filter.types = hash_graph_store::entity::EntityTableTypeScope::Include { - entity_type_ids: vec![person_entity_type()], - }; + params.filter.entity_type_ids = Some(vec![person_entity_type()]); let response = api .query_entities_table(actor_id, params) @@ -917,9 +902,7 @@ async fn excluded_type_base_urls_leave_the_universe_and_the_summary() { let mut params = page_params(10); params.include_summary = true; - params.filter.types = hash_graph_store::entity::EntityTableTypeScope::Exclude { - entity_type_base_urls: vec![page_entity_type().base_url], - }; + params.filter.excluded_type_base_urls = vec![page_entity_type().base_url]; let response = api .query_entities_table(actor_id, params) @@ -1053,14 +1036,35 @@ async fn link_endpoints_hide_entities_the_actor_cannot_view() { .await .expect("the forbid policy should be inserted"); - let rows = collect_all_pages(&mut api, 10, EntityTableSorting::default()).await; + let mut params = page_params(10); + params.include_summary = true; + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); assert!( - rows.iter().all(|row| row.entity_id != hidden_id), + response.rows.iter().all(|row| row.entity_id != hidden_id), "the forbidden entity should not appear as its own row", ); - let link_row = rows + // The summary shares the page's policy filter, so neither the count nor + // the pills may reveal an entity the actor cannot view. + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!( + summary.count, 6, + "the count should leave out the forbidden entity", + ); + assert_eq!( + summary.entity_type_ids[&person_entity_type()], + 3, + "the person pill should leave out the forbidden entity", + ); + + let link_row = response + .rows .iter() .find(|row| row.source_entity.is_some() || row.target_entity.is_some()) .expect("the link row should be visible"); @@ -1123,7 +1127,7 @@ async fn included_webs_scope_the_page_and_empty_include_matches_nothing() { } #[tokio::test] -async fn drafts_are_hidden_unless_requested() { +async fn drafts_are_never_rows() { let mut database = DatabaseTestWrapper::new().await; let mut api = insert(&mut database).await; let actor_id = api.account_id; @@ -1156,27 +1160,36 @@ async fn drafts_are_hidden_unless_requested() { .await .expect("the draft should be created"); - let rows = collect_all_pages(&mut api, 10, EntityTableSorting::default()).await; - assert_eq!(rows.len(), 5, "drafts stay hidden by default"); + let mut params = page_params(10); + params.include_summary = true; + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); - // Paging with a small limit exercises the draft-aware keyset, whose - // cursor carries the draft id as a third sort column. - let rows = collect_all_pages_with(&mut api, 2, EntityTableSorting::default(), true).await; - assert_eq!(rows.len(), 6, "the draft shows up when requested"); + assert_eq!(response.rows.len(), 5, "the draft is not a row"); + assert!( + response + .rows + .iter() + .all(|row| row.entity_id.draft_id.is_none()), + ); assert_eq!( - rows.iter() - .filter(|row| row.entity_id.draft_id.is_some()) - .count(), - 1, + response + .summary + .expect("the summary should be present when requested") + .count, + 5, + "the draft is not counted either", ); -} -#[tokio::test] -async fn multi_type_entities_count_once_and_pill_under_each_type() { - let mut database = DatabaseTestWrapper::new().await; - let mut api = insert(&mut database).await; - let actor_id = api.account_id; + // Paging exercises the same exclusion on the keyset. + let rows = collect_all_pages(&mut api, 2, EntityTableSorting::default()).await; + assert_eq!(rows.len(), 5); +} +/// Creates an entity carrying both the person and the page type. +async fn create_person_and_page(api: &mut DatabaseApi<'_>) { let mut properties: serde_json::Map = serde_json::from_str(entity::PERSON_ALICE_V1).expect("the entity fixture should parse"); properties.extend( @@ -1185,7 +1198,7 @@ async fn multi_type_entities_count_once_and_pill_under_each_type() { ); api.create_entity( - actor_id, + api.account_id, CreateEntityParams { web_id: WebId::new(api.account_id), entity_uuid: None, @@ -1211,6 +1224,15 @@ async fn multi_type_entities_count_once_and_pill_under_each_type() { ) .await .expect("the multi-type entity should be created"); +} + +#[tokio::test] +async fn multi_type_entities_count_once_and_pill_under_each_type() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + create_person_and_page(&mut api).await; let mut params = page_params(10); params.include_summary = true; @@ -1252,3 +1274,110 @@ async fn multi_type_entities_count_once_and_pill_under_each_type() { "the titles should stay parallel to the two types", ); } + +#[tokio::test] +async fn a_selection_cannot_bring_back_an_excluded_type() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + create_person_and_page(&mut api).await; + + let mut params = page_params(10); + params.include_summary = true; + params.filter.entity_type_ids = Some(vec![person_entity_type()]); + params.filter.excluded_type_base_urls = vec![page_entity_type().base_url]; + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + // The multi-type entity carries the selected person type, but its page + // type is excluded — the exclusion wins. + assert_eq!(response.rows.len(), 3, "only the pure persons should match"); + assert!( + response + .rows + .iter() + .all(|row| row.entity_type_ids == vec![person_entity_type()]), + ); + + let summary = response + .summary + .expect("the summary should be present when requested"); + assert_eq!(summary.count, 3); + assert_eq!( + summary.entity_type_ids, + HashMap::from([(person_entity_type(), 3)]), + "the excluded type should stay out of the pills under a selection", + ); +} + +#[tokio::test] +async fn a_continuation_reads_the_first_page_s_snapshot() { + let mut database = DatabaseTestWrapper::new().await; + let mut api = insert(&mut database).await; + let actor_id = api.account_id; + + // Oldest first, so anything created after page 1 would sort into the + // continuation's range if the snapshot were re-taken per page. + let oldest_first = EntityTableSorting { + key: EntityTableSortKey::CreatedAtDecisionTime, + ordering: Ordering::Ascending, + }; + + let mut params = page_params(2); + params.sort = oldest_first; + let first = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + let mut cursor = Some( + first + .cursor + .expect("a full page should hand out a continuation"), + ); + + // A new entity of a type the first page never saw: the pinned snapshot + // keeps it out of the rows, and the pinned universe keeps its type out of + // the page query's type clause. + create_person_and_page(&mut api).await; + + let mut continued = first.rows.len(); + while let Some(current) = Option::take(&mut cursor) { + let mut params = page_params(2); + params.sort = oldest_first; + params.cursor = Some(current); + + let response = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + + assert!( + response + .rows + .iter() + .all(|row| row.entity_type_ids.len() == 1), + "the entity created after the first page should stay out of the sequence", + ); + continued += response.rows.len(); + cursor = response.cursor; + } + + assert_eq!( + continued, 5, + "the sequence should carry the rows the first page's snapshot held", + ); + + // The same query without a cursor sees the new entity, so the assertion + // above is about the pinning rather than about the entity's visibility. + let mut params = page_params(10); + params.sort = oldest_first; + let fresh = api + .query_entities_table(actor_id, params) + .await + .expect("the table query should succeed"); + assert_eq!(fresh.rows.len(), 6); +} From 8cfdd9fc2f176a81999bcb4ce54dc5a2df7bcada Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Sat, 25 Jul 2026 14:53:03 +0200 Subject: [PATCH 05/18] BE-705: Derive the type universe on first pages only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A continuation whose sequence had no universe pinned — because it exceeded the limit — read that as "not derived yet" and paid the full summary scan again, on every page, only to drop the universe again each time. The token already says what its sequence runs on, so only a first page derives one. The cap moves into a function of its own and gets a test: the boundary and the deterministic order it needs for the cursor were unpinned. --- .../store/postgres/knowledge/entity/table.rs | 78 +++++++++++++++---- libs/@local/graph/store/src/entity/table.rs | 10 ++- 2 files changed, 69 insertions(+), 19 deletions(-) diff --git a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs index 657b4356757..8a90616917f 100644 --- a/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs +++ b/libs/@local/graph/postgres-store/src/store/postgres/knowledge/entity/table.rs @@ -11,7 +11,7 @@ //! first page's database state through the cursor. use alloc::borrow::Cow; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use error_stack::{Report, ResultExt as _}; use futures::TryStreamExt as _; @@ -297,6 +297,24 @@ struct LinkEndpointEditions { target: Option, } +/// The type universe to pin for a page sequence, or [`None`] when the summary +/// held more types than [`TYPE_UNIVERSE_LIMIT`]. +/// +/// Past the limit the include clause stops being worth its cost in bind +/// parameters and cursor bytes, and the scope filter alone decides the rows +/// either way. +fn pin_universe(type_ids: &HashMap) -> Option> { + if type_ids.len() > TYPE_UNIVERSE_LIMIT { + return None; + } + + let mut universe = type_ids.keys().cloned().collect::>(); + // The universe is sent as query parameters and travels inside the cursor — + // keep its order deterministic. + universe.sort_unstable(); + Some(universe) +} + /// The sort key plus the uuid tiebreaker — the keyset the cursor's position /// carries. Drafts are never rows, so the entity uuid alone breaks ties. fn sorting_records(sort: EntityTableSorting) -> Vec> { @@ -406,7 +424,11 @@ where }); } - let needs_universe = type_selection.is_none() && type_universe.is_none(); + // Only a first page derives a universe: a continuation's token already + // says what its sequence runs on, including that the universe was too + // large to pin — where deriving it again would pay the summary scan + // per page only to drop it again. + let needs_universe = params.cursor.is_none() && type_selection.is_none(); // Without narrowing filters the page's full filters equal the scope // (the include clause is result-neutral and the excluded base URLs // live inside the scope filter), so the count can ride along in the @@ -428,20 +450,12 @@ where }; if needs_universe { type_universe = scope_summaries.as_ref().and_then(|summaries| { - let type_ids = summaries - .type_ids - .as_ref() - .expect("the type summary should always be requested"); - // Past the limit the clause stops being worth its cost in - // parameters and cursor bytes, and the scope filter alone - // decides the rows either way. - (type_ids.len() <= TYPE_UNIVERSE_LIMIT).then(|| { - let mut universe = type_ids.keys().cloned().collect::>(); - // The universe is sent as query parameters and travels - // inside the cursor — keep its order deterministic. - universe.sort_unstable(); - universe - }) + pin_universe( + summaries + .type_ids + .as_ref() + .expect("the type summary should always be requested"), + ) }); } @@ -1047,6 +1061,38 @@ mod tests { .expect("the timestamp should deserialize") } + #[test] + fn a_universe_is_pinned_in_order_up_to_the_limit() { + let universe = |count: usize| { + pin_universe( + &(0..count) + .map(|index| { + ( + VersionedUrl::from_str(&format!( + "https://example.com/types/entity-type/type-{index}/v/1" + )) + .expect("the URL should be a valid versioned URL"), + 1, + ) + }) + .collect(), + ) + }; + + let pinned = universe(TYPE_UNIVERSE_LIMIT).expect("the universe should be pinned"); + assert_eq!(pinned.len(), TYPE_UNIVERSE_LIMIT); + assert!( + pinned.is_sorted(), + "the universe travels in the cursor, so its order should be deterministic", + ); + + assert_eq!( + universe(TYPE_UNIVERSE_LIMIT + 1), + None, + "a universe past the limit should be dropped rather than pinned", + ); + } + #[test] fn cursor_token_roundtrip() { let cursor = EntityTableCursor { diff --git a/libs/@local/graph/store/src/entity/table.rs b/libs/@local/graph/store/src/entity/table.rs index 9e1b94ccb3a..72ba0d380bb 100644 --- a/libs/@local/graph/store/src/entity/table.rs +++ b/libs/@local/graph/store/src/entity/table.rs @@ -231,9 +231,13 @@ pub struct EntityTableCursor { pub transaction_time: Timestamp, /// The decision-time instant the sequence reads at. pub decision_time: Timestamp, - /// The type universe derived from the first page's summary. [`None`] when - /// the page ran on a type selection, which the client re-sends instead, or - /// when the universe exceeded [`TYPE_UNIVERSE_LIMIT`] and was dropped. + /// The type universe derived from the first page's summary, pinned for the + /// rest of the sequence. + /// + /// [`None`] means no universe is pinned — the sequence runs on a type + /// selection the client re-sends, or its universe exceeded + /// [`TYPE_UNIVERSE_LIMIT`]. Either way the following pages take it as it + /// stands rather than deriving one of their own. pub type_universe: Option>, pub sort: EntityTableSorting, pub position: EntityQueryCursor<'static>, From 880792ff8f15c7c152c202791a3ce30d7d2bb510 Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Fri, 24 Jul 2026 14:18:13 +0200 Subject: [PATCH 06/18] BE-705: Move the entities table view onto the table endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table view reads its rows through the dedicated queryEntitiesTable endpoint instead of the subgraph query, through a decoupled hook that accumulates pages, deep-merges the closed-type maps, and processes responses in an effect — Apollo's onCompleted swallows exceptions in production. Contract breaks reach Sentry and the error UI with their message, a failed follow-up page keeps the rows on screen next to a retry banner, and pins that resolve to no version surface as an error instead of an empty table. The Grid and Graph views stay on the subgraph query, whose hook goes back to serving only them. --- .../queries/knowledge/entity.queries.ts | 6 + .../hash-frontend/src/pages/entities.page.tsx | 9 +- .../src/pages/shared/entities-visualizer.tsx | 374 +++++++++++---- .../entities-table-data.ts | 26 -- .../entities-visualizer/entities-table.tsx | 67 ++- .../build-property-filter-clause.test.ts | 105 +++++ .../build-property-filter-clause.ts | 102 ++++- .../shared/use-available-types.ts | 96 +++- .../use-entities-table-data.tsx | 160 ------- .../use-entities-table-query.tsx | 427 ++++++++++++++++++ ...generate-table-data-from-endpoint-rows.ts} | 369 ++++++++------- .../use-entities-visualizer-data.tsx | 88 ++-- apps/hash-frontend/src/shared/is-archived.ts | 29 +- apps/hash-frontend/src/shared/is-of-type.ts | 9 +- .../table-header/bulk-actions-dropdown.tsx | 7 +- 15 files changed, 1285 insertions(+), 589 deletions(-) create mode 100644 apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.test.ts delete mode 100644 apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data.tsx create mode 100644 apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query.tsx rename apps/hash-frontend/src/pages/shared/entities-visualizer/{use-entities-table-data/generate-table-data-from-rows.ts => use-entities-table-query/generate-table-data-from-endpoint-rows.ts} (54%) diff --git a/apps/hash-frontend/src/graphql/queries/knowledge/entity.queries.ts b/apps/hash-frontend/src/graphql/queries/knowledge/entity.queries.ts index 4cb81293fa0..de892cc3648 100644 --- a/apps/hash-frontend/src/graphql/queries/knowledge/entity.queries.ts +++ b/apps/hash-frontend/src/graphql/queries/knowledge/entity.queries.ts @@ -39,6 +39,12 @@ export const summarizeEntitiesQuery = gql` } `; +export const queryEntitiesTableQuery = gql` + query queryEntitiesTable($request: QueryEntitiesTableParams!) { + queryEntitiesTable(request: $request) + } +`; + export const updateEntityMutation = gql` mutation updateEntity($entityUpdate: EntityUpdateDefinition!) { # This is a scalar, which has no selection. diff --git a/apps/hash-frontend/src/pages/entities.page.tsx b/apps/hash-frontend/src/pages/entities.page.tsx index 271b1e37a84..cde06be5522 100644 --- a/apps/hash-frontend/src/pages/entities.page.tsx +++ b/apps/hash-frontend/src/pages/entities.page.tsx @@ -195,6 +195,13 @@ const EntitiesPage: NextPageWithLayout = () => { return {}; }, [router]); + // A stable identity — the column list feeds the table query hook's + // processing effect, which a fresh array on every render would re-run. + const hideColumns = useMemo( + () => (entityTypeId ? ["entityTypes" as const] : []), + [entityTypeId], + ); + const { latestEntityTypes } = useLatestEntityTypesOptional({ includeArchived: true, }); @@ -345,7 +352,7 @@ const EntitiesPage: NextPageWithLayout = () => { diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx index 5ca67914396..c06d1178bed 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx @@ -5,6 +5,7 @@ import { extractBaseUrl } from "@blockprotocol/type-system"; import { LoadingSpinner } from "@hashintel/design-system"; import { typedEntries } from "@local/advanced-types/typed-entries"; import { + type EntityTableSummary, getClosedMultiEntityTypeFromMap, type HashEntity, } from "@local/hash-graph-sdk/entity"; @@ -29,7 +30,11 @@ import { visualizerHeaderHeight, } from "./entities-visualizer/header"; import { createDefaultFilterState } from "./entities-visualizer/shared/filter-state"; -import { useAvailableTypes } from "./entities-visualizer/shared/use-available-types"; +import { + type SummarySource, + useAvailableTypes, +} from "./entities-visualizer/shared/use-available-types"; +import { useEntitiesTableQuery } from "./entities-visualizer/use-entities-table-query"; import { useEntitiesVisualizerData } from "./entities-visualizer/use-entities-visualizer-data"; import { EntityGraphVisualizer } from "./entity-graph-visualizer"; import { useSlideStack } from "./slide-stack"; @@ -38,6 +43,7 @@ import { TOP_CONTEXT_BAR_HEIGHT } from "./top-context-bar"; import { visualizerViewIcons } from "./visualizer-views"; import type { ColumnSort } from "../../components/grid/utils/sorting"; +import type { ArchivableEntity } from "../../shared/is-archived"; import type { EntitiesTableRow, SortableEntitiesTableColumnKey, @@ -49,6 +55,7 @@ import type { BaseUrl, ClosedMultiEntityType, EntityId, + PropertyObject, VersionedUrl, WebId, } from "@blockprotocol/type-system"; @@ -58,6 +65,7 @@ import type { EntityQuerySortingPath, EntityQuerySortingRecord, EntityQuerySortingToken, + EntityTableSorting, NullOrdering, Ordering, } from "@local/hash-graph-client"; @@ -85,6 +93,17 @@ const allFileEntityTypeBaseUrl = allFileEntityTypeOntologyIds.map( ({ entityTypeBaseUrl }) => entityTypeBaseUrl, ); +const tableSortKeyByColumnKey = { + entityLabel: "label", + lastEdited: "editionCreatedAtDecisionTime", + created: "createdAtDecisionTime", + entityTypes: "typeTitle", + archived: "archived", +} as const satisfies Record< + SortableEntitiesTableColumnKey, + EntityTableSorting["key"] +>; + const generateGraphSort = ( columnKey: SortableEntitiesTableColumnKey, direction: "asc" | "desc", @@ -176,7 +195,13 @@ export const EntitiesVisualizer: FunctionComponent<{ createDefaultFilterState(internalWebs.map(({ webId }) => webId)), ); - const [cursor, setCursor] = useState(); + const [subgraphCursor, setSubgraphCursor] = useState(); + const [tableCursor, setTableCursor] = useState(); + + const resetCursors = useCallback(() => { + setSubgraphCursor(undefined); + setTableCursor(undefined); + }, []); const [activeConversionsWithoutTitle, _setActiveConversions] = useState<{ [columnBaseUrl: BaseUrl]: VersionedUrl; } | null>(null); @@ -190,9 +215,9 @@ export const EntitiesVisualizer: FunctionComponent<{ >( (newConversionsOrUpdater) => { _setActiveConversions(newConversionsOrUpdater); - setCursor(undefined); + resetCursors(); }, - [setCursor], + [resetCursors], ); const setFilterState = useCallback( @@ -206,9 +231,9 @@ export const EntitiesVisualizer: FunctionComponent<{ ? newFilterStateOrUpdater(prev) : newFilterStateOrUpdater, ); - setCursor(undefined); + resetCursors(); }, - [setCursor], + [resetCursors], ); const [view, _setView] = useState("Table"); @@ -216,9 +241,9 @@ export const EntitiesVisualizer: FunctionComponent<{ const setView = useCallback( (newView: VisualizerView) => { _setView(newView); - setCursor(undefined); + resetCursors(); }, - [setCursor], + [resetCursors], ); const [sort, _setSort] = useState>( @@ -231,9 +256,9 @@ export const EntitiesVisualizer: FunctionComponent<{ const setSort = useCallback( (newSort: ColumnSort) => { _setSort(newSort); - setCursor(undefined); + resetCursors(); }, - [setCursor], + [resetCursors], ); const graphSort = useMemo( @@ -241,10 +266,40 @@ export const EntitiesVisualizer: FunctionComponent<{ [sort], ); + const tableSort = useMemo( + () => ({ + key: tableSortKeyByColumnKey[sort.columnKey], + ordering: sort.direction === "asc" ? "ascending" : "descending", + }), + [sort], + ); + const isTypePinned = !!entityTypeBaseUrl || !!entityTypeId; + const usesTableEndpoint = view === "Table"; + + /** + * The table endpoint's first page carries the type summary. Mirroring it + * into state lets `useAvailableTypes` (called before the table query, which + * needs its resolved pins) derive the filter chips from it without a fetch + * of its own. + */ + const [tableSummary, setTableSummary] = useState( + null, + ); + const [tableSummaryError, setTableSummaryError] = useState(); + + const summarySource = useMemo( + () => + usesTableEndpoint && !isTypePinned + ? { mode: "external", summary: tableSummary, error: tableSummaryError } + : { mode: "fetch" }, + [usesTableEndpoint, isTypePinned, tableSummary, tableSummaryError], + ); + const { availableEntityTypes, + pinnedEntityTypeIds, propertyFilterData, loading: availableTypesLoading, typeUniverse, @@ -255,22 +310,42 @@ export const EntitiesVisualizer: FunctionComponent<{ internalWebs, entityTypeBaseUrl, entityTypeIds: entityTypeId ? [entityTypeId] : undefined, + summarySource, + }); + + const conversions = useMemo( + () => + activeConversionsWithoutTitle + ? typedEntries(activeConversionsWithoutTitle).map( + ([columnBaseUrl, dataTypeId]) => ({ + path: [columnBaseUrl], + dataTypeId, + }), + ) + : undefined, + [activeConversionsWithoutTitle], + ); + + const tableQuery = useEntitiesTableQuery({ + conversions, + cursor: tableCursor, + enabled: usesTableEndpoint, + filterState, + hideArchivedColumn: !filterState.includeArchived, + hideColumns, + hasPinnedTypes: isTypePinned, + internalWebs, + limit: 500, + resolvedPinnedEntityTypeIds: pinnedEntityTypeIds, + sort: tableSort, }); const entitiesData = useEntitiesVisualizerData({ - conversions: activeConversionsWithoutTitle - ? typedEntries(activeConversionsWithoutTitle).map( - ([columnBaseUrl, dataTypeId]) => ({ - path: [columnBaseUrl], - dataTypeId, - }), - ) - : undefined, - cursor, + conversions, + cursor: subgraphCursor, entityTypeBaseUrl, entityTypeIds: entityTypeId ? [entityTypeId] : undefined, filterState, - hideColumns, internalWebs, limit: 500, sort: graphSort, @@ -279,6 +354,14 @@ export const EntitiesVisualizer: FunctionComponent<{ view, }); + useEffect(() => { + setTableSummary(tableQuery.summary); + }, [tableQuery.summary]); + + useEffect(() => { + setTableSummaryError(tableQuery.error); + }, [tableQuery.error]); + const [dataLoading, setDataLoading] = useState(entitiesData.loading); const [visualizerData, setVisualizerData] = useState(entitiesData); @@ -291,34 +374,60 @@ export const EntitiesVisualizer: FunctionComponent<{ } = visualizerData; const closedMultiEntityTypes = useMemo(() => { - if (!entities || !definitions || !closedMultiEntityTypesRootMap) { + const typesRootMap = usesTableEndpoint + ? tableQuery.closedMultiEntityTypes + : closedMultiEntityTypesRootMap; + + if (!typesRootMap) { return []; } + const rowTypeIdLists = usesTableEndpoint + ? (tableQuery.tableData?.rows.map((row) => + row.entityTypes.map((rowEntityType) => rowEntityType.entityTypeId), + ) ?? []) + : (entities?.map((entity) => entity.metadata.entityTypeIds) ?? []); + const relevantEntityTypesMap = new Map(); - for (const { metadata } of entities) { - const closedMultiEntityType = getClosedMultiEntityTypeFromMap( - closedMultiEntityTypesRootMap, - metadata.entityTypeIds, - ); + for (const [firstTypeId, ...otherTypeIds] of rowTypeIdLists) { + if (!firstTypeId) { + continue; + } - const key = metadata.entityTypeIds.toSorted().join(","); + const entityTypeIds: [VersionedUrl, ...VersionedUrl[]] = [ + firstTypeId, + ...otherTypeIds, + ]; + const key = entityTypeIds.toSorted().join(","); - relevantEntityTypesMap.set(key, closedMultiEntityType); + if (!relevantEntityTypesMap.has(key)) { + relevantEntityTypesMap.set( + key, + getClosedMultiEntityTypeFromMap(typesRootMap, entityTypeIds), + ); + } } - const relevantTypes = Array.from(relevantEntityTypesMap.values()); - - return relevantTypes; - }, [entities, definitions, closedMultiEntityTypesRootMap]); + return Array.from(relevantEntityTypesMap.values()); + }, [ + usesTableEndpoint, + tableQuery.closedMultiEntityTypes, + tableQuery.tableData, + entities, + closedMultiEntityTypesRootMap, + ]); const activeConversions = useMemo(() => { + const activeDefinitions = usesTableEndpoint + ? tableQuery.definitions + : definitions; + return activeConversionsWithoutTitle ? Object.fromEntries( typedEntries(activeConversionsWithoutTitle).map( ([columnBaseUrl, dataTypeId]) => { - const dataType = definitions?.dataTypes[dataTypeId]; + const dataType = activeDefinitions?.dataTypes[dataTypeId]; if (!dataType) { throw new Error( @@ -337,14 +446,18 @@ export const EntitiesVisualizer: FunctionComponent<{ ), ) : null; - }, [activeConversionsWithoutTitle, definitions]); + }, [ + activeConversionsWithoutTitle, + usesTableEndpoint, + tableQuery.definitions, + definitions, + ]); /** - * We don't want to clear the old table data when a new request is triggered, - * so we hold the visualizerData here rather than relying on the useEntitiesVisualizerData hook directly, - * as it will clear the data when a new request is triggered. - * - * An alternative would be to have an onComplete callback in the hook. + * The subgraph hook clears its data when a new request starts. Holding the + * last loaded state here keeps the Grid and Graph views' previous results on + * screen while the next ones load. The Table view does not need this — its + * query hook accumulates pages itself. */ useEffect(() => { setDataLoading(entitiesData.loading); @@ -542,27 +655,45 @@ export const EntitiesVisualizer: FunctionComponent<{ >([]); const nextPage = useCallback(() => { - setCursor(nextCursor ?? undefined); - }, [nextCursor]); + if (usesTableEndpoint) { + setTableCursor(tableQuery.cursor ?? undefined); + } else { + setSubgraphCursor(nextCursor ?? undefined); + } + }, [usesTableEndpoint, tableQuery.cursor, nextCursor]); - const selectedEntities = useMemo(() => { - if (view !== "Table" || selectedTableRows.length === 0 || !entities) { + const selectedEntities = useMemo(() => { + if (view !== "Table") { return []; } - const selectedEntityIds = new Set( - selectedTableRows.map(({ entityId }) => entityId), - ); - - return entities.filter((entity) => - selectedEntityIds.has(entity.metadata.recordId.entityId), - ); - }, [entities, selectedTableRows, view]); + return selectedTableRows.map((row) => ({ + metadata: { + recordId: { entityId: row.entityId }, + entityTypeIds: row.entityTypes.map( + (rowEntityType) => rowEntityType.entityTypeId, + ), + archived: row.archived, + }, + properties: Object.fromEntries( + row.applicableProperties.map((baseUrl) => [ + baseUrl, + row[baseUrl]?.value, + ]), + ) as PropertyObject, + })); + }, [selectedTableRows, view]); const handleBulkActionCompleted = useCallback(() => { - void entitiesData.refetch(); + // A rejected refetch needs no handling here — the failure comes back + // through the hook's error state. + if (usesTableEndpoint) { + tableQuery.refetch().catch(() => {}); + } else { + entitiesData.refetch().catch(() => {}); + } setSelectedTableRows([]); - }, [entitiesData]); + }, [usesTableEndpoint, tableQuery, entitiesData]); // The universe only feeds the default view — with a pinned type or an // explicit selection the main query runs on its own type clause, so a failed @@ -572,9 +703,32 @@ export const EntitiesVisualizer: FunctionComponent<{ !isTypePinned && filterState.type.selectedTypeIds === null; - const showLoading = !subgraph || !closedMultiEntityTypesRootMap; + const activeError = usesTableEndpoint + ? tableQuery.error + : visualizerData.error; + + // A failed page query with nothing to show gets the error state. With data + // on screen the stale results stay visible alongside a retry banner instead. + const queryBlocksResults = + !!activeError && (usesTableEndpoint ? !tableQuery.tableData : !subgraph); + + const blockingError = typeUniverseBlocksResults + ? typeUniverseError + : activeError; + + // Unresolvable pins only clear when the pinned types change, so offering a + // retry would be a dead button. + const errorIsRetryable = !(usesTableEndpoint && tableQuery.unresolvablePins); - const { totalResultCount } = visualizerData; + const showLoading = usesTableEndpoint + ? !tableQuery.tableData + : !subgraph || !closedMultiEntityTypesRootMap; + + const resultsLoading = usesTableEndpoint ? tableQuery.loading : dataLoading; + + const totalResultCount = usesTableEndpoint + ? tableQuery.totalResultCount + : visualizerData.totalResultCount; return ( @@ -599,7 +753,7 @@ export const EntitiesVisualizer: FunctionComponent<{ } right={ <> - + - {typeUniverseBlocksResults ? ( + {typeUniverseBlocksResults || queryBlocksResults ? ( Something went wrong loading entities. - + {blockingError ? ( + palette.gray[70] }} + > + {blockingError.message} + + ) : null} + {errorIsRetryable ? ( + + ) : null} ) : showLoading ? ( ) : ( - + <> + {activeError ? ( + + + {tableQuery.dataIsStale + ? "Something went wrong refreshing entities — showing the previous results." + : "Something went wrong loading more entities."} + + {errorIsRetryable ? ( + + ) : null} + + ) : null} + + )} ); diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table-data.ts b/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table-data.ts index d1a7f5a894b..93de5b1711f 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table-data.ts +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table-data.ts @@ -1,4 +1,3 @@ -import type { EntitiesVisualizerData } from "./use-entities-visualizer-data"; import type { ActorEntityUuid, BaseUrl, @@ -10,14 +9,9 @@ import type { WebId, } from "@blockprotocol/type-system"; import type { SizedGridColumn } from "@glideapps/glide-data-grid"; -import type { - SerializedEntity, - SerializedSubgraph, -} from "@local/hash-graph-sdk/entity"; import type { ClosedDataTypeDefinition, ClosedMultiEntityTypesDefinitions, - ClosedMultiEntityTypesRootMap, } from "@local/hash-graph-sdk/ontology"; export type EntitiesTableRowPropertyCell = { @@ -85,16 +79,6 @@ export interface EntitiesTableColumn extends SizedGridColumn { id: EntitiesTableColumnKey; } -export type GenerateEntitiesTableDataParams = { - closedMultiEntityTypesRootMap: ClosedMultiEntityTypesRootMap; - definitions: ClosedMultiEntityTypesDefinitions; - entities: SerializedEntity[]; - subgraph: SerializedSubgraph; - hasSomeLinks?: boolean; - hideColumns?: (keyof EntitiesTableRow)[]; - hideArchivedColumn?: boolean; -}; - export type EntityTypeTableFilterData = { entityTypeId: VersionedUrl; title: string; @@ -126,13 +110,3 @@ export type EntitiesTableData = { rows: EntitiesTableRow[]; visibleDataTypeIdsByPropertyBaseUrl: VisibleDataTypeIdsByPropertyBaseUrl; }; - -export type UpdateTableDataFn = ( - params: Pick< - EntitiesVisualizerData, - "definitions" | "entities" | "subgraph" - > & { - appendRows: boolean; - closedMultiEntityTypesRootMap: ClosedMultiEntityTypesRootMap; - }, -) => void; diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table.tsx index bebc08c5607..4cbf14acd7d 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table.tsx +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/entities-table.tsx @@ -54,7 +54,6 @@ import type { EntitiesTableRow, SortableEntitiesTableColumnKey, } from "./entities-table-data"; -import type { EntitiesVisualizerData } from "./use-entities-visualizer-data"; import type { ActorEntityUuid, BaseUrl, @@ -97,40 +96,38 @@ const emptyTableData: EntitiesTableData = { visibleDataTypeIdsByPropertyBaseUrl: {}, }; -export const EntitiesTable: FunctionComponent< - Pick & { - activeConversions: { - [columnBaseUrl: BaseUrl]: { - dataTypeId: VersionedUrl; - title: string; - }; - } | null; - csvFileTitle: string; - currentlyDisplayedColumnsRef: MutableRefObject; - currentlyDisplayedRowsRef: RefObject; - disableTypeClick?: boolean; - handleEntityClick: (entityId: EntityId) => void; - loading: boolean; - isViewingOnlyPages: boolean; - maxHeight: string | number; - hasMoreRowsAvailable: boolean; - loadMoreRows?: () => void; - selectedRows: EntitiesTableRow[]; - setActiveConversions: Dispatch< - SetStateAction<{ - [columnBaseUrl: BaseUrl]: VersionedUrl; - } | null> - >; - setSelectedRows: (rows: EntitiesTableRow[]) => void; - setSelectedEntityType: (params: { entityTypeId: VersionedUrl }) => void; - setShowSearch: (showSearch: boolean) => void; - showSearch: boolean; - sort: GridSort; - setSort: (sort: GridSort) => void; - tableData: EntitiesTableData | null; - totalResultCount: number | null; - } -> = ({ +export const EntitiesTable: FunctionComponent<{ + activeConversions: { + [columnBaseUrl: BaseUrl]: { + dataTypeId: VersionedUrl; + title: string; + }; + } | null; + csvFileTitle: string; + currentlyDisplayedColumnsRef: MutableRefObject; + currentlyDisplayedRowsRef: RefObject; + disableTypeClick?: boolean; + handleEntityClick: (entityId: EntityId) => void; + loading: boolean; + isViewingOnlyPages: boolean; + maxHeight: string | number; + hasMoreRowsAvailable: boolean; + loadMoreRows?: () => void; + selectedRows: EntitiesTableRow[]; + setActiveConversions: Dispatch< + SetStateAction<{ + [columnBaseUrl: BaseUrl]: VersionedUrl; + } | null> + >; + setSelectedRows: (rows: EntitiesTableRow[]) => void; + setSelectedEntityType: (params: { entityTypeId: VersionedUrl }) => void; + setShowSearch: (showSearch: boolean) => void; + showSearch: boolean; + sort: GridSort; + setSort: (sort: GridSort) => void; + tableData: EntitiesTableData | null; + totalResultCount: number | null; +}> = ({ activeConversions, csvFileTitle, currentlyDisplayedColumnsRef, diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.test.ts b/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.test.ts new file mode 100644 index 00000000000..7db84fb08ee --- /dev/null +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; + +import { + buildEndpointPropertyFilter, + buildPropertyFilterClause, +} from "./build-property-filter-clause"; + +import type { PropertyFilter } from "./property-filter"; +import type { BaseUrl } from "@blockprotocol/type-system"; + +const baseUrl = "https://example.com/types/property-type/age/" as BaseUrl; + +const propertyFilter = ( + overrides: Partial, +): PropertyFilter => ({ + id: "filter", + baseUrl, + title: "Age", + kind: "number", + operator: "equals", + value: "30", + ...overrides, +}); + +describe("buildEndpointPropertyFilter", () => { + it("builds value-less operators without a value", () => { + expect( + buildEndpointPropertyFilter( + propertyFilter({ operator: "hasAnyValue", value: undefined }), + ), + ).toEqual({ type: "hasAnyValue", property: baseUrl }); + }); + + it("coerces number-kind values into numbers", () => { + expect( + buildEndpointPropertyFilter( + propertyFilter({ operator: "greaterThan", value: "30" }), + ), + ).toEqual({ type: "greaterThan", property: baseUrl, value: 30 }); + }); + + it("keeps string-kind values untrimmed", () => { + expect( + buildEndpointPropertyFilter( + propertyFilter({ + kind: "string", + operator: "startsWith", + value: " Alice", + }), + ), + ).toEqual({ type: "startsWith", property: baseUrl, value: " Alice" }); + }); + + it("renders an incomplete filter inert", () => { + expect( + buildEndpointPropertyFilter( + propertyFilter({ operator: "equals", value: undefined }), + ), + ).toBeNull(); + expect( + buildEndpointPropertyFilter( + propertyFilter({ operator: "greaterThan", value: "not a number" }), + ), + ).toBeNull(); + }); + + it("renders a kind and operator mismatch inert instead of sending it", () => { + // A string-kind value on an ordering comparator would be rejected by the + // graph's Real-typed wire field as a whole-query error. + expect( + buildEndpointPropertyFilter( + propertyFilter({ kind: "string", operator: "greaterThan", value: "a" }), + ), + ).toBeNull(); + expect( + buildEndpointPropertyFilter( + propertyFilter({ kind: "number", operator: "startsWith", value: "3" }), + ), + ).toBeNull(); + }); + + it("agrees with the subgraph builder on which filters are inert", () => { + // `isPropertyFilterActive` answers for both query paths through the + // subgraph builder, so the two must never disagree on null-ness — a + // divergence shows an active-looking pill whose filter the table drops. + const cases: Partial[] = [ + { operator: "equals", value: "30" }, + { operator: "equals", value: undefined }, + { operator: "hasAnyValue", value: undefined }, + { operator: "greaterThan", value: "30" }, + { operator: "greaterThan", value: "not a number" }, + { kind: "string", operator: "greaterThan", value: "a" }, + { kind: "number", operator: "startsWith", value: "3" }, + { kind: "string", operator: "contains", value: " x " }, + { kind: "string", operator: "endsWith", value: "" }, + ]; + + for (const overrides of cases) { + const filter = propertyFilter(overrides); + expect(buildEndpointPropertyFilter(filter) === null).toBe( + buildPropertyFilterClause(filter) === null, + ); + } + }); +}); diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.ts b/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.ts index 3a02882e971..46880ee4e94 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.ts +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/property-filters/build-property-filter-clause.ts @@ -1,6 +1,7 @@ import type { PropertyFilter } from "./property-filter"; import type { BaseUrl } from "@blockprotocol/type-system"; import type { Filter } from "@local/hash-graph-client"; +import type { EntityTablePropertyFilter } from "@local/hash-graph-sdk/entity"; const propertyPath = (baseUrl: BaseUrl) => ["properties", baseUrl]; @@ -87,20 +88,107 @@ export const buildPropertyFilterClause = ( return { equal: [{ path }, { parameter }] }; case "notEquals": return { notEqual: [{ path }, { parameter }] }; + // The ordering comparators take numbers and the text operators take + // strings, mirroring the gates in {@link buildEndpointPropertyFilter} — + // the two builders must agree on which filters are inert, since + // {@link isPropertyFilterActive} answers for both. case "greaterThan": - return { greater: [{ path }, { parameter }] }; + return typeof parameter === "number" + ? { greater: [{ path }, { parameter }] } + : null; case "greaterThanOrEqual": - return { greaterOrEqual: [{ path }, { parameter }] }; + return typeof parameter === "number" + ? { greaterOrEqual: [{ path }, { parameter }] } + : null; case "lessThan": - return { less: [{ path }, { parameter }] }; + return typeof parameter === "number" + ? { less: [{ path }, { parameter }] } + : null; case "lessThanOrEqual": - return { lessOrEqual: [{ path }, { parameter }] }; + return typeof parameter === "number" + ? { lessOrEqual: [{ path }, { parameter }] } + : null; case "contains": - return { containsSegment: [{ path }, { parameter }] }; + return typeof parameter === "string" + ? { containsSegment: [{ path }, { parameter }] } + : null; case "startsWith": - return { startsWith: [{ path }, { parameter }] }; + return typeof parameter === "string" + ? { startsWith: [{ path }, { parameter }] } + : null; case "endsWith": - return { endsWith: [{ path }, { parameter }] }; + return typeof parameter === "string" + ? { endsWith: [{ path }, { parameter }] } + : null; + } +}; + +/** + * Translates a single property filter into the table endpoint's property + * filter, or returns `null` when the filter contributes no constraint (it is + * incomplete or its value is invalid for its kind) — the endpoint counterpart + * of {@link buildPropertyFilterClause}. + */ +export const buildEndpointPropertyFilter = ( + filter: PropertyFilter, +): EntityTablePropertyFilter | null => { + const property = filter.baseUrl; + + switch (filter.operator) { + case "hasAnyValue": + return { type: "hasAnyValue", property }; + case "isEmpty": + return { type: "isEmpty", property }; + case "isTrue": + return { type: "isTrue", property }; + case "isFalse": + return { type: "isFalse", property }; + default: + break; + } + + const parameter = coerceValueParameter(filter); + + if (parameter === null) { + return null; + } + + switch (filter.operator) { + case "equals": + return { type: "equals", property, value: parameter }; + case "notEquals": + return { type: "notEquals", property, value: parameter }; + // The ordering comparators take numbers and the text operators take + // strings. Which operators a filter's kind offers is UI convention, so a + // mismatched value renders the filter inert rather than a rejected query. + case "greaterThan": + return typeof parameter === "number" + ? { type: "greaterThan", property, value: parameter } + : null; + case "greaterThanOrEqual": + return typeof parameter === "number" + ? { type: "greaterThanOrEqual", property, value: parameter } + : null; + case "lessThan": + return typeof parameter === "number" + ? { type: "lessThan", property, value: parameter } + : null; + case "lessThanOrEqual": + return typeof parameter === "number" + ? { type: "lessThanOrEqual", property, value: parameter } + : null; + case "contains": + return typeof parameter === "string" + ? { type: "containsSegment", property, value: parameter } + : null; + case "startsWith": + return typeof parameter === "string" + ? { type: "startsWith", property, value: parameter } + : null; + case "endsWith": + return typeof parameter === "string" + ? { type: "endsWith", property, value: parameter } + : null; } }; diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/use-available-types.ts b/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/use-available-types.ts index 30f735d0040..40477dff1be 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/use-available-types.ts +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/shared/use-available-types.ts @@ -18,6 +18,7 @@ import type { import type { EntitiesFilterState } from "./filter-state"; import type { FilterMetadataForProperty } from "./property-filters/property-filter"; import type { BaseUrl, VersionedUrl, WebId } from "@blockprotocol/type-system"; +import type { EntityTableSummary } from "@local/hash-graph-sdk/entity"; export type AvailableType = { entityTypeId: VersionedUrl; @@ -25,18 +26,44 @@ export type AvailableType = { count: number; }; +/** + * Where the type summary the available types derive from comes from: fetched + * by the hook itself, or provided by the caller — the table endpoint's first + * page carries one. + */ +export type SummarySource = + | { mode: "fetch" } + | { + mode: "external"; + /** The provided summary, or `null` while the caller still loads it. */ + summary: EntityTableSummary | null; + /** + * Set when the caller's summary failed to load, so a `null` + * {@link summary} reads as an error instead of loading forever. + */ + error?: Error; + }; + export const useAvailableTypes = ({ filterState, internalWebs, entityTypeBaseUrl, entityTypeIds, + summarySource, }: { filterState: EntitiesFilterState; internalWebs: { webId: WebId }[]; entityTypeBaseUrl?: BaseUrl; entityTypeIds?: VersionedUrl[]; + summarySource: SummarySource; }): { availableEntityTypes: AvailableType[]; + /** + * The pinned types, with a base-URL pin resolved to all its versions. + * `null` when no type is pinned, or while a base-URL pin still awaits the + * loaded entity types. + */ + pinnedEntityTypeIds: VersionedUrl[] | null; propertyFilterData: FilterMetadataForProperty[]; loading: boolean; /** @@ -60,7 +87,8 @@ export const useAvailableTypes = ({ const { propertyTypes } = usePropertyTypes(); const isTypePinned = !!entityTypeBaseUrl || !!entityTypeIds?.length; - const shouldFetchAvailableTypes = !isTypePinned; + const shouldFetchAvailableTypes = + !isTypePinned && summarySource.mode === "fetch"; const pinnedEntityTypeIds = useMemo(() => { if (entityTypeIds?.length) { @@ -111,16 +139,41 @@ export const useAvailableTypes = ({ }, }); + /** + * The summary the available types derive from, wherever it came from. + * `null` while none has arrived. + */ + const summaryTypeData = useMemo<{ + typeIds: Record; + typeTitles: Record; + } | null>(() => { + if (summarySource.mode === "external") { + return summarySource.summary + ? { + typeIds: summarySource.summary.entityTypeIds, + typeTitles: summarySource.summary.entityTypeTitles, + } + : null; + } + + return data + ? { + typeIds: data.summarizeEntities.typeIds ?? {}, + typeTitles: data.summarizeEntities.typeTitles ?? {}, + } + : null; + }, [summarySource, data]); + const { availableEntityTypes, propertyFilterData } = useMemo<{ availableEntityTypes: AvailableType[]; propertyFilterData: FilterMetadataForProperty[]; }>(() => { - if (shouldFetchAvailableTypes && !data) { + if (!isTypePinned && !summaryTypeData) { return { availableEntityTypes: [], propertyFilterData: [] }; } - const typeIds = data?.summarizeEntities.typeIds ?? {}; - const typeTitles = data?.summarizeEntities.typeTitles ?? {}; + const typeIds = summaryTypeData?.typeIds ?? {}; + const typeTitles = summaryTypeData?.typeTitles ?? {}; const availableTypes = Object.entries(typeIds) .map(([entityTypeId, count]) => { @@ -137,10 +190,10 @@ export const useAvailableTypes = ({ return { availableEntityTypes: availableTypes, propertyFilterData: [] }; } - const availableEntityTypeIds = shouldFetchAvailableTypes + const availableEntityTypeIds = !isTypePinned ? (Object.keys(typeIds) as VersionedUrl[]) : (pinnedEntityTypeIds ?? []); - const selectedAvailableEntityTypeIds = shouldFetchAvailableTypes + const selectedAvailableEntityTypeIds = !isTypePinned ? filterState.type.selectedTypeIds ? [...filterState.type.selectedTypeIds].filter((typeId) => availableEntityTypeIds.includes(typeId), @@ -165,14 +218,14 @@ export const useAvailableTypes = ({ propertyFilterData: availableProperties, }; }, [ - data, + summaryTypeData, dataTypes, entityTypeParentIds, entityTypes, + isTypePinned, pinnedEntityTypeIds, filterState.type.selectedTypeIds, propertyTypes, - shouldFetchAvailableTypes, ]); const propertyFilterDataLoading = @@ -184,17 +237,27 @@ export const useAvailableTypes = ({ // silently render the whole workspace as "0 entities", so it surfaces as an // error instead. const typeUniverse = useMemo(() => { + if (summarySource.mode === "external") { + return summarySource.summary + ? (Object.keys(summarySource.summary.entityTypeIds) as VersionedUrl[]) + : null; + } + if (!data?.summarizeEntities.typeIds) { return null; } return Object.keys(data.summarizeEntities.typeIds) as VersionedUrl[]; - }, [data]); + }, [summarySource, data]); // Only fatal when it leaves us without a universe — a failed background // refresh with a cached universe still renders (slightly stale) results, - // which beats flipping a working page into an error state. + // which beats flipping a working page into an error state. An external + // summary reports its failures through its own path. const typeUniverseError = useMemo(() => { + if (summarySource.mode === "external") { + return typeUniverse === null ? summarySource.error : undefined; + } if (typeUniverse !== null) { return undefined; } @@ -210,14 +273,19 @@ export const useAvailableTypes = ({ } return undefined; - }, [data, error, typeUniverse]); + }, [summarySource, data, error, typeUniverse]); return { availableEntityTypes, + pinnedEntityTypeIds, propertyFilterData, - loading: shouldFetchAvailableTypes - ? loading || propertyFilterDataLoading - : propertyFilterDataLoading, + loading: + summarySource.mode === "external" + ? (summarySource.summary === null && !summarySource.error) || + propertyFilterDataLoading + : shouldFetchAvailableTypes + ? loading || propertyFilterDataLoading + : propertyFilterDataLoading, typeUniverse, typeUniverseError, refetchTypeUniverse: refetch, diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data.tsx deleted file mode 100644 index b83b41cf712..00000000000 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data.tsx +++ /dev/null @@ -1,160 +0,0 @@ -import { useCallback, useState } from "react"; - -import { isBaseUrl } from "@blockprotocol/type-system"; -import { typedEntries } from "@local/advanced-types/typed-entries"; -import { serializeSubgraph } from "@local/hash-graph-sdk/subgraph"; - -import { generateTableDataFromRows } from "./use-entities-table-data/generate-table-data-from-rows"; - -import type { - EntitiesTableColumn, - EntitiesTableData, - EntitiesTableRow, - UpdateTableDataFn, - VisibleDataTypeIdsByPropertyBaseUrl, -} from "./entities-table-data"; -import type { EntitiesVisualizerData } from "./use-entities-visualizer-data"; -import type { ClosedMultiEntityTypesRootMap } from "@local/hash-graph-sdk/ontology"; - -export const useEntitiesTableData = ({ - hideColumns, - hideArchivedColumn, -}: { - hideColumns?: (keyof EntitiesTableRow)[]; - hideArchivedColumn?: boolean; -}): { - tableData: EntitiesTableData | null; - updateTableData: UpdateTableDataFn; -} => { - const [tableData, setTableData] = useState(null); - - const updateTableData = useCallback( - ({ - appendRows, - closedMultiEntityTypesRootMap, - definitions, - entities, - subgraph, - }: Pick & { - appendRows: boolean; - closedMultiEntityTypesRootMap: ClosedMultiEntityTypesRootMap; - }) => { - if (!definitions) { - throw new Error("Definitions are required"); - } - - if (!entities) { - throw new Error("Entities are required"); - } - - if (!subgraph) { - throw new Error("Subgraph is required"); - } - - const resultFromRows = generateTableDataFromRows({ - closedMultiEntityTypesRootMap, - definitions, - entities: entities.map((entity) => entity.toJSON()), - subgraph: serializeSubgraph(subgraph), - hideColumns, - hideArchivedColumn, - }); - - setTableData((currentTableData) => { - if (appendRows && currentTableData) { - /** - * When paginating, append rows and merge the per-row metadata needed - * to render the accumulated table. Filter state and available filter - * options are derived from the whole result set, not from visible rows here. - */ - - const combinedVisibleDataTypeIdsByPropertyBaseUrl: VisibleDataTypeIdsByPropertyBaseUrl = - resultFromRows.visibleDataTypeIdsByPropertyBaseUrl; - for (const [baseUrl, dataTypeIds] of typedEntries( - currentTableData.visibleDataTypeIdsByPropertyBaseUrl, - )) { - combinedVisibleDataTypeIdsByPropertyBaseUrl[baseUrl] ??= new Set(); - combinedVisibleDataTypeIdsByPropertyBaseUrl[baseUrl] = - combinedVisibleDataTypeIdsByPropertyBaseUrl[baseUrl].union( - dataTypeIds, - ); - } - - const combinedEntityTypesWithMultipleVersionsPresent = new Set([ - ...currentTableData.entityTypesWithMultipleVersionsPresent, - ...resultFromRows.entityTypesWithMultipleVersionsPresent, - ]); - - const addedColumnIds: Set = new Set(); - const combinedColumns: EntitiesTableColumn[] = []; - - for (const column of [ - ...currentTableData.columns, - ...resultFromRows.columns, - ]) { - if (addedColumnIds.has(column.id)) { - continue; - } - - addedColumnIds.add(column.id); - - combinedColumns.push(column); - } - - return { - rows: [...currentTableData.rows, ...resultFromRows.rows], - /** - * Each page's response only carries the data types referenced by - * that page's entities, so we union the pools to keep every - * accumulated row resolvable. - */ - dataTypeDefinitions: { - ...currentTableData.dataTypeDefinitions, - ...resultFromRows.dataTypeDefinitions, - }, - columns: combinedColumns.sort((a, b) => { - /** - * The first page might not have source and target columns added (if there are no links), but a later one will. - * We want source and target columns to come before the property columns, so we sort property columns to the end. - */ - - const isAPropertyColumn = isBaseUrl(a.id); - const isBPropertyColumn = isBaseUrl(b.id); - - if (isAPropertyColumn && !isBPropertyColumn) { - return 1; - } - - if (!isAPropertyColumn && isBPropertyColumn) { - return -1; - } - - if (isAPropertyColumn && isBPropertyColumn) { - return a.title.localeCompare(b.title); - } - - return 0; - }), - entityTypesWithMultipleVersionsPresent: - combinedEntityTypesWithMultipleVersionsPresent, - visibleDataTypeIdsByPropertyBaseUrl: - combinedVisibleDataTypeIdsByPropertyBaseUrl, - }; - } - - // This is the first page, so return the result without combining. - return { - ...resultFromRows, - columns: resultFromRows.columns, - rows: resultFromRows.rows, - }; - }); - }, - [hideArchivedColumn, hideColumns], - ); - - return { - tableData, - updateTableData, - }; -}; diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query.tsx new file mode 100644 index 00000000000..0ce82efff62 --- /dev/null +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query.tsx @@ -0,0 +1,427 @@ +import { useQuery } from "@apollo/client"; +import * as Sentry from "@sentry/nextjs"; +import { useEffect, useMemo, useState } from "react"; + +import { noisySystemBaseUrls } from "@local/hash-isomorphic-utils/graph-queries"; + +import { queryEntitiesTableQuery } from "../../../graphql/queries/knowledge/entity.queries"; +import { apolloClient } from "../../../lib/apollo-client"; +import { buildEndpointPropertyFilter } from "./shared/property-filters/build-property-filter-clause"; +import { generateTableDataFromEndpointRows } from "./use-entities-table-query/generate-table-data-from-endpoint-rows"; + +import type { + QueryEntitiesTableQuery, + QueryEntitiesTableQueryVariables, +} from "../../../graphql/api-types.gen"; +import type { + EntitiesTableData, + EntitiesTableRow, +} from "./entities-table-data"; +import type { EntitiesFilterState } from "./shared/filter-state"; +import type { VersionedUrl, WebId } from "@blockprotocol/type-system"; +import type { EntityTableSorting } from "@local/hash-graph-client"; +import type { + ConversionRequest, + EntityTableRow, + EntityTableSummary, + EntityTableTypeScope, + EntityTableWebScope, +} from "@local/hash-graph-sdk/entity"; +import type { + ClosedMultiEntityTypesRootMap, + EntityTypeResolveDefinitions, +} from "@local/hash-graph-sdk/ontology"; + +export type EntitiesTableQueryData = { + /** The closed types of every accumulated page, deep-merged. */ + closedMultiEntityTypes: ClosedMultiEntityTypesRootMap | null; + /** Continuation for the next page, or `null` when there is none (yet). */ + cursor: string | null; + /** + * The rows in {@link tableData} belong to a previous request while the + * current one has not produced a page yet — kept so a failed refresh does + * not blank the table. + */ + dataIsStale: boolean; + /** The type definitions of every accumulated page, merged. */ + definitions: EntityTypeResolveDefinitions | null; + /** + * A failed page query or a response the table could not process. Pages + * already on screen stay in {@link tableData} so the error can be shown + * alongside them. + */ + error?: Error; + hadCachedContent: boolean; + loading: boolean; + refetch: () => Promise; + /** The first page's summary: the filtered count plus the scope's type maps. */ + summary: EntityTableSummary | null; + tableData: EntitiesTableData | null; + totalResultCount: number | null; + /** + * The page pins types none of which resolve to a version, so no query can + * be built. Retrying cannot help — the state only clears when the pinned + * types change. + */ + unresolvablePins: boolean; +}; + +/** + * The pages accumulated for one set of request filters. A response for + * different filters resets the accumulation, so stale pages never mix into a + * new sequence. + */ +type AccumulatedPages = { + requestKey: string; + rows: EntityTableRow[]; + closedMultiEntityTypes: ClosedMultiEntityTypesRootMap; + definitions: EntityTypeResolveDefinitions; + tableData: EntitiesTableData | null; + summary: EntityTableSummary | null; + nextCursor: string | null; + /** + * The continuation tokens already folded into {@link rows}, so a re-emitted + * response (e.g. Apollo's cache pass before the network pass) does not + * append its page twice. + */ + consumedCursors: Set; + processingError?: Error; +}; + +const mergeClosedMultiEntityTypeMaps = ( + target: ClosedMultiEntityTypesRootMap, + source: ClosedMultiEntityTypesRootMap, +): ClosedMultiEntityTypesRootMap => { + const merged: ClosedMultiEntityTypesRootMap = { ...target }; + + for (const [key, sourceEntry] of Object.entries(source)) { + const existing = merged[key]; + + merged[key] = existing + ? { + schema: sourceEntry.schema, + inner: mergeClosedMultiEntityTypeMaps( + existing.inner ?? {}, + sourceEntry.inner ?? {}, + ), + } + : sourceEntry; + } + + return merged; +}; + +const mergeDefinitions = ( + target: EntityTypeResolveDefinitions, + source: EntityTypeResolveDefinitions, +): EntityTypeResolveDefinitions => ({ + dataTypes: { ...target.dataTypes, ...source.dataTypes }, + propertyTypes: { ...target.propertyTypes, ...source.propertyTypes }, + entityTypes: { ...target.entityTypes, ...source.entityTypes }, +}); + +const asError = (thrown: unknown): Error => + thrown instanceof Error ? thrown : new Error(String(thrown)); + +/** + * Reads a page of the entities table through the dedicated `queryEntitiesTable` + * endpoint and accumulates pages into render-ready {@link EntitiesTableData}. + * + * Responses are processed in an effect rather than in Apollo's `onCompleted`, + * whose exceptions are silently dropped in production builds — a response the + * table cannot process surfaces through {@link EntitiesTableQueryData.error}. + */ +export const useEntitiesTableQuery = (params: { + conversions?: ConversionRequest[]; + /** Continuation token of the page to fetch, from a previous response. */ + cursor?: string; + enabled: boolean; + filterState: EntitiesFilterState; + hideArchivedColumn?: boolean; + hideColumns?: (keyof EntitiesTableRow)[]; + /** Whether the page pins types, making it wait for {@link params.resolvedPinnedEntityTypeIds}. */ + hasPinnedTypes: boolean; + internalWebs: { webId: WebId }[]; + /** The page size — the endpoint requires one. */ + limit: number; + /** + * The pinned types resolved to versions — `null` while they still resolve. + */ + resolvedPinnedEntityTypeIds: VersionedUrl[] | null; + sort: EntityTableSorting; +}): EntitiesTableQueryData => { + const { + conversions, + cursor, + enabled, + filterState, + hideArchivedColumn, + hideColumns, + hasPinnedTypes, + internalWebs, + limit, + resolvedPinnedEntityTypeIds, + sort, + } = params; + + const awaitingPinnedTypes = + hasPinnedTypes && resolvedPinnedEntityTypeIds === null; + + /** + * Pins that resolved to no version would query with an empty include list + * and render an empty table as "this type has no entities" — misleading, so + * it surfaces as an error instead. + */ + const unresolvablePins = + hasPinnedTypes && resolvedPinnedEntityTypeIds?.length === 0; + + const internalWebIds = useMemo( + () => internalWebs.map(({ webId }) => webId), + [internalWebs], + ); + + const variables = useMemo(() => { + if (!enabled || awaitingPinnedTypes || unresolvablePins) { + return null; + } + + const webs: EntityTableWebScope = filterState.web.includeOtherWebs + ? { + type: "exclude", + webs: internalWebIds.filter( + (webId) => !filterState.web.selectedInternalWebIds.has(webId), + ), + } + : { + type: "include", + webs: internalWebIds.filter((webId) => + filterState.web.selectedInternalWebIds.has(webId), + ), + }; + + const selectedTypeIds = filterState.type.selectedTypeIds; + const includedTypeIds = + resolvedPinnedEntityTypeIds ?? + (selectedTypeIds ? [...selectedTypeIds] : null); + + const types: EntityTableTypeScope = includedTypeIds + ? { type: "include", entityTypeIds: includedTypeIds } + : { + type: "exclude", + entityTypeBaseUrls: [...noisySystemBaseUrls], + }; + + const propertyFilters = filterState.propertyFilters + .map(buildEndpointPropertyFilter) + .filter((propertyFilter) => propertyFilter !== null); + + return { + request: { + conversions, + filter: { + webs, + types, + includeArchived: filterState.includeArchived, + includeDrafts: false, + propertyFilters: + propertyFilters.length > 0 ? propertyFilters : undefined, + }, + cursor: cursor ?? undefined, + limit, + sort, + includeSummary: !cursor, + includeEntityTypes: "resolvedWithDataTypeChildren", + }, + }; + }, [ + enabled, + awaitingPinnedTypes, + unresolvablePins, + conversions, + cursor, + filterState.web, + filterState.type.selectedTypeIds, + filterState.includeArchived, + filterState.propertyFilters, + internalWebIds, + limit, + resolvedPinnedEntityTypeIds, + sort, + ]); + + /** Everything except the cursor identifies the accumulation sequence. */ + const requestKey = useMemo(() => { + if (!variables) { + return null; + } + + const { + cursor: _cursor, + includeSummary: _includeSummary, + ...rest + } = variables.request; + + return JSON.stringify(rest); + }, [variables]); + + const { + data, + error: queryError, + loading, + refetch, + } = useQuery( + queryEntitiesTableQuery, + { + fetchPolicy: "cache-and-network", + skip: !variables, + variables: variables ?? undefined, + }, + ); + + const [accumulated, setAccumulated] = useState(null); + + useEffect(() => { + const response = data?.queryEntitiesTable; + + if (!response || !requestKey || !variables) { + return; + } + + const pageCursor = variables.request.cursor ?? null; + + setAccumulated((current) => { + const continues = + pageCursor !== null && current?.requestKey === requestKey; + + if (continues && current.consumedCursors.has(pageCursor)) { + return current; + } + + try { + if (!response.definitions) { + throw new Error("The response carries no type definitions"); + } + if (!response.closedMultiEntityTypes) { + throw new Error("The response carries no closed types"); + } + if (pageCursor === null && !response.summary) { + // Every first page requests a summary. Without this guard a missing + // one would leave the filter chips loading forever. + throw new Error("The response carries no summary"); + } + + const rows = continues + ? [...current.rows, ...response.rows] + : [...response.rows]; + const closedMultiEntityTypes = continues + ? mergeClosedMultiEntityTypeMaps( + current.closedMultiEntityTypes, + response.closedMultiEntityTypes, + ) + : response.closedMultiEntityTypes; + const definitions = continues + ? mergeDefinitions(current.definitions, response.definitions) + : response.definitions; + + return { + requestKey, + rows, + closedMultiEntityTypes, + definitions, + tableData: generateTableDataFromEndpointRows({ + closedMultiEntityTypesRootMap: closedMultiEntityTypes, + definitions, + endpointRows: rows, + hideColumns, + hideArchivedColumn, + }), + summary: response.summary ?? (continues ? current.summary : null), + nextCursor: response.cursor ?? null, + consumedCursors: continues + ? new Set([...current.consumedCursors, pageCursor]) + : new Set(pageCursor === null ? [] : [pageCursor]), + }; + } catch (thrown) { + // The pages already shown stay intact, and the cursor is not marked + // consumed so a retry processes the page again. + return current + ? { ...current, processingError: asError(thrown) } + : { + requestKey, + rows: [], + closedMultiEntityTypes: {}, + definitions: { + dataTypes: {}, + propertyTypes: {}, + entityTypes: {}, + }, + tableData: null, + summary: null, + nextCursor: null, + consumedCursors: new Set(), + processingError: asError(thrown), + }; + } + }); + }, [data, requestKey, variables, hideColumns, hideArchivedColumn]); + + const hadCachedContent = useMemo( + () => + !!variables && + !!apolloClient.readQuery({ + query: queryEntitiesTableQuery, + variables, + }), + [variables], + ); + + // Processing errors are backend/frontend contract breaks that reproduce on + // every retry — without the report they are invisible in production. + const processingError = accumulated?.processingError; + useEffect(() => { + if (processingError) { + Sentry.captureException(processingError); + } + }, [processingError]); + + return useMemo(() => { + // Stale accumulation (from previous filters) keeps the old rows on screen + // while the new first page loads, but must not offer its continuation. + const isCurrent = + accumulated !== null && + requestKey !== null && + accumulated.requestKey === requestKey; + + const error = + queryError ?? + accumulated?.processingError ?? + (unresolvablePins + ? new Error("The pinned type could not be resolved to any version") + : undefined); + + return { + closedMultiEntityTypes: accumulated?.closedMultiEntityTypes ?? null, + cursor: isCurrent ? accumulated.nextCursor : null, + dataIsStale: accumulated !== null && !isCurrent, + definitions: accumulated?.definitions ?? null, + error, + hadCachedContent, + loading: loading || (enabled && awaitingPinnedTypes), + refetch, + summary: accumulated?.summary ?? null, + tableData: accumulated?.tableData ?? null, + totalResultCount: accumulated?.summary?.count ?? null, + unresolvablePins, + }; + }, [ + accumulated, + requestKey, + queryError, + unresolvablePins, + hadCachedContent, + loading, + enabled, + awaitingPinnedTypes, + refetch, + ]); +}; diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data/generate-table-data-from-rows.ts b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query/generate-table-data-from-endpoint-rows.ts similarity index 54% rename from apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data/generate-table-data-from-rows.ts rename to apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query/generate-table-data-from-endpoint-rows.ts index bb25615cd23..75c134a182b 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-data/generate-table-data-from-rows.ts +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query/generate-table-data-from-endpoint-rows.ts @@ -1,6 +1,5 @@ import { format } from "date-fns"; -import { getEntityRevision } from "@blockprotocol/graph/stdlib"; import { extractBaseUrl, extractVersion, @@ -10,12 +9,8 @@ import { typedEntries, typedKeys } from "@local/advanced-types/typed-entries"; import { getClosedMultiEntityTypeFromMap, getDisplayFieldsForClosedEntityType, - HashEntity, } from "@local/hash-graph-sdk/entity"; -import { - generateEntityLabel, - generateLinkEntityLabel, -} from "@local/hash-isomorphic-utils/generate-entity-label"; +import { generateEntityLabel } from "@local/hash-isomorphic-utils/generate-entity-label"; import { blockProtocolEntityTypes } from "@local/hash-isomorphic-utils/ontology-type-ids"; import { includesPageEntityTypeId } from "@local/hash-isomorphic-utils/page-entity-type-ids"; import { simplifyProperties } from "@local/hash-isomorphic-utils/simplify-properties"; @@ -28,10 +23,21 @@ import type { EntitiesTableData, EntitiesTableRow, EntitiesTableRowPropertyCell, - GenerateEntitiesTableDataParams, VisibleDataTypeIdsByPropertyBaseUrl, } from "../entities-table-data"; -import type { BaseUrl, VersionedUrl } from "@blockprotocol/type-system"; +import type { + BaseUrl, + ClosedMultiEntityType, + VersionedUrl, +} from "@blockprotocol/type-system"; +import type { + EntityTableLinkEndpoint, + EntityTableRow as EndpointRow, +} from "@local/hash-graph-sdk/entity"; +import type { + ClosedMultiEntityTypesRootMap, + EntityTypeResolveDefinitions, +} from "@local/hash-graph-sdk/ontology"; import type { PageProperties } from "@local/hash-isomorphic-utils/system-types/shared"; const staticColumnDefinitionsByKey: Record< @@ -87,7 +93,7 @@ const staticColumnDefinitionsByKey: Record< let canvas: HTMLCanvasElement | undefined = undefined; -export const getTextWidth = (text: string) => { +const getTextWidth = (text: string) => { canvas ??= document.createElement("canvas"); const context = canvas.getContext("2d")!; @@ -98,18 +104,102 @@ export const getTextWidth = (text: string) => { return metrics.width; }; -export const generateTableDataFromRows = ( - params: GenerateEntitiesTableDataParams, -): EntitiesTableData => { - const { +const assembleEntitiesTableColumns = ({ + propertyColumns, + sharedTypeTitle, + hideColumns, + hideArchivedColumn, + allRowsMissSource, + allRowsMissTarget, +}: { + propertyColumns: EntitiesTableColumn[]; + /** The type title shared by every row, taking over the label column's header. */ + sharedTypeTitle: string | undefined; + hideColumns?: (keyof EntitiesTableRow)[]; + hideArchivedColumn?: boolean; + allRowsMissSource: boolean; + allRowsMissTarget: boolean; +}): EntitiesTableColumn[] => { + const columns: EntitiesTableColumn[] = [ + { + title: sharedTypeTitle ?? "Entity", + id: "entityLabel", + width: 300, + grow: 1, + }, + ]; + + const columnsToHide = hideColumns ? [...hideColumns] : []; + if (hideArchivedColumn) { + columnsToHide.push("archived"); + } + + if (allRowsMissSource) { + columnsToHide.push("sourceEntity"); + } + + if (allRowsMissTarget) { + columnsToHide.push("targetEntity"); + } + + for (const [columnKey, definition] of typedEntries( + staticColumnDefinitionsByKey, + )) { + if (!columnsToHide.includes(columnKey)) { + columns.push(definition); + } + } + + columns.push( + ...propertyColumns.sort((a, b) => a.title.localeCompare(b.title)), + ); + + return columns; +}; + +const linkEndpointCell = ( + endpoint: EntityTableLinkEndpoint, + closedMultiEntityTypesRootMap: ClosedMultiEntityTypesRootMap, +): NonNullable => { + const closedMultiEntityType = getClosedMultiEntityTypeFromMap( closedMultiEntityTypesRootMap, - definitions, - entities, - subgraph, - hideColumns, - hideArchivedColumn, - } = params; + endpoint.entityTypeIds, + ); + + let isLink = false; + for (const entityType of closedMultiEntityType.allOf) { + for (const typeOrAncestor of entityType.allOf) { + if (typeOrAncestor.$id === blockProtocolEntityTypes.link.entityTypeId) { + isLink = true; + break; + } + } + } + + return { + entityId: endpoint.entityId, + label: endpoint.label ?? endpoint.entityId, + icon: getDisplayFieldsForClosedEntityType(closedMultiEntityType).icon, + isLink, + }; +}; +/** + * Builds the table's rows and columns from `queryEntitiesTable` endpoint rows. + */ +export const generateTableDataFromEndpointRows = ({ + closedMultiEntityTypesRootMap, + definitions, + endpointRows, + hideColumns, + hideArchivedColumn, +}: { + closedMultiEntityTypesRootMap: ClosedMultiEntityTypesRootMap; + definitions: EntityTypeResolveDefinitions; + endpointRows: EndpointRow[]; + hideColumns?: (keyof EntitiesTableRow)[]; + hideArchivedColumn?: boolean; +}): EntitiesTableData => { let noSource = 0; let noTarget = 0; @@ -124,17 +214,27 @@ export const generateTableDataFromRows = ( const rows: EntitiesTableRow[] = []; - for (const [index, serializedEntity] of entities.entries()) { - const entity = new HashEntity(serializedEntity); - - const closedMultiEntityType = getClosedMultiEntityTypeFromMap( - closedMultiEntityTypesRootMap, - entity.metadata.entityTypeIds, - ); - - const entityLabel = generateEntityLabel(closedMultiEntityType, entity); - - for (const entityTypeId of entity.metadata.entityTypeIds) { + for (const [index, endpointRow] of endpointRows.entries()) { + const closedMultiEntityType: ClosedMultiEntityType = + getClosedMultiEntityTypeFromMap( + closedMultiEntityTypesRootMap, + endpointRow.entityTypeIds, + ); + + const entityLabel = + endpointRow.label ?? + generateEntityLabel(closedMultiEntityType, { + properties: endpointRow.properties, + metadata: { + recordId: { + entityId: endpointRow.entityId, + editionId: endpointRow.entityEditionId, + }, + entityTypeIds: endpointRow.entityTypeIds, + }, + }); + + for (const entityTypeId of endpointRow.entityTypeIds) { const baseUrl = extractBaseUrl(entityTypeId); if ( @@ -154,9 +254,6 @@ export const generateTableDataFromRows = ( const entityTypeTitles = new Set(); for (const entityTypeMetadata of closedMultiEntityType.allOf) { if (index === 0) { - /** - * We add the titles of the types of the first entity to the set. - */ entityTypeTitlesSharedAcrossAllEntities.add(entityTypeMetadata.title); } else { entityTypeTitles.add(entityTypeMetadata.title); @@ -170,38 +267,15 @@ export const generateTableDataFromRows = ( } } - /** - * Check for any titles in our shared set that aren't present in the current entity's types. - * If they aren't present, remove them from the shared set – they aren't shared. - */ - for (const sharedTitle of entityTypeTitlesSharedAcrossAllEntities) { - if (!entityTypeTitles.has(sharedTitle)) { - entityTypeTitlesSharedAcrossAllEntities.delete(sharedTitle); + if (index > 0) { + for (const sharedTitle of entityTypeTitlesSharedAcrossAllEntities) { + if (!entityTypeTitles.has(sharedTitle)) { + entityTypeTitlesSharedAcrossAllEntities.delete(sharedTitle); + } } } - const entityId = entity.metadata.recordId.entityId; - - const isPage = includesPageEntityTypeId(entity.metadata.entityTypeIds); - - /** - * @todo: consider displaying handling this differently for pages, where - * updates on nested blocks/text entities may be a better indicator of - * when a page has been last edited. - */ - const lastEdited = format( - new Date(entity.metadata.temporalVersioning.decisionTime.start.limit), - "yyyy-MM-dd HH:mm", - ); - - const lastEditedById = entity.metadata.provenance.edition.createdById; - - const created = format( - new Date(entity.metadata.provenance.createdAtDecisionTime), - "yyyy-MM-dd HH:mm", - ); - - const createdById = entity.metadata.provenance.createdById; + const isPage = includesPageEntityTypeId(endpointRow.entityTypeIds); const propertyCellsForRow: Record = {}; @@ -215,25 +289,26 @@ export const generateTableDataFromRows = ( if (!propertyType) { throw new Error( - `Property type not found for ${propertyTypeId} in ${entityId}`, + `Property type not found for ${propertyTypeId} in ${endpointRow.entityId}`, ); } const isArray = "items" in schema || "items" in propertyType.oneOf[0]; - if (entity.properties[baseUrl] !== undefined) { - const propertyMetadata = entity.propertyMetadata([baseUrl]); + const value = endpointRow.properties[baseUrl]; + if (value !== undefined) { + const propertyMetadata = endpointRow.propertiesMetadata.value[baseUrl]; if (!propertyMetadata) { throw new Error( - `Property metadata not found for ${baseUrl} in ${entityId}`, + `Property metadata not found for ${baseUrl} in ${endpointRow.entityId}`, ); } propertyCellsForRow[baseUrl] = { isArray, propertyMetadata, - value: entity.properties[baseUrl], + value, }; } @@ -248,76 +323,27 @@ export const generateTableDataFromRows = ( } } - let sourceEntity: EntitiesTableRow["sourceEntity"]; - let targetEntity: EntitiesTableRow["targetEntity"]; - if (entity.linkData) { - const source = getEntityRevision(subgraph, entity.linkData.leftEntityId); - const target = getEntityRevision(subgraph, entity.linkData.rightEntityId); - - const sourceClosedMultiEntityType = source - ? getClosedMultiEntityTypeFromMap( - closedMultiEntityTypesRootMap, - source.metadata.entityTypeIds, - ) - : undefined; - - const sourceEntityLabel = - !source || !sourceClosedMultiEntityType - ? entity.linkData.leftEntityId - : source.linkData - ? generateLinkEntityLabel(subgraph, source, { - closedType: sourceClosedMultiEntityType, - entityTypeDefinitions: definitions, - closedMultiEntityTypesRootMap, - }) - : generateEntityLabel(sourceClosedMultiEntityType, source); - - const sourceDisplayFields = sourceClosedMultiEntityType - ? getDisplayFieldsForClosedEntityType(sourceClosedMultiEntityType) - : undefined; - - sourceEntity = { - entityId: entity.linkData.leftEntityId, - label: sourceEntityLabel, - icon: sourceDisplayFields?.icon, - isLink: !!source?.linkData, - }; - - const targetClosedMultiEntityType = target - ? getClosedMultiEntityTypeFromMap( - closedMultiEntityTypesRootMap, - target.metadata.entityTypeIds, - ) - : undefined; - - const targetEntityLabel = - !target || !targetClosedMultiEntityType - ? entity.linkData.leftEntityId - : target.linkData - ? generateLinkEntityLabel(subgraph, target, { - closedType: targetClosedMultiEntityType, - entityTypeDefinitions: definitions, - closedMultiEntityTypesRootMap, - }) - : generateEntityLabel(targetClosedMultiEntityType, target); - - const targetDisplayFields = targetClosedMultiEntityType - ? getDisplayFieldsForClosedEntityType(targetClosedMultiEntityType) - : undefined; - - targetEntity = { - entityId: entity.linkData.rightEntityId, - label: targetEntityLabel, - icon: targetDisplayFields?.icon, - isLink: !!target?.linkData, - }; - } else { + const sourceEntity = endpointRow.sourceEntity + ? linkEndpointCell( + endpointRow.sourceEntity, + closedMultiEntityTypesRootMap, + ) + : undefined; + const targetEntity = endpointRow.targetEntity + ? linkEndpointCell( + endpointRow.targetEntity, + closedMultiEntityTypesRootMap, + ) + : undefined; + if (!sourceEntity) { noSource += 1; + } + if (!targetEntity) { noTarget += 1; } for (const [baseUrl, { metadata }] of typedEntries( - entity.propertiesMetadata.value, + endpointRow.propertiesMetadata.value, )) { if (metadata && "dataTypeId" in metadata && metadata.dataTypeId) { dataTypesByProperty[baseUrl] ??= new Set(); @@ -326,21 +352,17 @@ export const generateTableDataFromRows = ( if (!dataType) { throw new Error( - `Could not find dataType with id ${metadata.dataTypeId} in subgraph`, + `Could not find dataType with id ${metadata.dataTypeId} in the definitions`, ); } - /** - * As there is only one instance of each DataType in the subgraph, it'll be the same object in memory, - * and the Set equality check will work. - */ dataTypesByProperty[baseUrl].add(dataType); } } rows.push({ - rowId: entityId, - entityId, + rowId: endpointRow.entityId, + entityId: endpointRow.entityId, entityLabel, entityIcon, entityTypes: closedMultiEntityType.allOf.map((entityType) => { @@ -368,14 +390,20 @@ export const generateTableDataFromRows = ( version: extractVersion(entityType.$id), }; }), - webId: extractWebIdFromEntityId(entity.entityId), + webId: extractWebIdFromEntityId(endpointRow.entityId), archived: isPage - ? simplifyProperties(entity.properties as PageProperties).archived - : entity.metadata.archived, - lastEdited, - lastEditedById, - created, - createdById, + ? simplifyProperties(endpointRow.properties as PageProperties).archived + : endpointRow.archived, + lastEdited: format( + new Date(endpointRow.editionCreatedAtDecisionTime), + "yyyy-MM-dd HH:mm", + ), + lastEditedById: endpointRow.lastEditedBy, + created: format( + new Date(endpointRow.createdAtDecisionTime), + "yyyy-MM-dd HH:mm", + ), + createdById: endpointRow.createdBy, sourceEntity, targetEntity, applicableProperties: typedKeys(closedMultiEntityType.properties), @@ -383,47 +411,16 @@ export const generateTableDataFromRows = ( }); } - const propertyColumns = Array.from(propertyColumnsMap.values()); - - const columns: EntitiesTableColumn[] = [ - { - title: - entityTypeTitlesSharedAcrossAllEntities.size === 0 - ? "Entity" - : entityTypeTitlesSharedAcrossAllEntities.values().next().value!, - id: "entityLabel", - width: 300, - grow: 1, - }, - ]; - - const columnsToHide = hideColumns ? [...hideColumns] : []; - if (hideArchivedColumn) { - columnsToHide.push("archived"); - } - - if (noSource === rows.length) { - columnsToHide.push("sourceEntity"); - } - - if (noTarget === rows.length) { - columnsToHide.push("targetEntity"); - } - - for (const [columnKey, definition] of typedEntries( - staticColumnDefinitionsByKey, - )) { - if (!columnsToHide.includes(columnKey)) { - columns.push(definition); - } - } - - columns.push( - ...propertyColumns.sort((a, b) => a.title.localeCompare(b.title)), - ); - return { - columns, + columns: assembleEntitiesTableColumns({ + propertyColumns: Array.from(propertyColumnsMap.values()), + sharedTypeTitle: entityTypeTitlesSharedAcrossAllEntities.values().next() + .value, + hideColumns, + hideArchivedColumn, + allRowsMissSource: noSource === rows.length, + allRowsMissTarget: noTarget === rows.length, + }), dataTypeDefinitions: definitions.dataTypes, rows, entityTypesWithMultipleVersionsPresent: entityTypesWithMultipleVersions, diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx index c29d3c000c4..295f1cfdcd9 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-visualizer-data.tsx @@ -16,7 +16,6 @@ import { import { apolloClient } from "../../../lib/apollo-client"; import { buildEntitiesFilter } from "./shared/build-filter"; import { traversalPathsForView } from "./shared/traversal-paths"; -import { useEntitiesTableData } from "./use-entities-table-data"; import type { QueryEntitySubgraphQuery, @@ -25,13 +24,7 @@ import type { SummarizeEntitiesQueryVariables, } from "../../../graphql/api-types.gen"; import type { VisualizerView } from "../visualizer-views"; -import type { - EntitiesTableData, - EntitiesTableRow, - UpdateTableDataFn, -} from "./entities-table-data"; import type { EntitiesFilterState } from "./shared/filter-state"; -import type { ApolloQueryResult } from "@apollo/client"; import type { EntityRootType, Subgraph } from "@blockprotocol/graph"; import type { BaseUrl, VersionedUrl, WebId } from "@blockprotocol/type-system"; import type { @@ -42,47 +35,43 @@ import type { export type EntitiesVisualizerData = Partial< Pick< QueryEntitySubgraphQuery["queryEntitySubgraph"], - "closedMultiEntityTypes" | "definitions" | "cursor" + "closedMultiEntityTypes" | "definitions" > > & { + cursor?: EntityQueryCursor | null; entities?: HashEntity[]; + /** Set when the page query failed. */ + error?: Error; hadCachedContent: boolean; loading: boolean; /** - * Whether or not a network request is in process. - * Note that if is hasCachedContent is true, data for the given query is available before loading is complete. - * The cached content will be replaced automatically and the value updated when the network request completes. - */ - /** - * Refetches the subgraph query only — the type universe keeps its last - * snapshot, so entities of types that appeared after it loaded stay hidden - * until the universe query itself refetches. Resolves to `undefined` without - * querying while the type universe is still awaited. + * Refetches the page query. Resolves to `undefined` without querying while + * the type universe is still awaited. */ - refetch: () => Promise< - ApolloQueryResult | undefined - >; + refetch: () => Promise; subgraph?: Subgraph>; - tableData: EntitiesTableData | null; totalResultCount: number | null; - updateTableData: UpdateTableDataFn; }; +/** + * Reads the Grid and Graph views' data through `queryEntitySubgraph`. The + * Table view reads through `useEntitiesTableQuery` instead, which skips the + * queries here. + */ export const useEntitiesVisualizerData = (params: { conversions?: ConversionRequest[]; cursor?: EntityQueryCursor; entityTypeBaseUrl?: BaseUrl; entityTypeIds?: VersionedUrl[]; filterState: EntitiesFilterState; - hideColumns?: (keyof EntitiesTableRow)[]; internalWebs: { webId: WebId }[]; - limit?: number; + limit: number; sort?: EntityQuerySortingRecord; /** * The type universe from `useAvailableTypes` — `null` while the summary is in * flight, and permanently for pinned types, which never fetch it. The default * (no type selection) view sends it as an include-type clause and holds its - * queries back until it arrives — see {@link buildEntitiesFilter}. + * queries back until it is available — see {@link buildEntitiesFilter}. */ typeUniverse: VersionedUrl[] | null; /** @@ -99,7 +88,6 @@ export const useEntitiesVisualizerData = (params: { entityTypeBaseUrl, entityTypeIds, filterState, - hideColumns, internalWebs, limit, sort, @@ -108,11 +96,6 @@ export const useEntitiesVisualizerData = (params: { view, } = params; - const { tableData, updateTableData } = useEntitiesTableData({ - hideColumns, - hideArchivedColumn: !filterState.includeArchived, - }); - const internalWebIds = useMemo( () => internalWebs.map(({ webId }) => webId), [internalWebs], @@ -142,6 +125,8 @@ export const useEntitiesVisualizerData = (params: { !entityTypeIds?.length && filterState.type.selectedTypeIds === null; + const skip = view === "Table" || awaitingTypeUniverse; + const variables = useMemo( () => ({ request: { @@ -168,7 +153,7 @@ export const useEntitiesVisualizerData = (params: { SummarizeEntitiesQuery, SummarizeEntitiesQueryVariables >(summarizeEntitiesQuery, { - skip: awaitingTypeUniverse, + skip, variables: { request: { filter, @@ -179,32 +164,12 @@ export const useEntitiesVisualizerData = (params: { }, }); - const { data, loading, refetch } = useQuery< + const { data, error, loading, refetch } = useQuery< QueryEntitySubgraphQuery, QueryEntitySubgraphQueryVariables >(queryEntitySubgraphQuery, { fetchPolicy: "cache-and-network", - skip: awaitingTypeUniverse, - onCompleted: (completedData) => { - if (view === "Graph") { - return; - } - - const newSubgraph = deserializeQueryEntitySubgraphResponse( - completedData.queryEntitySubgraph, - ).subgraph; - - const newEntities = getRoots(newSubgraph); - - updateTableData({ - appendRows: !!cursor, - closedMultiEntityTypesRootMap: - completedData.queryEntitySubgraph.closedMultiEntityTypes ?? {}, - definitions: completedData.queryEntitySubgraph.definitions, - entities: newEntities, - subgraph: newSubgraph, - }); - }, + skip, variables, }); @@ -212,16 +177,19 @@ export const useEntitiesVisualizerData = (params: { // filter has no type clause at all — exactly the unestimable shape the gate // exists to prevent. Drop the call instead. const guardedRefetch = useCallback(async () => { - if (awaitingTypeUniverse) { + if (skip) { return undefined; } return refetch(); - }, [awaitingTypeUniverse, refetch]); + }, [skip, refetch]); const hadCachedContent = useMemo( () => - !!apolloClient.readQuery({ query: queryEntitySubgraphQuery, variables }), + !!apolloClient.readQuery({ + query: queryEntitySubgraphQuery, + variables, + }), [variables], ); @@ -248,26 +216,24 @@ export const useEntitiesVisualizerData = (params: { () => ({ ...data?.queryEntitySubgraph, entities, + error, hadCachedContent, loading: loading || (awaitingTypeUniverse && !typeUniverseError), refetch: guardedRefetch, subgraph, - tableData, totalResultCount: summaryData?.summarizeEntities.count ?? null, - updateTableData, }), [ data?.queryEntitySubgraph, summaryData?.summarizeEntities, entities, + error, hadCachedContent, loading, awaitingTypeUniverse, typeUniverseError, guardedRefetch, subgraph, - tableData, - updateTableData, ], ); }; diff --git a/apps/hash-frontend/src/shared/is-archived.ts b/apps/hash-frontend/src/shared/is-archived.ts index ee0137a12be..988e6dd616e 100644 --- a/apps/hash-frontend/src/shared/is-archived.ts +++ b/apps/hash-frontend/src/shared/is-archived.ts @@ -4,12 +4,35 @@ import { isEntityPageEntity, isType } from "./is-of-type"; import type { DataTypeWithMetadata, - Entity, + EntityId, EntityTypeWithMetadata, + PropertyObject, PropertyTypeWithMetadata, + VersionedUrl, } from "@blockprotocol/type-system"; import type { PageProperties } from "@local/hash-isomorphic-utils/system-types/shared"; +/** + * The slice of an entity that archival state and the archive actions read. + * Full entities satisfy it structurally, as do lighter objects assembled from + * table rows. + * + * Deliberately not a `Pick` of `Entity`: a Pick would drag in the full entity + * metadata (provenance, temporal versioning) that row-assembled objects lack, + * and `entityTypeIds` weakens the metadata's non-empty tuple to a plain array + * so those objects qualify. `archived` stays optional because the ontology + * type union members in {@link isItemArchived} carry no such field, which is + * also why callers probe it with an `in` check. + */ +export type ArchivableEntity = { + metadata: { + recordId: { entityId: EntityId }; + entityTypeIds: VersionedUrl[]; + archived?: boolean; + }; + properties: PropertyObject; +}; + export const isTypeArchived = ( type: | EntityTypeWithMetadata @@ -17,7 +40,7 @@ export const isTypeArchived = ( | DataTypeWithMetadata, ) => type.metadata.temporalVersioning.transactionTime.end.kind === "exclusive"; -export const isPageArchived = (pageEntity: Entity) => { +export const isPageArchived = (pageEntity: ArchivableEntity) => { if (!isEntityPageEntity(pageEntity)) { throw new Error("Not a page entity"); } @@ -31,7 +54,7 @@ export const isPageArchived = (pageEntity: Entity) => { export const isItemArchived = ( item: - | Entity + | ArchivableEntity | EntityTypeWithMetadata | PropertyTypeWithMetadata | DataTypeWithMetadata, diff --git a/apps/hash-frontend/src/shared/is-of-type.ts b/apps/hash-frontend/src/shared/is-of-type.ts index 2084beb5c37..962b6de961e 100644 --- a/apps/hash-frontend/src/shared/is-of-type.ts +++ b/apps/hash-frontend/src/shared/is-of-type.ts @@ -2,14 +2,14 @@ import { includesPageEntityTypeId } from "@local/hash-isomorphic-utils/page-enti import type { DataTypeWithMetadata, - Entity, EntityTypeWithMetadata, PropertyTypeWithMetadata, + VersionedUrl, } from "@blockprotocol/type-system"; export const isType = ( item: - | Entity + | { metadata: { entityTypeIds: VersionedUrl[] } } | EntityTypeWithMetadata | PropertyTypeWithMetadata | DataTypeWithMetadata, @@ -39,5 +39,6 @@ export const isTypeDataType = ( | DataTypeWithMetadata, ) => type.schema.kind === "dataType"; -export const isEntityPageEntity = (item: Entity) => - includesPageEntityTypeId(item.metadata.entityTypeIds); +export const isEntityPageEntity = (item: { + metadata: { entityTypeIds: VersionedUrl[] }; +}) => includesPageEntityTypeId(item.metadata.entityTypeIds); diff --git a/apps/hash-frontend/src/shared/table-header/bulk-actions-dropdown.tsx b/apps/hash-frontend/src/shared/table-header/bulk-actions-dropdown.tsx index d1970c6b040..5b95f55b36d 100644 --- a/apps/hash-frontend/src/shared/table-header/bulk-actions-dropdown.tsx +++ b/apps/hash-frontend/src/shared/table-header/bulk-actions-dropdown.tsx @@ -22,7 +22,6 @@ import { type WebId, } from "@blockprotocol/type-system"; import { CaretDownSolidIcon, Chip } from "@hashintel/design-system"; -import { isEntity } from "@local/hash-isomorphic-utils/entity-store"; import { useArchivePage } from "../../components/hooks/use-archive-page"; import { archiveEntityMutation } from "../../graphql/queries/knowledge/entity.queries"; @@ -60,12 +59,12 @@ import type { UnarchivePropertyTypeMutation, UnarchivePropertyTypeMutationVariables, } from "../../graphql/api-types.gen"; -import type { HashEntity } from "@local/hash-graph-sdk/entity"; +import type { ArchivableEntity } from "../is-archived"; import type { FunctionComponent } from "react"; export const BulkActionsDropdown: FunctionComponent<{ selectedItems: ( - | HashEntity + | ArchivableEntity | EntityTypeWithMetadata | PropertyTypeWithMetadata | DataTypeWithMetadata @@ -139,7 +138,7 @@ export const BulkActionsDropdown: FunctionComponent<{ return false; } - if (isEntity(item)) { + if (!isType(item)) { return true; } From 51ca62ade4245adf819a8f4666ab912004eb7fea Mon Sep 17 00:00:00 2001 From: Tim Diekmann Date: Sat, 25 Jul 2026 14:32:42 +0200 Subject: [PATCH 07/18] BE-705: Let the table query own its page sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refreshing the table after a bulk action re-ran the page in flight, whose cursor pins the snapshot the sequence started on — and whose response the dedup guard then discarded as a replay. Archiving a row on any table past its first page changed nothing on screen, with no error. The hook now owns its cursor and exposes loadMore/restart: a restart drops the accumulation and reads the sequence from its first page, where the mutation is visible. Owning the cursor also settles which sequence it belongs to. A cursor is only sent when the current accumulation handed it out, so a navigation that swaps the pinned type without remounting can no longer continue a foreign sequence — which skipped its first pages and, with the summary suppressed for cursored requests, left the filter chips loading forever. Appending a page no longer re-derives the rows before it: the generator carries its aggregates forward, so a page costs a page instead of the whole table. Retrying a failed first page now restarts the table query rather than a summary query the table does not read. Type exclusions ride alongside a selection through the endpoint's new filter field, so selecting a pill keeps the noisy system types out of the rows, the count, and the pills. The row generator gets its first tests, and the builder-agreement test enumerates the operator and kind space instead of sampling it. --- .../src/components/grid/grid.tsx | 4 +- .../src/components/grid/utils.ts | 3 + .../src/pages/shared/entities-visualizer.tsx | 35 +-- .../build-property-filter-clause.test.ts | 50 ++-- .../use-entities-table-query.tsx | 246 ++++++++++-------- ...le-data-from-endpoint-rows.browser.test.ts | 214 +++++++++++++++ .../generate-table-data-from-endpoint-rows.ts | 135 +++++++--- 7 files changed, 506 insertions(+), 181 deletions(-) create mode 100644 apps/hash-frontend/src/pages/shared/entities-visualizer/use-entities-table-query/generate-table-data-from-endpoint-rows.browser.test.ts diff --git a/apps/hash-frontend/src/components/grid/grid.tsx b/apps/hash-frontend/src/components/grid/grid.tsx index 99b89095c6c..ca30f6ed4e6 100644 --- a/apps/hash-frontend/src/components/grid/grid.tsx +++ b/apps/hash-frontend/src/components/grid/grid.tsx @@ -10,7 +10,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { gridRowHeight } from "@local/hash-isomorphic-utils/data-grid"; -import { getCellHorizontalPadding } from "./utils"; +import { getCellHorizontalPadding, gridHeaderBaseFont } from "./utils"; import { ColumnFilterMenu } from "./utils/column-filter-menu"; import { ConversionMenu, @@ -136,8 +136,6 @@ const gridHeaderHeight = 42; export const gridHeaderHeightWithBorder = gridHeaderHeight + 1; -export const gridHeaderBaseFont = "600 14px Inter"; - export const gridHorizontalScrollbarHeight = 17; export const Grid = < diff --git a/apps/hash-frontend/src/components/grid/utils.ts b/apps/hash-frontend/src/components/grid/utils.ts index c7ee639b82f..c5861170fbf 100644 --- a/apps/hash-frontend/src/components/grid/utils.ts +++ b/apps/hash-frontend/src/components/grid/utils.ts @@ -2,6 +2,9 @@ import { getMiddleCenterBias, GridCellKind } from "@glideapps/glide-data-grid"; import type { CustomCell, DrawArgs, Theme } from "@glideapps/glide-data-grid"; +/** The font a grid draws its column headers in. */ +export const gridHeaderBaseFont = "600 14px Inter"; + /** * @returns vertical center of a grid cell, relative to the visible grid area */ diff --git a/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx b/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx index c06d1178bed..0ff423a1e63 100644 --- a/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx +++ b/apps/hash-frontend/src/pages/shared/entities-visualizer.tsx @@ -195,12 +195,12 @@ export const EntitiesVisualizer: FunctionComponent<{ createDefaultFilterState(internalWebs.map(({ webId }) => webId)), ); + // The table query owns its own cursor: it drops it when the request changes, + // so only the subgraph path needs one here. const [subgraphCursor, setSubgraphCursor] = useState(); - const [tableCursor, setTableCursor] = useState(); const resetCursors = useCallback(() => { setSubgraphCursor(undefined); - setTableCursor(undefined); }, []); const [activeConversionsWithoutTitle, _setActiveConversions] = useState<{ [columnBaseUrl: BaseUrl]: VersionedUrl; @@ -328,7 +328,6 @@ export const EntitiesVisualizer: FunctionComponent<{ const tableQuery = useEntitiesTableQuery({ conversions, - cursor: tableCursor, enabled: usesTableEndpoint, filterState, hideArchivedColumn: !filterState.includeArchived, @@ -656,11 +655,11 @@ export const EntitiesVisualizer: FunctionComponent<{ const nextPage = useCallback(() => { if (usesTableEndpoint) { - setTableCursor(tableQuery.cursor ?? undefined); + tableQuery.loadMore(); } else { setSubgraphCursor(nextCursor ?? undefined); } - }, [usesTableEndpoint, tableQuery.cursor, nextCursor]); + }, [usesTableEndpoint, tableQuery, nextCursor]); const selectedEntities = useMemo(() => { if (view !== "Table") { @@ -685,10 +684,10 @@ export const EntitiesVisualizer: FunctionComponent<{ }, [selectedTableRows, view]); const handleBulkActionCompleted = useCallback(() => { - // A rejected refetch needs no handling here — the failure comes back - // through the hook's error state. + // A rejected read needs no handling here — the failure comes back through + // the hook's error state. if (usesTableEndpoint) { - tableQuery.refetch().catch(() => {}); + tableQuery.restart().catch(() => {}); } else { entitiesData.refetch().catch(() => {}); } @@ -798,12 +797,14 @@ export const EntitiesVisualizer: FunctionComponent<{ {errorIsRetryable ? (