diff --git a/specifyweb/frontend/js_src/lib/components/Molecules/AutoComplete.tsx b/specifyweb/frontend/js_src/lib/components/Molecules/AutoComplete.tsx index cf3efbe00d3..b449a6c1d05 100644 --- a/specifyweb/frontend/js_src/lib/components/Molecules/AutoComplete.tsx +++ b/specifyweb/frontend/js_src/lib/components/Molecules/AutoComplete.tsx @@ -79,6 +79,9 @@ export function AutoComplete({ inputProps = {}, value: currentValue, pendingValueRef, + onScrollEnd, + isLoadingMore = false, + extraItems, }: { readonly source: | RA> @@ -113,6 +116,9 @@ export function AutoComplete({ * typing */ readonly pendingValueRef?: React.MutableRefObject; + readonly onScrollEnd?: () => void; + readonly isLoadingMore?: boolean; + readonly extraItems?: RA>; }): JSX.Element { const [results, setResults] = React.useState< RA> | undefined @@ -235,8 +241,17 @@ export function AutoComplete({ * only one element that starts with the current value (the current element), * thus the filtered list of items has only one item. */ + // Append paginated extra items to the base results + const allResults = React.useMemo( + () => + extraItems !== undefined && extraItems.length > 0 + ? [...(results ?? []), ...extraItems] + : results, + [results, extraItems] + ); + const ignoreFilter = currentValue === pendingValue; - const itemSource = ignoreFilter ? (results ?? []) : filteredItems; + const itemSource = ignoreFilter ? (allResults ?? []) : filteredItems; const pendingItem = results?.find( ({ label, searchValue }) => (searchValue ?? label) === pendingValue @@ -430,6 +445,16 @@ export function AutoComplete({ shadow-gray-400 dark:border dark:border-gray-500 dark:bg-neutral-900 `} ref={dataListRefCallback} + onScroll={ + typeof onScrollEnd === 'function' + ? (event: React.UIEvent): void => { + const target = event.currentTarget; + const nearBottom = + target.scrollHeight - target.scrollTop - target.clientHeight < 40; + if (nearBottom) onScrollEnd(); + } + : undefined + } > {isLoading && ( @@ -527,7 +552,12 @@ export function AutoComplete({ )} )} - {!listHasItems && ( + {isLoadingMore && ( +
  • + {commonText.loading()} +
    • + )} + {!listHasItems && !isLoadingMore && (
    {formsText.nothingFound()}
    diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx index 9ce17ed4f47..c730a34d6a1 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx @@ -1189,7 +1189,7 @@ export const userPreferenceDefinitions = { title: preferencesText.treeSearchAlgorithm(), requiresReload: false, visible: true, - defaultValue: 'contains', + defaultValue: 'startsWith', values: [ { value: 'startsWith', diff --git a/specifyweb/frontend/js_src/lib/components/QueryComboBox/__tests__/treeSearchEfficiency.test.tsx b/specifyweb/frontend/js_src/lib/components/QueryComboBox/__tests__/treeSearchEfficiency.test.tsx new file mode 100644 index 00000000000..29294f6dee6 --- /dev/null +++ b/specifyweb/frontend/js_src/lib/components/QueryComboBox/__tests__/treeSearchEfficiency.test.tsx @@ -0,0 +1,79 @@ +/** + * Tests for tree search efficiency improvements (#7752). + * + * Tree tables (Taxon, Geography, Storage, etc.) can have 200K+ rows. + * Using LIKE '%pattern%' causes full table scans; LIKE 'pattern%' can use + * B-tree indexes. The typeahead also doesn't need 1000 results — 50 is + * more than enough for a dropdown. + */ +import { requireContext } from '../../../tests/helpers'; +import { serializeResource } from '../../DataModel/serializers'; +import { tables } from '../../DataModel/tables'; +import { queryFieldFilterSpecs } from '../../QueryBuilder/FieldFilterSpec'; +import { makeComboBoxQuery } from '../helpers'; +import { QUERY_COMBO_BOX_PAGE_SIZE } from '../index'; + +requireContext(); + +describe('Tree search efficiency', () => { + test('makeComboBoxQuery for tree tables uses startsWith by default', () => { + const query = makeComboBoxQuery({ + fieldName: 'fullName', + value: 'rosa', + isTreeTable: true, + typeSearch: { + table: tables.Taxon, + searchFields: [[tables.Taxon.strictGetLiteralField('fullName')]], + name: 'Taxon', + title: 'Taxon', + formatter: '', + displayFields: undefined, + format: '%s', + }, + specialConditions: [], + }); + + const serialized = serializeResource(query); + const searchField = serialized.fields.find( + (field: { readonly isDisplay: boolean }) => !field.isDisplay + ); + + // startsWith operator (id 15), NOT like (id 0) + expect(searchField?.operStart).toBe(queryFieldFilterSpecs.startsWith.id); + // Value should NOT be wrapped in % wildcards + expect(searchField?.startValue).toBe('rosa'); + }); + + test('makeComboBoxQuery for non-tree tables uses startsWith by default', () => { + const query = makeComboBoxQuery({ + fieldName: 'lastName', + value: 'smith', + isTreeTable: false, + typeSearch: { + table: tables.Agent, + searchFields: [[tables.Agent.strictGetLiteralField('lastName')]], + name: 'Agent', + title: 'Agent', + formatter: '', + displayFields: undefined, + format: '%s', + }, + specialConditions: [], + }); + + const serialized = serializeResource(query); + const searchField = serialized.fields.find( + (field: { readonly isDisplay: boolean }) => !field.isDisplay + ); + + // Non-tree tables also default to startsWith + expect(searchField?.operStart).toBe(queryFieldFilterSpecs.startsWith.id); + expect(searchField?.startValue).toBe('smith'); + }); + + test('search limit is at most 50', () => { + // The exported constant from index.tsx controls runQuery limit. + // If someone bumps it back to 1000, this test catches it. + expect(QUERY_COMBO_BOX_PAGE_SIZE).toBeLessThanOrEqual(50); + }); +}); diff --git a/specifyweb/frontend/js_src/lib/components/QueryComboBox/index.tsx b/specifyweb/frontend/js_src/lib/components/QueryComboBox/index.tsx index cb9c588a9c4..a1acc40216b 100644 --- a/specifyweb/frontend/js_src/lib/components/QueryComboBox/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/QueryComboBox/index.tsx @@ -55,6 +55,9 @@ import { useTreeData } from './useTreeData'; import { TreeDefinitionContext } from './useTreeData'; import { useTypeSearch } from './useTypeSearch'; +// Results are fetched in pages of this size and loaded on scroll. +export const QUERY_COMBO_BOX_PAGE_SIZE = 50; + /* * REFACTOR: split this component * TEST: add tests for this @@ -309,87 +312,139 @@ export function QueryComboBox({ const parentTreeDefinition = React.useContext(TreeDefinitionContext); const treeDefinition = fetchedTreeDefinition ?? parentTreeDefinition; - // FEATURE: use main table field if type search is not defined - const fetchSource = React.useCallback( - async (value: string): Promise>> => - isLoaded && typeof typeSearch === 'object' && typeof resource === 'object' - ? Promise.all( - typeSearch.searchFields - .map((fields) => - makeComboBoxQuery({ + // Pagination state for batch-on-scroll loading + const [isLoadingMore, setIsLoadingMore] = React.useState(false); + const paginationRef = React.useRef({ + hasMore: true, + offset: 0, + queries: [] as RA>, + }); + + const makeQueries = React.useCallback( + (value: string) => + typeof typeSearch === 'object' && typeof resource === 'object' + ? typeSearch.searchFields + .map((fields) => + makeComboBoxQuery({ + fieldName: fields + .map(({ name }) => name) + .join(backboneFieldSeparator), + value, + isTreeTable: isTreeTable(field.relatedTable.name), + typeSearch, + specialConditions: getQueryComboBoxConditions({ + resource, fieldName: fields .map(({ name }) => name) .join(backboneFieldSeparator), - value, - isTreeTable: isTreeTable(field.relatedTable.name), - typeSearch, - specialConditions: getQueryComboBoxConditions({ - resource, - fieldName: fields - .map(({ name }) => name) - .join(backboneFieldSeparator), - collectionRelationships: - typeof collectionRelationships === 'object' - ? collectionRelationships - : undefined, - treeData: - typeof treeData === 'object' ? treeData : undefined, - relatedTable, - subViewRelationship, - treeDefinition, - }), - }) - ) - .map(serializeResource) - .map((query) => ({ - ...query, - fields: query.fields.map((field, index) => ({ - ...field, - position: index, - })), - })) - .map(async (query) => - runQuery(query, { - collectionId: forceCollection ?? relatedCollectionId, - // REFACTOR: allow customizing these arbitrary limits - limit: 1000, - }) - ) - ).then((responses) => - /* - * If there are multiple search fields and both returns the - * same record, it may be presented in results twice. Would - * be fixed by using OR queries - * REFACTOR: refactor to use OR queries across fields once - * supported - */ - responses.flat().map(([id, label]) => ({ - data: getResourceApiUrl( - field.isRelationship - ? field.relatedTable.name - : resource.specifyTable.name, - id - ), - label, + collectionRelationships: + typeof collectionRelationships === 'object' + ? collectionRelationships + : undefined, + treeData: + typeof treeData === 'object' ? treeData : undefined, + relatedTable, + subViewRelationship, + treeDefinition, + }), + }) + ) + .map(serializeResource) + .map((query) => ({ + ...query, + fields: query.fields.map((queryField, index) => ({ + ...queryField, + position: index, + })), })) - ) : [], [ field, - isLoaded, typeSearch, relatedTable, subViewRelationship, collectionRelationships, - forceCollection, - relatedCollectionId, resource, treeData, - fetchedTreeDefinition, - parentTreeDefinition, + treeDefinition, ] ); + const rowsToItems = React.useCallback( + ( + responses: RA> + ): RA> => + responses.flat().map(([id, label]) => ({ + data: getResourceApiUrl( + field.isRelationship + ? field.relatedTable.name + : resource!.specifyTable.name, + id + ), + label, + })), + [field, resource] + ); + + const runPage = React.useCallback( + async (offset: number) => + Promise.all( + paginationRef.current.queries.map(async (query) => + runQuery(query, { + collectionId: forceCollection ?? relatedCollectionId, + limit: QUERY_COMBO_BOX_PAGE_SIZE, + offset, + }) + ) + ), + [forceCollection, relatedCollectionId] + ); + + // FEATURE: use main table field if type search is not defined + const fetchSource = React.useCallback( + async (value: string): Promise>> => { + if (!isLoaded || typeof typeSearch !== 'object' || typeof resource !== 'object') + return []; + + // Reset pagination on new search + setExtraItems([]); + const queries = makeQueries(value); + paginationRef.current = { hasMore: true, offset: 0, queries }; + + const responses = await runPage(0); + const items = rowsToItems(responses); + paginationRef.current.offset = QUERY_COMBO_BOX_PAGE_SIZE; + paginationRef.current.hasMore = responses.some( + (r) => r.length >= QUERY_COMBO_BOX_PAGE_SIZE + ); + return items; + }, + [isLoaded, typeSearch, resource, makeQueries, rowsToItems, runPage] + ); + + const handleScrollEnd = React.useCallback((): void => { + if (!paginationRef.current.hasMore || isLoadingMore) return; + + setIsLoadingMore(true); + void runPage(paginationRef.current.offset) + .then((responses) => { + const newItems = rowsToItems(responses); + paginationRef.current.offset += QUERY_COMBO_BOX_PAGE_SIZE; + paginationRef.current.hasMore = responses.some( + (r) => r.length >= QUERY_COMBO_BOX_PAGE_SIZE + ); + return newItems; + }) + .then((newItems) => { + setIsLoadingMore(false); + setExtraItems((prev) => [...prev, ...newItems]); + }) + .catch(() => setIsLoadingMore(false)); + }, [rowsToItems, runPage, isLoadingMore]); + + // Extra items appended via scroll pagination + const [extraItems, setExtraItems] = React.useState>>([]); + const canAdd = !RESTRICT_ADDING.has(field.relatedTable.name) && hasTablePermission(field.relatedTable.name, 'create', targetCollectionId); @@ -437,6 +492,8 @@ export function QueryComboBox({ type: 'text', [titlePosition]: 'top', }} + extraItems={extraItems} + isLoadingMore={isLoadingMore} pendingValueRef={pendingValueRef} source={fetchSource} value={ @@ -452,6 +509,7 @@ export function QueryComboBox({ updateValue(data); }} onCleared={(): void => updateValue('')} + onScrollEnd={handleScrollEnd} onNewValue={ formType !== 'formTable' && canAdd ? (): void =>