Improve tree search efficiency

specifyweb/frontend/js_src/lib/components/Molecules/AutoComplete.tsx

@@ -79,6 +79,9 @@ export function AutoComplete<T>({
   inputProps = {},
   value: currentValue,
   pendingValueRef,
+  onScrollEnd,
+  isLoadingMore = false,
+  extraItems,
 }: {
   readonly source:
     | RA<AutoCompleteItem<T>>
@@ -113,6 +116,9 @@ export function AutoComplete<T>({
    * typing
    */
   readonly pendingValueRef?: React.MutableRefObject<string>;
+  readonly onScrollEnd?: () => void;
+  readonly isLoadingMore?: boolean;
+  readonly extraItems?: RA<AutoCompleteItem<T>>;
 }): JSX.Element {
   const [results, setResults] = React.useState<
     RA<AutoCompleteItem<T>> | undefined
@@ -235,8 +241,17 @@ export function AutoComplete<T>({
    * only one element that starts with the current value (the current element),
    * thus the filtered list of items has only one item.
    */
+  // Append paginated extra items to the base results
+  const allResults = React.useMemo(
+    () =>
+      extraItems !== undefined && extraItems.length > 0
+        ? [...(results ?? []), ...extraItems]
+        : results,
+    [results, extraItems]
+  );
+
   const ignoreFilter = currentValue === pendingValue;
-  const itemSource = ignoreFilter ? (results ?? []) : filteredItems;
+  const itemSource = ignoreFilter ? (allResults ?? []) : filteredItems;
 
   const pendingItem = results?.find(
     ({ label, searchValue }) => (searchValue ?? label) === pendingValue
@@ -430,6 +445,16 @@ export function AutoComplete<T>({
             shadow-gray-400 dark:border dark:border-gray-500 dark:bg-neutral-900
           `}
           ref={dataListRefCallback}
+          onScroll={
+            typeof onScrollEnd === 'function'
+              ? (event: React.UIEvent<HTMLUListElement>): void => {
+                  const target = event.currentTarget;
+                  const nearBottom =
+                    target.scrollHeight - target.scrollTop - target.clientHeight < 40;
+                  if (nearBottom) onScrollEnd();
+                }
+              : undefined
+          }
         >
           {isLoading && (
             <Combobox.Option<'li'>
@@ -527,7 +552,12 @@ export function AutoComplete<T>({
               )}
             </Combobox.Option>
           )}
-          {!listHasItems && (
+          {isLoadingMore && (
+            <li className={`${optionClassName(false, false)} cursor-auto text-gray-500`}>
+              {commonText.loading()}
+            </li>
+          )}
+          {!listHasItems && !isLoadingMore && (
             <div className={`${optionClassName(false, false)} cursor-auto`}>
               {formsText.nothingFound()}
             </div>

specifyweb/frontend/js_src/lib/components/Preferences/UserDefinitions.tsx

@@ -1189,7 +1189,7 @@ export const userPreferenceDefinitions = {
             title: preferencesText.treeSearchAlgorithm(),
             requiresReload: false,
             visible: true,
-            defaultValue: 'contains',
+            defaultValue: 'startsWith',
             values: [
               {
                 value: 'startsWith',

specifyweb/frontend/js_src/lib/components/QueryComboBox/__tests__/treeSearchEfficiency.test.tsx

@@ -0,0 +1,79 @@
+/**
+ * Tests for tree search efficiency improvements (#7752).
+ *
+ * Tree tables (Taxon, Geography, Storage, etc.) can have 200K+ rows.
+ * Using LIKE '%pattern%' causes full table scans; LIKE 'pattern%' can use
+ * B-tree indexes. The typeahead also doesn't need 1000 results — 50 is
+ * more than enough for a dropdown.
+ */
+import { requireContext } from '../../../tests/helpers';
+import { serializeResource } from '../../DataModel/serializers';
+import { tables } from '../../DataModel/tables';
+import { queryFieldFilterSpecs } from '../../QueryBuilder/FieldFilterSpec';
+import { makeComboBoxQuery } from '../helpers';
+import { QUERY_COMBO_BOX_PAGE_SIZE } from '../index';
+
+requireContext();
+
+describe('Tree search efficiency', () => {
+  test('makeComboBoxQuery for tree tables uses startsWith by default', () => {
+    const query = makeComboBoxQuery({
+      fieldName: 'fullName',
+      value: 'rosa',
+      isTreeTable: true,
+      typeSearch: {
+        table: tables.Taxon,
+        searchFields: [[tables.Taxon.strictGetLiteralField('fullName')]],
+        name: 'Taxon',
+        title: 'Taxon',
+        formatter: '',
+        displayFields: undefined,
+        format: '%s',
+      },
+      specialConditions: [],
+    });
+
+    const serialized = serializeResource(query);
+    const searchField = serialized.fields.find(
+      (field: { readonly isDisplay: boolean }) => !field.isDisplay
+    );
+
+    // startsWith operator (id 15), NOT like (id 0)
+    expect(searchField?.operStart).toBe(queryFieldFilterSpecs.startsWith.id);
+    // Value should NOT be wrapped in % wildcards
+    expect(searchField?.startValue).toBe('rosa');
+  });
+
+  test('makeComboBoxQuery for non-tree tables uses startsWith by default', () => {
+    const query = makeComboBoxQuery({
+      fieldName: 'lastName',
+      value: 'smith',
+      isTreeTable: false,
+      typeSearch: {
+        table: tables.Agent,
+        searchFields: [[tables.Agent.strictGetLiteralField('lastName')]],
+        name: 'Agent',
+        title: 'Agent',
+        formatter: '',
+        displayFields: undefined,
+        format: '%s',
+      },
+      specialConditions: [],
+    });
+
+    const serialized = serializeResource(query);
+    const searchField = serialized.fields.find(
+      (field: { readonly isDisplay: boolean }) => !field.isDisplay
+    );
+
+    // Non-tree tables also default to startsWith
+    expect(searchField?.operStart).toBe(queryFieldFilterSpecs.startsWith.id);
+    expect(searchField?.startValue).toBe('smith');
+  });
+
+  test('search limit is at most 50', () => {
+    // The exported constant from index.tsx controls runQuery limit.
+    // If someone bumps it back to 1000, this test catches it.
+    expect(QUERY_COMBO_BOX_PAGE_SIZE).toBeLessThanOrEqual(50);
+  });
+});

specifyweb/frontend/js_src/lib/components/QueryComboBox/index.tsx

@@ -55,6 +55,9 @@ import { useTreeData } from './useTreeData';
 import { TreeDefinitionContext } from './useTreeData';
 import { useTypeSearch } from './useTypeSearch';
 
+// Results are fetched in pages of this size and loaded on scroll.
+export const QUERY_COMBO_BOX_PAGE_SIZE = 50;
+
 /*
  * REFACTOR: split this component
  * TEST: add tests for this
@@ -309,87 +312,139 @@ export function QueryComboBox({
   const parentTreeDefinition = React.useContext(TreeDefinitionContext);
   const treeDefinition = fetchedTreeDefinition ?? parentTreeDefinition;
 
-  // FEATURE: use main table field if type search is not defined
-  const fetchSource = React.useCallback(
-    async (value: string): Promise<RA<AutoCompleteItem<string>>> =>
-      isLoaded && typeof typeSearch === 'object' && typeof resource === 'object'
-        ? Promise.all(
-            typeSearch.searchFields
-              .map((fields) =>
-                makeComboBoxQuery({
+  // Pagination state for batch-on-scroll loading
+  const [isLoadingMore, setIsLoadingMore] = React.useState(false);
+  const paginationRef = React.useRef({
+    hasMore: true,
+    offset: 0,
+    queries: [] as RA<ReturnType<typeof serializeResource>>,
+  });
+
+  const makeQueries = React.useCallback(
+    (value: string) =>
+      typeof typeSearch === 'object' && typeof resource === 'object'
+        ? typeSearch.searchFields
+            .map((fields) =>
+              makeComboBoxQuery({
+                fieldName: fields
+                  .map(({ name }) => name)
+                  .join(backboneFieldSeparator),
+                value,
+                isTreeTable: isTreeTable(field.relatedTable.name),
+                typeSearch,
+                specialConditions: getQueryComboBoxConditions({
+                  resource,
                   fieldName: fields
                     .map(({ name }) => name)
                     .join(backboneFieldSeparator),
-                  value,
-                  isTreeTable: isTreeTable(field.relatedTable.name),
-                  typeSearch,
-                  specialConditions: getQueryComboBoxConditions({
-                    resource,
-                    fieldName: fields
-                      .map(({ name }) => name)
-                      .join(backboneFieldSeparator),
-                    collectionRelationships:
-                      typeof collectionRelationships === 'object'
-                        ? collectionRelationships
-                        : undefined,
-                    treeData:
-                      typeof treeData === 'object' ? treeData : undefined,
-                    relatedTable,
-                    subViewRelationship,
-                    treeDefinition,
-                  }),
-                })
-              )
-              .map(serializeResource)
-              .map((query) => ({
-                ...query,
-                fields: query.fields.map((field, index) => ({
-                  ...field,
-                  position: index,
-                })),
-              }))
-              .map(async (query) =>
-                runQuery<readonly [id: number, label: LocalizedString]>(query, {
-                  collectionId: forceCollection ?? relatedCollectionId,
-                  // REFACTOR: allow customizing these arbitrary limits
-                  limit: 1000,
-                })
-              )
-          ).then((responses) =>
-            /*
-             * If there are multiple search fields and both returns the
-             * same record, it may be presented in results twice. Would
-             * be fixed by using OR queries
-             * REFACTOR: refactor to use OR queries across fields once
-             *   supported
-             */
-            responses.flat().map(([id, label]) => ({
-              data: getResourceApiUrl(
-                field.isRelationship
-                  ? field.relatedTable.name
-                  : resource.specifyTable.name,
-                id
-              ),
-              label,
+                  collectionRelationships:
+                    typeof collectionRelationships === 'object'
+                      ? collectionRelationships
+                      : undefined,
+                  treeData:
+                    typeof treeData === 'object' ? treeData : undefined,
+                  relatedTable,
+                  subViewRelationship,
+                  treeDefinition,
+                }),
+              })
+            )
+            .map(serializeResource)
+            .map((query) => ({
+              ...query,
+              fields: query.fields.map((queryField, index) => ({
+                ...queryField,
+                position: index,
+              })),
             }))
-          )
         : [],
     [
       field,
-      isLoaded,
       typeSearch,
       relatedTable,
       subViewRelationship,
       collectionRelationships,
-      forceCollection,
-      relatedCollectionId,
       resource,
       treeData,
-      fetchedTreeDefinition,
-      parentTreeDefinition,
+      treeDefinition,
     ]
   );
 
+  const rowsToItems = React.useCallback(
+    (
+      responses: RA<RA<readonly [id: number, label: LocalizedString]>>
+    ): RA<AutoCompleteItem<string>> =>
+      responses.flat().map(([id, label]) => ({
+        data: getResourceApiUrl(
+          field.isRelationship
+            ? field.relatedTable.name
+            : resource!.specifyTable.name,
+          id
+        ),
+        label,
+      })),
+    [field, resource]
+  );
+
+  const runPage = React.useCallback(
+    async (offset: number) =>
+      Promise.all(
+        paginationRef.current.queries.map(async (query) =>
+          runQuery<readonly [id: number, label: LocalizedString]>(query, {
+            collectionId: forceCollection ?? relatedCollectionId,
+            limit: QUERY_COMBO_BOX_PAGE_SIZE,
+            offset,
+          })
+        )
+      ),
+    [forceCollection, relatedCollectionId]
+  );
+
+  // FEATURE: use main table field if type search is not defined
+  const fetchSource = React.useCallback(
+    async (value: string): Promise<RA<AutoCompleteItem<string>>> => {
+      if (!isLoaded || typeof typeSearch !== 'object' || typeof resource !== 'object')
+        return [];
+
+      // Reset pagination on new search
+      setExtraItems([]);
+      const queries = makeQueries(value);
+      paginationRef.current = { hasMore: true, offset: 0, queries };
+
+      const responses = await runPage(0);
+      const items = rowsToItems(responses);
+      paginationRef.current.offset = QUERY_COMBO_BOX_PAGE_SIZE;
+      paginationRef.current.hasMore = responses.some(
+        (r) => r.length >= QUERY_COMBO_BOX_PAGE_SIZE
+      );
+      return items;
+    },
+    [isLoaded, typeSearch, resource, makeQueries, rowsToItems, runPage]
+  );
+
+  const handleScrollEnd = React.useCallback((): void => {
+    if (!paginationRef.current.hasMore || isLoadingMore) return;
+
+    setIsLoadingMore(true);
+    void runPage(paginationRef.current.offset)
+      .then((responses) => {
+        const newItems = rowsToItems(responses);
+        paginationRef.current.offset += QUERY_COMBO_BOX_PAGE_SIZE;
+        paginationRef.current.hasMore = responses.some(
+          (r) => r.length >= QUERY_COMBO_BOX_PAGE_SIZE
+        );
+        return newItems;
+      })
+      .then((newItems) => {
+        setIsLoadingMore(false);
+        setExtraItems((prev) => [...prev, ...newItems]);
+      })
+      .catch(() => setIsLoadingMore(false));
+  }, [rowsToItems, runPage, isLoadingMore]);
+
+  // Extra items appended via scroll pagination
+  const [extraItems, setExtraItems] = React.useState<RA<AutoCompleteItem<string>>>([]);
+
   const canAdd =
     !RESTRICT_ADDING.has(field.relatedTable.name) &&
     hasTablePermission(field.relatedTable.name, 'create', targetCollectionId);
@@ -437,6 +492,8 @@ export function QueryComboBox({
             type: 'text',
             [titlePosition]: 'top',
           }}
+          extraItems={extraItems}
+          isLoadingMore={isLoadingMore}
           pendingValueRef={pendingValueRef}
           source={fetchSource}
           value={
@@ -452,6 +509,7 @@ export function QueryComboBox({
             updateValue(data);
           }}
           onCleared={(): void => updateValue('')}
+          onScrollEnd={handleScrollEnd}
           onNewValue={
             formType !== 'formTable' && canAdd
               ? (): void =>