DRAFT: Deferred index creation (comments-based) - do not merge

.gitignore

@@ -14,3 +14,5 @@ target
 *.iml
 .idea
 **/ivy-ide-settings.properties
+PLAN-*.md
+CLAUDE.md

PLAN-deferred-index-comments.md

@@ -0,0 +1,27 @@
+# Deferred Index Creation — Comments-Based Model
+
+See integration guide at: `~/deferred-index-comments-integration-guide.md`
+See dev description at: `~/deferred-index-comments-dev.txt`
+
+## Summary
+
+Replace the tracking-table approach (`DeferredIndexOperation` table + DAO + state machine)
+with a comments-based declarative model (~3600 net lines removed):
+
+- `Index.isDeferred()` — deferred is a property on the index itself
+- Table comments store permanent `DEFERRED:[name|cols|unique]` segments
+- MetaDataProvider merges comments with physical catalog: declared indexes always have `isDeferred()=true`
+- Executor scans for indexes that are `isDeferred()=true` but not physically present
+- `getMissingDeferredIndexStatements()` exposes raw SQL for custom execution
+- IF EXISTS DDL for safe operations on deferred indexes (built or not)
+- "Change the plan" — unbuilt deferred indexes modified in later upgrades without force-build
+
+## Branch
+
+`experimental/deferred-index-comments` (branched from `experimental/deferred-index-creation`)
+
+## Key Design Rule
+
+DEFERRED comment segments are **permanent** — never removed when the index is physically built.
+They are only modified during upgrade steps (removeIndex, changeIndex, renameIndex).
+The executor never writes comments.

morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProviderUtils.java

@@ -15,10 +15,20 @@
 
 package org.alfasoftware.morf.jdbc;
 
+import static org.alfasoftware.morf.metadata.SchemaUtils.index;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
+import org.alfasoftware.morf.metadata.Index;
+import org.alfasoftware.morf.metadata.SchemaUtils.IndexBuilder;
 import org.apache.commons.lang3.StringUtils;
 
 /**
@@ -65,6 +75,123 @@ public static Optional<String> getDataTypeFromColumnComment(String columnComment
   }
 
 
+  /**
+   * Label used in table comments to declare deferred indexes.
+   */
+  public static final String DEFERRED_COMMENT_LABEL = "DEFERRED";
+
+  /**
+   * Regex for extracting DEFERRED segments from table comments.
+   * Matches: /DEFERRED:[indexName|col1,col2|unique] or /DEFERRED:[indexName|col1,col2]
+   */
+  private static final Pattern DEFERRED_INDEX_REGEX = Pattern.compile("DEFERRED:\\[([^\\]]+)\\]");
+
+
+  /**
+   * Parses deferred index declarations from a table comment string.
+   * Each DEFERRED segment has the format: {@code DEFERRED:[indexName|col1,col2|unique]}
+   * where the {@code |unique} suffix is optional.
+   *
+   * @param tableComment the full table comment string, may be null or empty.
+   * @return a list of deferred indexes parsed from the comment, each with {@code isDeferred()=true}.
+   */
+  public static List<Index> parseDeferredIndexesFromComment(String tableComment) {
+    if (StringUtils.isEmpty(tableComment)) {
+      return Collections.emptyList();
+    }
+
+    List<Index> result = new ArrayList<>();
+    Matcher matcher = DEFERRED_INDEX_REGEX.matcher(tableComment);
+    while (matcher.find()) {
+      String content = matcher.group(1);
+      String[] parts = content.split("\\|");
+      if (parts.length < 2) {
+        continue;
+      }
+
+      String indexName = parts[0];
+      List<String> columns = Arrays.asList(parts[1].split(","));
+      boolean unique = parts.length >= 3 && "unique".equalsIgnoreCase(parts[2]);
+
+      IndexBuilder builder = index(indexName).columns(columns).deferred();
+      if (unique) {
+        builder = builder.unique();
+      }
+      result.add(builder);
+    }
+    return result;
+  }
+
+
+  /**
+   * Builds the DEFERRED comment segments for the given deferred indexes.
+   * Each index produces a segment like {@code /DEFERRED:[indexName|col1,col2|unique]}.
+   *
+   * @param deferredIndexes the deferred indexes to serialize.
+   * @return the concatenated DEFERRED segments, or an empty string if none.
+   */
+  public static String buildDeferredIndexCommentSegments(List<Index> deferredIndexes) {
+    if (deferredIndexes == null || deferredIndexes.isEmpty()) {
+      return "";
+    }
+
+    StringBuilder sb = new StringBuilder();
+    for (Index idx : deferredIndexes) {
+      sb.append("/").append(DEFERRED_COMMENT_LABEL).append(":[");
+      sb.append(idx.getName());
+      sb.append("|");
+      sb.append(idx.columnNames().stream().collect(Collectors.joining(",")));
+      if (idx.isUnique()) {
+        sb.append("|unique");
+      }
+      sb.append("]");
+    }
+    return sb.toString();
+  }
+
+
+  /**
+   * Merges deferred index declarations from a table comment with physical indexes
+   * loaded from the database catalog.
+   *
+   * <p>Physical indexes whose names match a deferred declaration are returned with
+   * {@code isDeferred()=true}. Deferred declarations with no physical counterpart
+   * are returned as virtual indexes (also with {@code isDeferred()=true}).</p>
+   *
+   * @param physicalIndexes the indexes loaded from the database catalog.
+   * @param tableComment the raw table comment string, may be null or empty.
+   * @return the merged index list, or {@code physicalIndexes} unchanged if
+   *         no deferred declarations are found.
+   */
+  public static List<Index> mergeDeferredIndexes(List<Index> physicalIndexes, String tableComment) {
+    List<Index> deferredFromComment = parseDeferredIndexesFromComment(tableComment);
+    if (deferredFromComment.isEmpty()) {
+      return physicalIndexes;
+    }
+
+    Set<String> deferredNames = deferredFromComment.stream()
+        .map(i -> i.getName().toUpperCase())
+        .collect(Collectors.toCollection(java.util.HashSet::new));
+
+    List<Index> merged = new ArrayList<>();
+    for (Index physical : physicalIndexes) {
+      if (deferredNames.contains(physical.getName().toUpperCase())) {
+        IndexBuilder builder = index(physical.getName()).columns(physical.columnNames()).deferred();
+        merged.add(physical.isUnique() ? builder.unique() : builder);
+        deferredNames.remove(physical.getName().toUpperCase());
+      } else {
+        merged.add(physical);
+      }
+    }
+    for (Index deferred : deferredFromComment) {
+      if (deferredNames.contains(deferred.getName().toUpperCase())) {
+        merged.add(deferred);
+      }
+    }
+    return merged;
+  }
+
+
   /**
    * Indexes which contain the suffix _PRF and a digit are to be ignored:
    * this allows performance testing of new index to verify their effect,

morf-core/src/main/java/org/alfasoftware/morf/jdbc/SqlDialect.java

@@ -29,6 +29,7 @@
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -307,7 +308,7 @@
    * @return SQL statements required to change a table name.
    */
   public Collection<String> renameTableStatements(Table from, Table to) {
     return ImmutableList.of("ALTER TABLE " + schemaNamePrefix(from) + from.getName() + " RENAME TO " + to.getName());
   }
 
 
@@ -3898,6 +3899,33 @@
   }
 
 
+  /**
+   * Generate the SQL to drop an index if it exists. Used for deferred indexes
+   * which may or may not have been physically built yet.
+   *
+   * @param table The table to drop the index from.
+   * @param indexToBeRemoved The index to be dropped.
+   * @return The SQL to drop the specified index if it exists.
+   */
+  public Collection<String> indexDropStatementsIfExists(@SuppressWarnings("unused") Table table, Index indexToBeRemoved) {
+    return ImmutableList.of("DROP INDEX IF EXISTS " + indexToBeRemoved.getName());
+  }
+
+
+  /**
+   * Generate the SQL to rename an index if it exists. Used for deferred indexes
+   * which may or may not have been physically built yet.
+   *
+   * @param table The table on which the index exists.
+   * @param fromIndexName The index to rename.
+   * @param toIndexName The new index name.
+   * @return The SQL to rename the index if it exists.
+   */
+  public Collection<String> renameIndexStatementsIfExists(Table table, String fromIndexName, String toIndexName) {
+    return ImmutableList.of("ALTER INDEX IF EXISTS " + schemaNamePrefix(table) + fromIndexName + " RENAME TO " + toIndexName);
+  }
+
+
   /**
    * Generates the SQL to create a table and insert the data specified in the {@link SelectStatement}.
    *
@@ -4047,6 +4075,104 @@
   }
 
 
+  /**
+   * Whether this dialect supports deferred index creation. When {@code true},
+   * indexes marked as deferred are queued for background creation via table
+   * comments. When {@code false}, deferred requests are silently converted to
+   * immediate index creation, because the platform's {@code CREATE INDEX} blocks
+   * DML and deferring would move the lock from the upgrade window (when no
+   * traffic is flowing) to post-startup (when it is).
+   *
+   * <p>The default returns {@code false}. Dialects that support non-blocking
+   * DDL (e.g. PostgreSQL {@code CONCURRENTLY}, Oracle {@code ONLINE}) should
+   * override this to return {@code true}.</p>
+   *
+   * @return {@code true} if deferred index creation is beneficial on this platform.
+   */
+  public boolean supportsDeferredIndexCreation() {
+    return false;
+  }
+
+
+  /**
+   * Generates the SQL to build a deferred index on an existing table. By default this
+   * delegates to {@link #addIndexStatements(Table, Index)}, which issues a standard
+   * {@code CREATE INDEX} statement. Platform-specific dialects may override this method
+   * to emit non-blocking variants (e.g. {@code CREATE INDEX CONCURRENTLY} on PostgreSQL).
+   *
+   * @param table The existing table.
+   * @param index The new index to build in the background.
+   * @return A collection of SQL statements.
+   */
+  public Collection<String> deferredIndexDeploymentStatements(Table table, Index index) {
+    return addIndexStatements(table, index);
+  }
+
+
+  /**
+   * Returns SQL to find table names that have DEFERRED index declarations in their
+   * table comments. Used by the executor to avoid a full schema scan on large databases.
+   *
+   * <p>The default implementation returns {@code null}, meaning the caller should
+   * fall back to a full schema scan. Dialects that use table comments override this
+   * to produce an efficient query against the database catalog.</p>
+   *
+   * @return SQL that returns a single-column result set of table names, or {@code null}
+   *         if the dialect does not support this optimization.
+   */
+  public String findTablesWithDeferredIndexesSql() {
+    return null;
+  }
+
+
+  /**
+   * Returns SQL to check whether a named index exists but is invalid (e.g. PostgreSQL's
+   * {@code indisvalid=false} after a crashed {@code CREATE INDEX CONCURRENTLY}). The SQL
+   * should return at least one row if the index is invalid, and no rows otherwise.
+   *
+   * <p>The default implementation returns {@code null}, meaning no invalid-index checking
+   * is performed. PostgreSQL overrides this.</p>
+   *
+   * @param indexName the index name to check.
+   * @return SQL that returns rows if the index is invalid, or {@code null} if not supported.
+   */
+  public String checkInvalidIndexSql(String indexName) {
+    return null;
+  }
+
+
+  /**
+   * Returns SQL to drop an invalid index before rebuilding it. Only called when
+   * {@link #checkInvalidIndexSql(String)} indicated the index is invalid.
+   *
+   * <p>The default implementation returns an empty list. PostgreSQL overrides this
+   * with {@code DROP INDEX IF EXISTS}.</p>
+   *
+   * @param indexName the index to drop.
+   * @return SQL statements to drop the invalid index.
+   */
+  public Collection<String> dropInvalidIndexStatements(String indexName) {
+    return Collections.emptyList();
+  }
+
+
+  /**
+   * Generates a {@code COMMENT ON TABLE} SQL statement that includes both the standard
+   * table metadata (e.g. REALNAME) and any deferred index declarations. The deferred index
+   * declarations are appended as {@code /DEFERRED:[indexName|col1,col2|unique]} segments.
+   *
+   * <p>The default implementation returns an empty list. Dialects that use table comments
+   * (PostgreSQL, Oracle, H2) override this to produce the appropriate SQL.</p>
+   *
+   * @param table The table to generate the comment for.
+   * @param deferredIndexes The deferred indexes to declare in the comment.
+   * @return A collection of SQL statements, or an empty list if the dialect does not support comments.
+   */
+  public Collection<String> generateTableCommentStatements(Table table, List<Index> deferredIndexes) {
+    return Collections.emptyList();
+  }
+
+
   /**
    * Helper method to create all index statements defined for a table
    *
@@ -4070,13 +4196,31 @@
    * @return The SQL to deploy the index on the table.
    */
   protected Collection<String> indexDeploymentStatements(Table table, Index index) {
+    return ImmutableList.of(buildCreateIndexStatement(table, index, ""));
+  }
+
+
+  /**
+   * Builds a {@code CREATE [UNIQUE] INDEX} statement with an optional keyword
+   * inserted between {@code INDEX} and the index name (e.g. {@code "CONCURRENTLY"}).
+   *
+   * @param table The table to create the index on.
+   * @param index The index to create.
+   * @param afterIndexKeyword keyword to insert after {@code INDEX}, or empty string for none.
+   * @return the complete CREATE INDEX SQL string.
+   */
+  protected String buildCreateIndexStatement(Table table, Index index, String afterIndexKeyword) {
     StringBuilder statement = new StringBuilder();
 
     statement.append("CREATE ");
     if (index.isUnique()) {
       statement.append("UNIQUE ");
     }
-    statement.append("INDEX ")
+    statement.append("INDEX ");
+    if (!afterIndexKeyword.isEmpty()) {
+      statement.append(afterIndexKeyword).append(' ');
+    }
+    statement
       .append(schemaNamePrefix(table))
       .append(index.getName())
       .append(" ON ")
@@ -4086,7 +4230,7 @@
       .append(Joiner.on(", ").join(index.columnNames()))
       .append(')');
 
-    return ImmutableList.of(statement.toString());
+    return statement.toString();
   }