DRAFT: Deferred index creation - do not merge

.gitignore

@@ -14,3 +14,5 @@ target
 *.iml
 .idea
 **/ivy-ide-settings.properties
+PLAN-*.md
+CLAUDE.md

morf-core/src/main/java/org/alfasoftware/morf/guicesupport/MorfModule.java

@@ -70,10 +70,11 @@ public Upgrade provideUpgrade(ConnectionResources connectionResources,
       ViewDeploymentValidator viewDeploymentValidator,
       DatabaseUpgradePathValidationService databaseUpgradePathValidationService,
       GraphBasedUpgradeBuilderFactory graphBasedUpgradeBuilderFactory,
-      UpgradeConfigAndContext upgradeConfigAndContext) {
+      UpgradeConfigAndContext upgradeConfigAndContext,
+      org.alfasoftware.morf.upgrade.deferred.DeferredIndexReadinessCheck deferredIndexReadinessCheck) {
     return new Upgrade(connectionResources, factory, upgradeStatusTableService, viewChangesDeploymentHelper,
       viewDeploymentValidator, databaseUpgradePathValidationService, graphBasedUpgradeBuilderFactory,
-      upgradeConfigAndContext);
+      upgradeConfigAndContext, deferredIndexReadinessCheck);
   }
 }
 

morf-core/src/main/java/org/alfasoftware/morf/jdbc/SqlDialect.java

@@ -4047,6 +4047,40 @@ public Collection<String> addIndexStatements(Table table, Index index) {
   }
 
 
+  /**
+   * Whether this dialect supports deferred index creation. When {@code true},
+   * {@link org.alfasoftware.morf.upgrade.SchemaEditor#addIndexDeferred} queues
+   * the index for background creation. When {@code false}, deferred requests
+   * are silently converted to immediate index creation, because the platform's
+   * {@code CREATE INDEX} blocks DML and deferring would move the lock from the
+   * upgrade window (when no traffic is flowing) to post-startup (when it is).
+   *
+   * <p>The default returns {@code false}. Dialects that support non-blocking
+   * DDL (e.g. PostgreSQL {@code CONCURRENTLY}, Oracle {@code ONLINE}) should
+   * override this to return {@code true}.</p>
+   *
+   * @return {@code true} if deferred index creation is beneficial on this platform.
+   */
+  public boolean supportsDeferredIndexCreation() {
+    return false;
+  }
+
+
+  /**
+   * Generates the SQL to build a deferred index on an existing table. By default this
+   * delegates to {@link #addIndexStatements(Table, Index)}, which issues a standard
+   * {@code CREATE INDEX} statement. Platform-specific dialects may override this method
+   * to emit non-blocking variants (e.g. {@code CREATE INDEX CONCURRENTLY} on PostgreSQL).
+   *
+   * @param table The existing table.
+   * @param index The new index to build in the background.
+   * @return A collection of SQL statements.
+   */
+  public Collection<String> deferredIndexDeploymentStatements(Table table, Index index) {
+    return addIndexStatements(table, index);
+  }
+
+
   /**
    * Helper method to create all index statements defined for a table
    *
@@ -4070,13 +4104,31 @@ protected List<String> createAllIndexStatements(Table table) {
    * @return The SQL to deploy the index on the table.
    */
   protected Collection<String> indexDeploymentStatements(Table table, Index index) {
+    return ImmutableList.of(buildCreateIndexStatement(table, index, ""));
+  }
+
+
+  /**
+   * Builds a {@code CREATE [UNIQUE] INDEX} statement with an optional keyword
+   * inserted between {@code INDEX} and the index name (e.g. {@code "CONCURRENTLY"}).
+   *
+   * @param table The table to create the index on.
+   * @param index The index to create.
+   * @param afterIndexKeyword keyword to insert after {@code INDEX}, or empty string for none.
+   * @return the complete CREATE INDEX SQL string.
+   */
+  protected String buildCreateIndexStatement(Table table, Index index, String afterIndexKeyword) {
     StringBuilder statement = new StringBuilder();
 
     statement.append("CREATE ");
     if (index.isUnique()) {
       statement.append("UNIQUE ");
     }
-    statement.append("INDEX ")
+    statement.append("INDEX ");
+    if (!afterIndexKeyword.isEmpty()) {
+      statement.append(afterIndexKeyword).append(' ');
+    }
+    statement
       .append(schemaNamePrefix(table))
       .append(index.getName())
       .append(" ON ")
@@ -4086,7 +4138,7 @@ protected Collection<String> indexDeploymentStatements(Table table, Index index)
       .append(Joiner.on(", ").join(index.columnNames()))
       .append(')');
 
-    return ImmutableList.of(statement.toString());
+    return statement.toString();
   }
 
 

morf-core/src/main/java/org/alfasoftware/morf/upgrade/AbstractSchemaChangeVisitor.java

@@ -1,13 +1,18 @@
+
 package org.alfasoftware.morf.upgrade;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Optional;
 
 import org.alfasoftware.morf.jdbc.SqlDialect;
 import org.alfasoftware.morf.metadata.Index;
 import org.alfasoftware.morf.metadata.Schema;
 import org.alfasoftware.morf.metadata.Table;
 import org.alfasoftware.morf.sql.Statement;
+import org.alfasoftware.morf.upgrade.deferred.DeferredAddIndex;
+import org.alfasoftware.morf.upgrade.deferred.DeferredIndexChangeService;
+import org.alfasoftware.morf.upgrade.deferred.DeferredIndexChangeServiceImpl;
 
 /**
  * Common code between SchemaChangeVisitor implementors
@@ -20,6 +25,7 @@ public abstract class AbstractSchemaChangeVisitor implements SchemaChangeVisitor
   protected final Table idTable;
   protected final TableNameResolver  tracker;
 
+  private final DeferredIndexChangeService deferredIndexChangeService = new DeferredIndexChangeServiceImpl();
 
   public AbstractSchemaChangeVisitor(Schema currentSchema, UpgradeConfigAndContext upgradeConfigAndContext, SqlDialect sqlDialect,
                                      Table idTable) {
@@ -66,6 +72,7 @@ public void visit(AddTable addTable) {
   @Override
   public void visit(RemoveTable removeTable) {
     currentSchema = removeTable.apply(currentSchema);
+    deferredIndexChangeService.cancelAllPendingForTable(removeTable.getTable().getName()).forEach(this::visitStatement);
     writeStatements(sqlDialect.dropStatements(removeTable.getTable()));
   }
 
@@ -80,37 +87,57 @@ public void visit(AddColumn addColumn) {
   @Override
   public void visit(ChangeColumn changeColumn) {
     currentSchema = changeColumn.apply(currentSchema);
+    deferredIndexChangeService.updatePendingColumnName(changeColumn.getTableName(), changeColumn.getFromColumn().getName(), changeColumn.getToColumn().getName()).forEach(this::visitStatement);
     writeStatements(sqlDialect.alterTableChangeColumnStatements(currentSchema.getTable(changeColumn.getTableName()), changeColumn.getFromColumn(), changeColumn.getToColumn()));
   }
 
 
   @Override
   public void visit(RemoveColumn removeColumn) {
     currentSchema = removeColumn.apply(currentSchema);
+    deferredIndexChangeService.cancelPendingReferencingColumn(removeColumn.getTableName(), removeColumn.getColumnDefinition().getName()).forEach(this::visitStatement);
     writeStatements(sqlDialect.alterTableDropColumnStatements(currentSchema.getTable(removeColumn.getTableName()), removeColumn.getColumnDefinition()));
   }
 
 
   @Override
   public void visit(RemoveIndex removeIndex) {
     currentSchema = removeIndex.apply(currentSchema);
-    writeStatements(sqlDialect.indexDropStatements(currentSchema.getTable(removeIndex.getTableName()), removeIndex.getIndexToBeRemoved()));
+    String tableName = removeIndex.getTableName();
+    String indexName = removeIndex.getIndexToBeRemoved().getName();
+    if (deferredIndexChangeService.hasPendingDeferred(tableName, indexName)) {
+      deferredIndexChangeService.cancelPending(tableName, indexName).forEach(this::visitStatement);
+    } else {
+      writeStatements(sqlDialect.indexDropStatements(currentSchema.getTable(tableName), removeIndex.getIndexToBeRemoved()));
+    }
   }
 
 
   @Override
   public void visit(ChangeIndex changeIndex) {
     currentSchema = changeIndex.apply(currentSchema);
-    writeStatements(sqlDialect.indexDropStatements(currentSchema.getTable(changeIndex.getTableName()), changeIndex.getFromIndex()));
-    writeStatements(sqlDialect.addIndexStatements(currentSchema.getTable(changeIndex.getTableName()), changeIndex.getToIndex()));
+    String tableName = changeIndex.getTableName();
+    Optional<DeferredAddIndex> existing = deferredIndexChangeService.getPendingDeferred(tableName, changeIndex.getFromIndex().getName());
+    if (existing.isPresent()) {
+      deferredIndexChangeService.cancelPending(tableName, changeIndex.getFromIndex().getName()).forEach(this::visitStatement);
+      deferredIndexChangeService.trackPending(new DeferredAddIndex(existing.get().getTableName(), changeIndex.getToIndex(), existing.get().getUpgradeUUID())).forEach(this::visitStatement);
+    } else {
+      writeStatements(sqlDialect.indexDropStatements(currentSchema.getTable(tableName), changeIndex.getFromIndex()));
+      writeStatements(sqlDialect.addIndexStatements(currentSchema.getTable(tableName), changeIndex.getToIndex()));
+    }
   }
 
 
   @Override
   public void visit(final RenameIndex renameIndex) {
     currentSchema = renameIndex.apply(currentSchema);
-    writeStatements(sqlDialect.renameIndexStatements(currentSchema.getTable(renameIndex.getTableName()),
-      renameIndex.getFromIndexName(), renameIndex.getToIndexName()));
+    String tableName = renameIndex.getTableName();
+    if (deferredIndexChangeService.hasPendingDeferred(tableName, renameIndex.getFromIndexName())) {
+      deferredIndexChangeService.updatePendingIndexName(tableName, renameIndex.getFromIndexName(), renameIndex.getToIndexName()).forEach(this::visitStatement);
+    } else {
+      writeStatements(sqlDialect.renameIndexStatements(currentSchema.getTable(tableName),
+        renameIndex.getFromIndexName(), renameIndex.getToIndexName()));
+    }
   }
 
 
@@ -120,6 +147,7 @@ public void visit(RenameTable renameTable) {
     currentSchema = renameTable.apply(currentSchema);
     Table newTable = currentSchema.getTable(renameTable.getNewTableName());
 
+    deferredIndexChangeService.updatePendingTableName(renameTable.getOldTableName(), renameTable.getNewTableName()).forEach(this::visitStatement);
     writeStatements(sqlDialect.renameTableStatements(oldTable, newTable));
   }
 
@@ -196,6 +224,22 @@ private void visitPortableSqlStatement(PortableSqlStatement sql) {
   }
 
 
+  /**
+   * @see org.alfasoftware.morf.upgrade.SchemaChangeVisitor#visit(org.alfasoftware.morf.upgrade.deferred.DeferredAddIndex)
+   */
+  @Override
+  public void visit(DeferredAddIndex deferredAddIndex) {
+    if (!sqlDialect.supportsDeferredIndexCreation()) {
+      // Dialect does not support deferred index creation — fall back to
+      // building the index immediately during the upgrade.
+      visit(new AddIndex(deferredAddIndex.getTableName(), deferredAddIndex.getNewIndex()));
+      return;
+    }
+    currentSchema = deferredAddIndex.apply(currentSchema);
+    deferredIndexChangeService.trackPending(deferredAddIndex).forEach(this::visitStatement);
+  }
+
+
   /**
    * @see org.alfasoftware.morf.upgrade.SchemaChangeVisitor#visit(org.alfasoftware.morf.upgrade.AddIndex)
    */

morf-core/src/main/java/org/alfasoftware/morf/upgrade/HumanReadableStatementProducer.java

@@ -160,6 +160,12 @@ public void addIndex(String tableName, Index index) {
         consumer.schemaChange(HumanReadableStatementHelper.generateAddIndexString(tableName, index));
       }
 
+      /** @see org.alfasoftware.morf.upgrade.SchemaEditor#addIndexDeferred(java.lang.String, org.alfasoftware.morf.metadata.Index) **/
+      @Override
+      public void addIndexDeferred(String tableName, Index index) {
+        consumer.schemaChange("Add index (deferred if supported): " + HumanReadableStatementHelper.generateAddIndexString(tableName, index));
+      }
+
       /** @see org.alfasoftware.morf.upgrade.SchemaEditor#addTable(org.alfasoftware.morf.metadata.Table) **/
       @Override
       public void addTable(Table definition) {

morf-core/src/main/java/org/alfasoftware/morf/upgrade/SchemaChangeAdaptor.java

@@ -1,5 +1,7 @@
 package org.alfasoftware.morf.upgrade;
 
+import org.alfasoftware.morf.upgrade.deferred.DeferredAddIndex;
+
 /**
  * Interface for adapting schema changes, i.e. {@link SchemaChange} implementations.
  *
@@ -169,6 +171,16 @@ public default RemoveSequence adapt(RemoveSequence removeSequence) {
   }
 
 
+  /**
+   * Perform adapt operation on a {@link DeferredAddIndex} instance.
+   *
+   * @param deferredAddIndex instance of {@link DeferredAddIndex} to adapt.
+   */
+  public default DeferredAddIndex adapt(DeferredAddIndex deferredAddIndex) {
+    return deferredAddIndex;
+  }
+
+
   /**
    * Simply uses the default implementation, which is already no-op.
    * By no-op, we mean non-changing: the input is passed through as output.
@@ -190,22 +202,22 @@ public Combining(SchemaChangeAdaptor first, SchemaChangeAdaptor second) {
       this.second = second;
     }
 
-    @Override
+    @Override
     public AddColumn adapt(AddColumn addColumn) {
       return second.adapt(first.adapt(addColumn));
     }
 
-    @Override
+    @Override
     public AddTable adapt(AddTable addTable) {
       return second.adapt(first.adapt(addTable));
     }
 
-    @Override
+    @Override
     public RemoveTable adapt(RemoveTable removeTable) {
       return second.adapt(first.adapt(removeTable));
     }
 
-    @Override
+    @Override
     public AddIndex adapt(AddIndex addIndex) {
       return second.adapt(first.adapt(addIndex));
     }
@@ -269,5 +281,13 @@ public AddSequence adapt(AddSequence addSequence) {
     public RemoveSequence adapt(RemoveSequence removeSequence) {
       return second.adapt(first.adapt(removeSequence));
     }
+
+    /**
+     * @see org.alfasoftware.morf.upgrade.SchemaChangeAdaptor#adapt(org.alfasoftware.morf.upgrade.deferred.DeferredAddIndex)
+     */
+    @Override
+    public DeferredAddIndex adapt(DeferredAddIndex deferredAddIndex) {
+      return second.adapt(first.adapt(deferredAddIndex));
+    }
   }
 }