Feature/ignore partition tables postgres

morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProvider.java

@@ -56,6 +56,7 @@
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 
 /**
@@ -116,6 +117,8 @@ public abstract class DatabaseMetaDataProvider implements Schema {
   private final LoadingCache<AName, Sequence> sequenceCache = CacheBuilder.newBuilder().build(CacheLoader.from(this::loadSequence));
 
   private final Supplier<Map<String, String>> databaseInformation = Suppliers.memoize(this::loadDatabaseInformation);
+  protected Supplier<ImmutableSet<String>> ignoredTables = Suppliers.memoize(this::loadIgnoredTables);
+  protected Supplier<ImmutableSet<String>> partitionedTables = Suppliers.memoize(this::loadPartitionedTables);
 
   /**
    * @param connection The database connection from which meta data should be provided.
@@ -148,6 +151,9 @@ private Map<String, String> loadDatabaseInformation() {
     }
   }
 
+  protected ImmutableSet<String> loadIgnoredTables() { return ImmutableSet.of(); }
+
+  protected ImmutableSet<String> loadPartitionedTables() { return ImmutableSet.of(); }
 
   /**
    * @see org.alfasoftware.morf.metadata.Schema#isEmptyDatabase()
@@ -306,6 +312,11 @@ protected Map<AName, RealName> loadAllTableNames() {
             throw new RuntimeSqlException("Error reading metadata for table ["+tableName+"]", e);
           }
         }
+        // add partitioned tables to list
+        partitionedTables.get().forEach(table -> {
+          RealName partionedTableName = createRealName(table, table);
+          tableNameMappings.put(partionedTableName, partionedTableName);
+        });
 
         long end = System.currentTimeMillis();
         Map<AName, RealName> tableNameMap = tableNameMappings.build();

morf-core/src/main/java/org/alfasoftware/morf/metadata/AdditionalMetadata.java

@@ -1,5 +1,6 @@
 package org.alfasoftware.morf.metadata;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -18,6 +19,26 @@ default Map<String, String> primaryKeyIndexNames() {
     throw new NotImplementedException("Not implemented yet.");
   }
 
+  /**
+   * Provides the names of all partitioned tables in the database. This applies for now for postgres. Note that the order of
+   * the tables in the result is not specified. The case of the
+   * table names may be preserved when logging progress, but should not be relied on for schema
+   * processing. A partitioned table is a table that has partitions.
+   *
+   * @return A collection of all partitioned table names available in the database.
+   */
+  default Collection<String> partitionedTableNames() { throw new NotImplementedException("Not implemented yet."); }
+
+  /**
+   * Provides the names of all partition tables in the database. This applies for now for postgres. Note that the order of
+   * the tables in the result is not specified. The case of the
+   * table names may be preserved when logging progress, but should not be relied on for schema
+   * processing. A partition table is a table that is a partition of a partitioned table.
+   *
+   * @return A collection of all partition table names available in the database.
+   */
+  default Collection<String> partitionTableNames() { throw new NotImplementedException("Not implemented yet."); }
+
   default Map<String, List<Index>> ignoredIndexes() {
     return Map.of();
   }

morf-h2/src/test/java/org/alfasoftware/morf/jdbc/h2/TestH2MetaDataProvider.java

@@ -33,7 +33,6 @@
 import static org.mockito.Mockito.*;
 import static org.mockito.Mockito.when;
 
-
 /**
  * Test class for {@link H2MetaDataProvider}
  *

morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestDatabaseUpgradeIntegration.java

@@ -1398,7 +1398,8 @@ public void testUpdateOfMissingFieldThrowingException() {
         () -> verifyUpgrade(expected, List.of(UpdateId.class, DropPrimaryKey.class, UpdateMissingField.class, AddColumn.class, UpdateField.class)));
 
     assertThat(exception.getMessage(), containsString("Error executing SQL"));
-    assertThat(exception.getMessage(), containsString("UPDATE WithDefaultValue SET missingColumn"));
+    assertThat(exception.getMessage(), containsString("UPDATE"));
+    assertThat(exception.getMessage(), containsString("WithDefaultValue SET missingColumn"));
   }
 
 

morf-integration-test/src/test/java/org/alfasoftware/morf/jdbc/TestDatabaseMetaDataProvider.java

@@ -33,7 +33,9 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
 import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -67,6 +69,7 @@
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.google.inject.Inject;
 
 import net.jcip.annotations.NotThreadSafe;
@@ -131,7 +134,12 @@ public class TestDatabaseMetaDataProvider {
           column("bigIntegerCol", DataType.BIG_INTEGER).defaultValue("8"),
           column("booleanCol", DataType.BOOLEAN).defaultValue("1"),
           column("integerTenCol", DataType.INTEGER).defaultValue("17"),
-          column("dateCol", DataType.DATE).defaultValue("2020-01-01"))
+          column("dateCol", DataType.DATE).defaultValue("2020-01-01")),
+      table("WithPartition")
+        .columns(
+          SchemaUtils.idColumn(),
+          column("stringCol", DataType.STRING, 20)
+          )
     ),
     schema(
       view("ViewWithTypes", select(field("primaryStringCol"), field("id")).from("WithTypes").crossJoin(tableRef("WithDefaults"))),
@@ -208,6 +216,7 @@ public void testViewsAndTables() throws SQLException {
         tableNameEqualTo("WithTypes"),
         tableNameEqualTo("WithDefaults"),
         tableNameEqualTo("WithLobs"),
+        tableNameEqualTo("WithPartition"),
         equalToIgnoringCase("WithTimestamp") // can read table names even if they contain unsupported columns
       )));
 
@@ -216,6 +225,7 @@ public void testViewsAndTables() throws SQLException {
         tableNameMatcher("WithTypes"),
         tableNameMatcher("WithDefaults"),
         tableNameMatcher("WithLobs"),
+        tableNameMatcher("WithPartition"),
         propertyMatcher(Table::getName, "name", equalToIgnoringCase("WithTimestamp")) // can read table names even if they contain unsupported columns
       )));
     }
@@ -284,7 +294,7 @@ public void testTableWithTypes() throws SQLException {
       ));
 
       schemaResource.getAdditionalMetadata().ifPresent(additionalMetadata ->
-        assertThat(additionalMetadata.ignoredIndexes().get("WithTypes".toLowerCase()), containsInAnyOrder(ImmutableList.of(
+        assertThat(additionalMetadata.ignoredIndexes().get("WITHTYPES"), containsInAnyOrder(ImmutableList.of(
           indexMatcher(index("WithTypes_PRF1").columns("decimalNineFiveCol", "bigIntegerCol"))
       ))));
     }
@@ -319,6 +329,36 @@ public void testTableWithLobs() throws SQLException {
   }
 
 
+  @Test
+  public void testTableWithPartition() throws SQLException {
+    boolean isPostgres = databaseType.equals("PGSQL");
+    // RE-CREATE table with two partitions on table WithPartition
+    try (Connection connection = database.getDataSource().getConnection()) {
+      if (isPostgres) {
+        String tableSchema = Strings.isNullOrEmpty(database.getSchemaName()) ? "" : database.getSchemaName() + ".";
+        connection.createStatement().executeUpdate("DROP TABLE " + tableSchema + "WithPartition");
+        connection.createStatement().executeUpdate("CREATE TABLE " + tableSchema + "WithPartition(id numeric(19) NOT NULL, stringCol VARCHAR(20)) PARTITION BY RANGE (id)");
+        connection.createStatement().executeUpdate("CREATE TABLE " + tableSchema + "WithPartition_p0 PARTITION OF " + tableSchema + "WithPartition FOR VALUES FROM (0) TO (10000)");
+        connection.createStatement().executeUpdate("CREATE TABLE " + tableSchema + "WithPartition_p1 PARTITION OF " + tableSchema + "WithPartition FOR VALUES FROM (10000) TO (99999)");
+      }
+    }
+
+    try(SchemaResource schemaResource = database.openSchemaResource()) {
+      assertTrue(schemaResource.tableExists("WithPartition"));
+
+      if (isPostgres) {
+        UncheckedExecutionException uncheckedExecutionException = assertThrows(UncheckedExecutionException.class, () -> schemaResource.getTable("WithPartition_p0"));
+        assertTrue("partition must not be found on getTable", uncheckedExecutionException.getMessage().contains("Table [WithPartition_p0/*] not found."));
+
+        Table table = schemaResource.getTable("WithPartition");
+        assertEquals("table must have 2 columns", 2, table.columns().size());
+        assertEquals("first column must match", "id", table.columns().get(0).getName());
+        assertEquals("second column column must match", "stringcol", table.columns().get(1).getName());
+      }
+    }
+  }
+
+
   @Test
   public void testTableWithDefaults() throws SQLException {
     try(SchemaResource schemaResource = database.openSchemaResource()) {

morf-integration-test/src/test/java/org/alfasoftware/morf/upgrade/TestDatabaseUpgradePathValidationService.java

@@ -82,6 +82,8 @@ public void setup() {
   public void tearDown() {
     dropUpgradeStatusTable();
     schemaManager.invalidateCache();
+    // to make following test on test suite run clean - org.alfasoftware.morf.upgrade.TestFullDeployment
+    schemaManager.dropAllTables();
   }
 
 

morf-postgresql/src/main/java/org/alfasoftware/morf/jdbc/postgresql/PostgreSQLMetaDataProvider.java

@@ -9,8 +9,10 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Types;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -30,6 +32,7 @@
 
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 
 /**
  * Provides meta data from a PostgreSQL database connection.
@@ -51,6 +54,47 @@ public PostgreSQLMetaDataProvider(Connection connection, String schemaName) {
   }
 
 
+  @Override
+  protected ImmutableSet<String> loadIgnoredTables() {
+    ImmutableSet.Builder<String> ignoredTables = new ImmutableSet.Builder<>();
+    try(Statement ignoredTablesStmt = connection.createStatement()) {
+      // distinguish partitioned tables from regular ones: relkind = 'p' (partition) or 'r' (regular) also can use boolean col relispartition
+      // a partition table attached has (r, true)
+      try (ResultSet ignoredTablesRs = ignoredTablesStmt.executeQuery("select relname from pg_class where relispartition and relkind = 'r'")) {
+        while (ignoredTablesRs.next()) {
+          ignoredTables.add(ignoredTablesRs.getString(1).toLowerCase(Locale.ROOT));
+        }
+      }
+    } catch (SQLException e) {
+        // ignore exception, if it fails then incompatible Postgres version
+    }
+    return ignoredTables.build();
+  }
+
+  @Override
+  protected ImmutableSet<String> loadPartitionedTables() {
+    ImmutableSet.Builder<String> partitionedTables = new ImmutableSet.Builder<>();
+    try(Statement partitionedTablesStmt = connection.createStatement()) {
+      // distinguish partitioned tables from regular ones: relkind = 'p' (partition) or 'r' (regular) also can use boolean col relispartition
+      // a partition table attached has (r, true)
+      // a partition table has (p, false)
+      try (ResultSet ignoredTablesRs = partitionedTablesStmt.executeQuery("select relname from pg_class where not relispartition and relkind = 'p'")) {
+        while (ignoredTablesRs.next()) {
+          partitionedTables.add(ignoredTablesRs.getString(1).toLowerCase(Locale.ROOT));
+        }
+      }
+    } catch (SQLException e) {
+      // ignore exception, if it fails then incompatible Postgres version
+    }
+    return partitionedTables.build();
+  }
+
+
+  @Override
+  protected boolean isIgnoredTable(RealName tableName) {
+    return ignoredTables.get().contains(tableName.getDbName().toLowerCase(Locale.ROOT)) || super.isIgnoredTable(tableName);
+  }
+
   @Override
   protected boolean isPrimaryKeyIndex(RealName indexName) {
     return indexName.getDbName().endsWith("_pk");
@@ -233,4 +277,15 @@ protected String buildSequenceSql(String schemaName) {
 
     return sequenceSqlBuilder.toString();
   }
+
+
+  @Override
+  public Collection<String> partitionedTableNames() {
+    return super.partitionedTables.get();
+  }
+
+  @Override
+  public Collection<String> partitionTableNames() {
+    return super.ignoredTables.get();
+  }
 }

morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSQLDialect.java

@@ -1642,16 +1642,16 @@ protected String expectedSelectWithJoinAndLimit() {
     return "SELECT Test.id, Alternate.stringField FROM " + tableName(TEST_TABLE) + " INNER JOIN " + tableName("Alternate") + " ON (Test.id = Alternate.id) LIMIT 25";
   }
 
-  
+
   /**
    * @see AbstractSqlDialectTest#expectedPortableSqlExpression()
    */
   @Override
   protected String expectedPortableSqlExpression() {
     return "SELECT CONCAT(first_name, ' ', last_name, ' (', params->>'role', ')') FROM testschema.Test";
   }
-  
-  
+
+
    @Override
   protected String expectedSelectWithOrderByWhereAndLimit() {
     return "SELECT id, stringField FROM " + tableName(TEST_TABLE) + " WHERE (stringField IS NOT NULL) ORDER BY id DESC LIMIT 10";