From 4bd59716ef8622e559999919339c7bb6102ec89d Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Wed, 15 Jan 2025 14:03:53 +0000 Subject: [PATCH 01/36] fix e2e workflow to run on release branches --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c710ef929..ab3be13ff 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,6 +65,7 @@ jobs: with: repository: cdapio/cdap-e2e-tests path: e2e + ref: release/6.11 - name: Cache uses: actions/cache@v4 From 10288ce385d33f56996c4e703c62ac925270be9a Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 27 Dec 2024 15:05:22 +0530 Subject: [PATCH 02/36] Add ExternalDocumentationLink in abstract sink --- .../cloudsql/mysql/CloudSQLMySQLSink.java | 5 +++ .../postgres/CloudSQLPostgreSQLSink.java | 5 +++ .../cdap/plugin/db/sink/AbstractDBSink.java | 31 +++++++++++++++++-- .../plugin/db/source/AbstractDBSource.java | 6 ++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 +++ .../io/cdap/plugin/postgres/PostgresSink.java | 5 +++ 6 files changed, 55 insertions(+), 2 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 86a8e6f52..da1414ff3 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -108,6 +108,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLMySQLErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index c8ca0d6dc..65b86366c 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -152,6 +152,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } + /** CloudSQL PostgreSQL sink config. */ public static class CloudSQLPostgreSQLSinkConfig extends AbstractDBSpecificSinkConfig { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 26a95405b..8d7264f1c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -25,6 +25,10 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -175,6 +179,16 @@ protected String getErrorDetailsProviderClassName() { return DBErrorDetailsProvider.class.getName(); } + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ + protected String getExternalDocumentationLink() { + return null; + } + @Override public void prepareRun(BatchSinkContext context) { String connectionString = dbSinkConfig.getConnectionString(); @@ -296,8 +310,21 @@ private Schema inferSchema(Class driverClass) { inferredFields.addAll(getSchemaReader().getSchemaFields(rs)); } } catch (SQLException e) { - throw new InvalidStageException("Error while reading table metadata", e); - + // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath + String errorMessageWithDetails = String.format("Error while reading table metadata." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessageWithDetails.endsWith(".")) { + errorMessageWithDetails = errorMessageWithDetails + "."; + } + errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, + externalDocumentationLink); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); } } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 559985758..70fc12e4b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -373,6 +373,12 @@ protected Class getDBRecordType() { return DBRecord.class; } + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ protected String getExternalDocumentationLink() { return null; } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 42488b31e..fee0a31fa 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -114,6 +114,11 @@ protected String getErrorDetailsProviderClassName() { return MysqlErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MYSQL_SUPPORTED_DOC_URL; + } + /** * MySQL action configuration. */ diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 3becf5f27..7682b8b0b 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -121,6 +121,11 @@ protected String getErrorDetailsProviderClassName() { return PostgresErrorDetailsProvider.class.getName(); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + /** * PostgreSQL action configuration. */ From 7fdc40ec89202e3e85bb888013bc466be29bb31e Mon Sep 17 00:00:00 2001 From: psainics Date: Thu, 23 Jan 2025 05:50:25 +0530 Subject: [PATCH 03/36] Return null for ErrorDetailsProvider by default --- .../main/java/io/cdap/plugin/db/sink/AbstractDBSink.java | 6 ++++-- .../java/io/cdap/plugin/db/source/AbstractDBSource.java | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 8d7264f1c..3252f70bd 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -176,7 +176,7 @@ public void validateOperations(FailureCollector collector, T dbSinkConfig, @Null * @return ErrorDetailsProvider class name */ protected String getErrorDetailsProviderClassName() { - return DBErrorDetailsProvider.class.getName(); + return null; } /** @@ -254,7 +254,9 @@ public void prepareRun(BatchSinkContext context) { context.getArguments().get(ETLDBOutputFormat.COMMIT_BATCH_SIZE)); } // set error details provider - context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + if (!Strings.isNullOrEmpty(getErrorDetailsProviderClassName())) { + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + } addOutputContext(context); } protected void addOutputContext(BatchSinkContext context) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 70fc12e4b..e3340fb87 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -239,7 +239,7 @@ protected SchemaReader getSchemaReader() { * @return ErrorDetailsProvider class name */ protected String getErrorDetailsProviderClassName() { - return DBErrorDetailsProvider.class.getName(); + return null; } private DriverCleanup loadPluginClassAndGetDriver(Class driverClass) @@ -299,7 +299,9 @@ public void prepareRun(BatchSourceContext context) throws Exception { schema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList())); } // set error details provider - context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + if (!Strings.isNullOrEmpty(getErrorDetailsProviderClassName())) { + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + } context.setInput(Input.of(sourceConfig.getReferenceName(), new SourceInputFormatProvider( DataDrivenETLDBInputFormat.class, connectionConfigAccessor.getConfiguration()))); } From 4137fe9b9182bf1a487532cfce342df76747060d Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 27 Jan 2025 18:34:05 +0530 Subject: [PATCH 04/36] Bump cdap.plugin.version to 2.13.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa4724a47..a3dacc4bf 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ UTF-8 6.11.0-SNAPSHOT - 2.11.1 + 2.13.0-SNAPSHOT 13.0.1 3.3.6 2.2.4 From 35607b5e476da4840a8e3e68c20b31e30aeaedec Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 28 Jan 2025 16:46:02 +0530 Subject: [PATCH 05/36] Revert cdap.plugin.version to 2.13.0-SNAPSHOT --- .../cdap/plugin/db/CommonSchemaReaderTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java index 0f5a3ca4a..cbe1361d0 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ProgramFailureException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -162,49 +163,49 @@ public void testGetSchemaThrowsExceptionOnNumericWithZeroPrecision() throws SQLE reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnArray() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.ARRAY); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnDatalink() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.DATALINK); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnDistinct() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.DISTINCT); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnJavaObject() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.JAVA_OBJECT); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnOther() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.OTHER); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnRef() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.REF); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnSQLXML() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.SQLXML); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnStruct() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.STRUCT); reader.getSchema(metadata, 1); From 1d8b9680023f6cdc8c1d055532beb643eccabb52 Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 29 Jan 2025 10:39:22 +0530 Subject: [PATCH 06/36] Remove DBErrorDetailsProvider --- .../plugin/db/DBErrorDetailsProvider.java | 131 ------------------ .../cdap/plugin/db/sink/AbstractDBSink.java | 13 +- .../plugin/db/source/AbstractDBSource.java | 13 +- .../mysql/MysqlErrorDetailsProvider.java | 4 +- .../PostgresErrorDetailsProvider.java | 4 +- 5 files changed, 18 insertions(+), 147 deletions(-) delete mode 100644 database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java b/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java deleted file mode 100644 index cc731d6ac..000000000 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBErrorDetailsProvider.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright © 2024 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.db; - -import com.google.common.base.Strings; -import com.google.common.base.Throwables; -import io.cdap.cdap.api.exception.ErrorCategory; -import io.cdap.cdap.api.exception.ErrorCodeType; -import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.cdap.api.exception.ErrorUtils; -import io.cdap.cdap.api.exception.ProgramFailureException; -import io.cdap.cdap.etl.api.exception.ErrorContext; -import io.cdap.cdap.etl.api.exception.ErrorDetailsProvider; - -import java.sql.SQLException; -import java.util.List; - -/** - * A custom ErrorDetailsProvider for Database plugins. - */ -public class DBErrorDetailsProvider implements ErrorDetailsProvider { - - public ProgramFailureException getExceptionDetails(Exception e, ErrorContext errorContext) { - List causalChain = Throwables.getCausalChain(e); - for (Throwable t : causalChain) { - if (t instanceof ProgramFailureException) { - // if causal chain already has program failure exception, return null to avoid double wrap. - return null; - } - if (t instanceof SQLException) { - return getProgramFailureException((SQLException) t, errorContext); - } - if (t instanceof IllegalArgumentException) { - return getProgramFailureException((IllegalArgumentException) t, errorContext); - } - if (t instanceof IllegalStateException) { - return getProgramFailureException((IllegalStateException) t, errorContext); - } - } - return null; - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link SQLException}. - * - * @param e The SQLException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(SQLException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String sqlState = e.getSQLState(); - int errorCode = e.getErrorCode(); - String errorMessageWithDetails = String.format( - "Error occurred in the phase: '%s' with sqlState: '%s', errorCode: '%s', errorMessage: %s", - errorContext.getPhase(), sqlState, errorCode, errorMessage); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessageWithDetails.endsWith(".")) { - errorMessageWithDetails = errorMessageWithDetails + "."; - } - errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, - externalDocumentationLink); - } - return ErrorUtils.getProgramFailureException(Strings.isNullOrEmpty(sqlState) ? - new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN) : getErrorCategoryFromSqlState(sqlState), - errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCode(errorCode, sqlState), true, - ErrorCodeType.SQLSTATE, sqlState, externalDocumentationLink, e); - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalArgumentException}. - * - * @param e The IllegalArgumentException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalArgumentException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.USER, false, e); - } - - /** - * Get a ProgramFailureException with the given error - * information from {@link IllegalStateException}. - * - * @param e The IllegalStateException to get the error information from. - * @return A ProgramFailureException with the given error information. - */ - private ProgramFailureException getProgramFailureException(IllegalStateException e, ErrorContext errorContext) { - String errorMessage = e.getMessage(); - String errorMessageFormat = "Error occurred in the phase: '%s'. Error message: %s"; - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, - String.format(errorMessageFormat, errorContext.getPhase(), errorMessage), ErrorType.SYSTEM, false, e); - } - - /** - * Get the external documentation link for the client errors if available. - * - * @return The external documentation link as a {@link String}. - */ - protected String getExternalDocumentationLink() { - return null; - } - - protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { - return ErrorType.UNKNOWN; - } - - protected ErrorCategory getErrorCategoryFromSqlState(String sqlState) { - return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); - } -} diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 3252f70bd..797abfc23 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -47,7 +47,6 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.DBConfig; -import io.cdap.plugin.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.Operation; import io.cdap.plugin.db.SchemaReader; @@ -313,18 +312,20 @@ private Schema inferSchema(Class driverClass) { } } catch (SQLException e) { // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); String errorMessageWithDetails = String.format("Error while reading table metadata." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); String externalDocumentationLink = getExternalDocumentationLink(); if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessageWithDetails.endsWith(".")) { - errorMessageWithDetails = errorMessageWithDetails + "."; + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; } - errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, - externalDocumentationLink); + errorMessage = String.format("%s For more details, see %s", errorMessageWithDetails, errorMessage); } throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index e3340fb87..3fd490ada 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -46,7 +46,6 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.DBConfig; -import io.cdap.plugin.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.TransactionIsolationLevel; @@ -201,18 +200,20 @@ private Schema loadSchemaFromDB(Class driverClass) } catch (SQLException e) { // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); String externalDocumentationLink = getExternalDocumentationLink(); if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessageWithDetails.endsWith(".")) { - errorMessageWithDetails = errorMessageWithDetails + "."; + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; } - errorMessageWithDetails = String.format("%s For more details, see %s", errorMessageWithDetails, - externalDocumentationLink); + errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); } throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - e.getMessage(), errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } finally { diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java index 251f0fc74..ca9a2b928 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java @@ -17,7 +17,7 @@ package io.cdap.plugin.mysql; import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.plugin.db.DBErrorDetailsProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; /** @@ -31,7 +31,7 @@ protected String getExternalDocumentationLink() { } @Override - protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { + protected ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { // https://dev.mysql.com/doc/refman/9.0/en/error-message-elements.html#error-code-ranges if (errorCode >= 1000 && errorCode <= 5999) { return ErrorType.USER; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java index 3202a3e28..a7de4e5dc 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java @@ -19,7 +19,7 @@ import com.google.common.base.Strings; import io.cdap.cdap.api.exception.ErrorCategory; import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.plugin.db.DBErrorDetailsProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; import java.util.HashMap; @@ -77,7 +77,7 @@ protected String getExternalDocumentationLink() { } @Override - protected ErrorType getErrorTypeFromErrorCode(int errorCode, String sqlState) { + protected ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && ERROR_CODE_TO_ERROR_TYPE.containsKey(sqlState.substring(0, 2))) { return ERROR_CODE_TO_ERROR_TYPE.get(sqlState.substring(0, 2)); From 35b1f6bc80effb002fcfe68216012a80df25cdc9 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 24 Jan 2025 11:27:20 +0530 Subject: [PATCH 07/36] Add OracleErrorDetailsProvider --- .../java/io/cdap/plugin/util/DBUtils.java | 1 + .../features/source/OracleRunTime.feature | 2 ++ .../oracle/OracleErrorDetailsProvider.java | 31 +++++++++++++++++++ .../io/cdap/plugin/oracle/OracleSink.java | 9 ++++++ .../io/cdap/plugin/oracle/OracleSource.java | 10 ++++++ 5 files changed, 53 insertions(+) create mode 100644 oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index ffcfdc375..6fad9a4ab 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -64,6 +64,7 @@ public final class DBUtils { public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; public static final String POSTGRES_SUPPORTED_DOC_URL = "https://www.postgresql.org/docs/current/errcodes-appendix.html"; + public static final String ORACLE_SUPPORTED_DOC_URL = "https://docs.oracle.com/en/error-help/db/ora-index.html"; public static final String CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/postgres/error-messages"; diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index 2d1ca9ad1..fca0b3981 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -338,7 +338,9 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink And Save and Deploy Pipeline And Run the Pipeline in Runtime And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java new file mode 100644 index 000000000..e0c770c9f --- /dev/null +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for Oracle plugin. + */ +public class OracleErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 45afbcb51..511281e9d 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -82,6 +82,15 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return OracleErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } /** * Oracle action configuration. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 6df62e63e..53f75613b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -71,6 +71,16 @@ protected Class getDBRecordType() { return OracleSourceDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return OracleErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("oracle", From 34e08e0857989bfa65ae2f4775b47cecfdcc4413 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 24 Jan 2025 09:46:33 +0530 Subject: [PATCH 08/36] Add SqlServerErrorDetailsProvider --- .../java/io/cdap/plugin/util/DBUtils.java | 2 ++ .../mssql/SqlServerErrorDetailsProvider.java | 31 +++++++++++++++++++ .../io/cdap/plugin/mssql/SqlServerSink.java | 10 ++++++ .../io/cdap/plugin/mssql/SqlServerSource.java | 10 ++++++ 4 files changed, 53 insertions(+) create mode 100644 mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 6fad9a4ab..34043d513 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -61,6 +61,8 @@ public final class DBUtils { public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + public static final String MSSQL_SUPPORTED_DOC_URL = + "https://docs.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors"; public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; public static final String POSTGRES_SUPPORTED_DOC_URL = "https://www.postgresql.org/docs/current/errcodes-appendix.html"; diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java new file mode 100644 index 000000000..90d1ce7b7 --- /dev/null +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mssql; + +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for SQL Server plugins. + */ +public class SqlServerErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } +} diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 0fa8991c5..223785f6c 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -88,6 +88,16 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return SqlServerErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } + /** * MSSQL action configuration. */ diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 9603b24db..01dd6c9ca 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -75,6 +75,11 @@ protected Class getDBRecordType() { return SqlServerSourceDBRecord.class; } + @Override + protected String getErrorDetailsProviderClassName() { + return SqlServerErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mssql", @@ -85,6 +90,11 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } + /** * MSSQL source config. */ From 9e86b11d6ade7a6098e8353c83680a6dc4dd5264 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 27 Jan 2025 15:19:40 +0530 Subject: [PATCH 09/36] Show correct DB name in error string --- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../plugin/cloudsql/mysql/CloudSQLMySQLAction.java | 3 ++- .../plugin/cloudsql/mysql/CloudSQLMySQLSink.java | 3 ++- .../plugin/cloudsql/mysql/CloudSQLMySQLSource.java | 3 ++- .../postgres/CloudSQLPostgreSQLAction.java | 3 ++- .../cloudsql/postgres/CloudSQLPostgreSQLSink.java | 3 ++- .../postgres/CloudSQLPostgreSQLSource.java | 3 ++- .../java/io/cdap/plugin/util/CloudSQLUtil.java | 14 +++++++++----- 8 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 5ff3357f2..1c0b76971 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageNumberOfSplits=Split-By Field Name must be specified if Number of Sp errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. Specify the Bounding Query. errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'URLDecoder: Illegal hex characters in escape (%) pattern - For input string: "$^"' errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table 'Invalidtable' is set correctly and -errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL PostgreSQL instance. +errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java index 0608edb75..770dd9030 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java @@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlActionConfig.instanceType, - cloudsqlMysqlActionConfig.connectionName); + cloudsqlMysqlActionConfig.connectionName, + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index da1414ff3..6cd1b0031 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -74,7 +74,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSinkConfig.connection.getInstanceType(), - cloudsqlMysqlSinkConfig.connection.getConnectionName()); + cloudsqlMysqlSinkConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index 8273169c0..201360c67 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSourceConfig.connection.getInstanceType(), - cloudsqlMysqlSourceConfig.connection.getConnectionName()); + cloudsqlMysqlSourceConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java index 1a3f8ad7b..5b13759f6 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java @@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlActionConfig.instanceType, - cloudsqlPostgresqlActionConfig.connectionName); + cloudsqlPostgresqlActionConfig.connectionName, + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index 65b86366c..060b67f82 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -81,7 +81,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSinkConfig.connection.getInstanceType(), - cloudsqlPostgresqlSinkConfig.connection.getConnectionName()); + cloudsqlPostgresqlSinkConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index e32651a9a..db3f2d708 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSourceConfig.connection.getInstanceType(), - cloudsqlPostgresqlSourceConfig.connection.getConnectionName()); + cloudsqlPostgresqlSourceConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java index 11595ac06..f704f2ad5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java @@ -31,6 +31,9 @@ public class CloudSQLUtil { public static final String INSTANCE_TYPE = "instanceType"; public static final String PUBLIC_INSTANCE = "public"; public static final String PRIVATE_INSTANCE = "private"; + public static final String CLOUDSQL_POSTGRESQL = "CloudSQL PostgreSQL"; + public static final String CLOUDSQL_MYSQL = "CloudSQL MySQL"; + /** * Utility method to check the Connection Name format of a CloudSQL instance. @@ -38,9 +41,10 @@ public class CloudSQLUtil { * @param failureCollector {@link FailureCollector} for the pipeline * @param instanceType CloudSQL instance type * @param connectionName Connection Name for the CloudSQL instance + * @param databaseType Type of CloudSQL instance- CloudSQL PostgreSQL, CLoudSQL MySQL */ public static void checkConnectionName( - FailureCollector failureCollector, String instanceType, String connectionName) { + FailureCollector failureCollector, String instanceType, String connectionName, String databaseType) { if (PUBLIC_INSTANCE.equalsIgnoreCase(instanceType)) { Pattern connectionNamePattern = @@ -50,16 +54,16 @@ public static void checkConnectionName( if (!matcher.matches()) { failureCollector .addFailure( - "Connection Name must be in the format :: to connect to " - + "a public CloudSQL PostgreSQL instance.", null) + String.format("Connection Name must be in the format :: to connect to " + + "a public %s instance.", databaseType), null) .withConfigProperty(CONNECTION_NAME); } } else { if (!InetAddresses.isInetAddress(connectionName)) { failureCollector .addFailure( - "Enter the internal IP address of the Compute Engine VM cloudsql proxy " - + "is running on, to connect to a private CloudSQL PostgreSQL instance.", null) + String.format("Enter the internal IP address of the Compute Engine VM cloudsql proxy " + + "is running on, to connect to a private %s instance.", databaseType), null) .withConfigProperty(CONNECTION_NAME); } } From fd55012c8f1526f2b802eae60e2d90889136b7c5 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Fri, 31 Jan 2025 12:27:52 +0530 Subject: [PATCH 10/36] e2e tests pipeline preview error fix --- .../src/e2e-test/resources/errorMessage.properties | 10 ++++++---- .../src/e2e-test/resources/errorMessage.properties | 12 ++++++------ .../e2e-test/features/mysqlsource/RunTime.feature | 2 +- .../e2e-test/features/source/OracleRunTime.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 6 ++++-- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 1c0b76971..d4fc1de28 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,7 +15,9 @@ errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table ' errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found -errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ - Table -errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Errors were encountered during validation. -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. +errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ + Table 'Table123' does not exist.. Please check the system logs for more details. +errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ + Exception while trying to validate schema of database table +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : java.io.IOException: You have an error in your SQL syntax; \ + check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 00721f148..c752d6ec1 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -13,11 +13,11 @@ errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. errormessageBlankHost=Exception while trying to validate schema of database table -errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ - Table 'Table123@' does not exist.. Please check the system logs for more details. +errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'SQL Server2' encountered : io.cdap.cdap.etl.api.validation.ValidationException: \ + Errors were encountered during validation. Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ 'connection' is invalid: Failed to assign value -errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Incorrect syntax near the keyword 'table'.. \ - Please check the system logs for more details. -errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Plugin with id SQL \ - Server:source.jdbc.sqlserver does not exist in program phase-1 of application +errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : io.cdap.cdap.api.exception.ProgramFailureException: \ + Error occurred while trying to get schema from database.Error message: 'Incorrect syntax near the keyword 'table'.'. +errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ + Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature index 1ad2f8cc1..0ea426da0 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature @@ -142,7 +142,7 @@ Feature: MySQL Source - Run time scenarios Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs Then Open and capture pipeline preview logs Then Verify the preview run status of pipeline in the logs is "failed" diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index fca0b3981..d6ad85cd4 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -296,7 +296,7 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs Then Verify the preview run status of pipeline in the logs is "failed" @ORACLE_SOURCE_TEST @BQ_SINK_TEST diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index b981faf81..e44f4c00a 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,5 +15,7 @@ errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Exception while trying to validate schema of database table '"table"' for connection errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"TARGETTABLE_ errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: ORA-00936: missing expression . \ - Please check the system logs for more details. +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ + java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. +blank.database.message=Required property 'database' has no value. +blank.connection.message=Exception while trying to validate schema of database table diff --git a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 6e1929245..f793e3be7 100644 --- a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -18,5 +18,5 @@ errorMessageInvalidSourceHost=SQL error while getting query schema: The connecti errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"targettable_ errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ - number of columns: 0.. Please check the system logs for more details. +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'PostgreSQL' encountered : \ + java.io.IOException: The column index is out of range: 1, number of columns: 0.. Please check the system logs for more details. From 858a7cc795fa1cdf375710b715f66a6b1ad7e19a Mon Sep 17 00:00:00 2001 From: Aakash Nayak Date: Mon, 10 Mar 2025 00:39:14 +0530 Subject: [PATCH 11/36] Removing snapshot for 6.11 release --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 4 ++-- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 21 insertions(+), 21 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 17aa5e48b..f48bc6ccd 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index df38e7267..ddb3b4e9b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index cb803d1ba..3154001e8 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 44dd1fe8c..95f278118 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.12.0-SNAPSHOT + 1.12.0 diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 8faab79ba..3357deb03 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 683dd2f43..33d30657e 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 39c3fcd52..0c059f9bf 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index b823356d8..2f6c83d2b 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index d8a78cd4d..a8c1f88d0 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 7ece99f31..c7cbbdbf2 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index c9dbaf035..f98bc9bbd 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index f5fc81a93..8a8316f13 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index bb5caf4c0..53b3e75f0 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index f7b559439..835948cf3 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index b21152fcd..3137b38df 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 Oracle plugin diff --git a/pom.xml b/pom.xml index a3dacc4bf..2e40cac37 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.0-SNAPSHOT + 1.12.0 pom Database Plugins Collection of database plugins @@ -724,7 +724,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.4.0-SNAPSHOT + 0.4.0 test diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 8f086e482..e1b2076db 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 26ce2e396..7e8c8cff8 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index c351a9d96..6d311cc52 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0-SNAPSHOT + 1.12.0 teradata-plugin From a3a0613415bf39dc0c88e84f7827213fe4dbb540 Mon Sep 17 00:00:00 2001 From: Aakash Nayak Date: Thu, 13 Mar 2025 14:54:13 +0530 Subject: [PATCH 12/36] Updating cdap version to 6.11 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2e40cac37..a2985a578 100644 --- a/pom.xml +++ b/pom.xml @@ -61,8 +61,8 @@ true UTF-8 - 6.11.0-SNAPSHOT - 2.13.0-SNAPSHOT + 6.11.0 + 2.13.0 13.0.1 3.3.6 2.2.4 From 27216859b29977ac7f9259ec549bf6ca372690ce Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 2 Apr 2025 10:23:11 +0530 Subject: [PATCH 13/36] Bump 1.12.1-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 4 ++-- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 20 insertions(+), 20 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index f48bc6ccd..f4251bb32 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index ddb3b4e9b..9ca5bef36 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 3154001e8..70e946e3d 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 95f278118..b692902e5 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT CloudSQL MySQL plugin @@ -45,7 +45,7 @@ io.cdap.plugin mysql-plugin - 1.12.0 + ${project.version} diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 3357deb03..1bc542518 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 33d30657e..58812af4d 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0c059f9bf..0c53ce6a7 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 2f6c83d2b..e30dca17d 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index a8c1f88d0..e229e104b 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index c7cbbdbf2..b69aa03f4 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f98bc9bbd..06491d4d4 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 8a8316f13..75d967686 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 53b3e75f0..7619593e9 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 835948cf3..232e70336 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 3137b38df..4e9800c7a 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index a2985a578..53f611119 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.0 + 1.12.1-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index e1b2076db..edf87555d 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 7e8c8cff8..7b4382341 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 6d311cc52..ab09612a5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.0 + 1.12.1-SNAPSHOT teradata-plugin From 30bf808842f9041a4eb175dcb494dc9b6fe0365b Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 2 Apr 2025 09:38:06 +0530 Subject: [PATCH 14/36] Bumped to 2.13.1-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53f611119..c58fec1f3 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ UTF-8 6.11.0 - 2.13.0 + 2.13.1-SNAPSHOT 13.0.1 3.3.6 2.2.4 From 525cf2e8d9e57389452a750f6b69b92a2a4b10c7 Mon Sep 17 00:00:00 2001 From: psainics Date: Tue, 18 Mar 2025 03:49:11 +0530 Subject: [PATCH 15/36] Error Management AbstractDBAction --- .../cdap/plugin/db/action/AbstractDBAction.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index a2be9cbf0..0eaac3148 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,12 +16,15 @@ package io.cdap.plugin.db.action; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; import java.sql.Driver; +import java.sql.SQLException; /** * Action that runs a db command. @@ -40,7 +43,18 @@ public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + try { + executeQuery.run(); + } catch (Exception e) { + if (e instanceof SQLException) { + DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); + throw dbe.getProgramFailureException((SQLException) e, null); + } + FailureCollector collector = context.getFailureCollector(); + collector.addFailure("Failed to execute query with message: " + e.getMessage(), null) + .withStacktrace(e.getStackTrace()); + collector.getOrThrowException(); + } } @Override From adf1d235a1d645556fbeaae8d020797343e0a0a5 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 14 Apr 2025 16:16:58 +0530 Subject: [PATCH 16/36] Add MariadbDBRecord implementations UPDATE DOCS --- mariadb-plugin/docs/Mariadb-batchsink.md | 68 +++++++++--------- mariadb-plugin/docs/Mariadb-batchsource.md | 70 +++++++++---------- mariadb-plugin/pom.xml | 5 ++ .../cdap/plugin/mariadb/MariadbDBRecord.java | 40 +++++++++++ .../plugin/mariadb/MariadbSchemaReader.java | 36 ++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 15 ++++ .../io/cdap/plugin/mariadb/MariadbSource.java | 46 ++++++++++++ 7 files changed, 208 insertions(+), 72 deletions(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java diff --git a/mariadb-plugin/docs/Mariadb-batchsink.md b/mariadb-plugin/docs/Mariadb-batchsink.md index 11176c0db..e4541fe67 100644 --- a/mariadb-plugin/docs/Mariadb-batchsink.md +++ b/mariadb-plugin/docs/Mariadb-batchsink.md @@ -60,41 +60,39 @@ connections. Data Types Mapping ---------- - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------- diff --git a/mariadb-plugin/docs/Mariadb-batchsource.md b/mariadb-plugin/docs/Mariadb-batchsource.md index 2b1fe3944..713af2ee8 100644 --- a/mariadb-plugin/docs/Mariadb-batchsource.md +++ b/mariadb-plugin/docs/Mariadb-batchsource.md @@ -78,43 +78,39 @@ with the tradeoff of higher memory usage. Data Types Mapping ---------- - - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ - + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------ diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index b69aa03f4..a387ee041 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -83,6 +83,11 @@ RELEASE compile + + io.cdap.plugin + mysql-plugin + ${project.version} + diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java new file mode 100644 index 000000000..94498c787 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.mysql.MysqlDBRecord; +import java.util.List; + +/** + * Writable class for MariaDB Source/Sink. + */ +public class MariadbDBRecord extends MysqlDBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public MariadbDBRecord() { + // Required by Hadoop DBRecordReader to create an instance + } + + public MariadbDBRecord(StructuredRecord record, List columnTypes) { + super(record, columnTypes); + } +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java new file mode 100644 index 000000000..37ac12a93 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlSchemaReader; +import java.util.Map; + +/** + * Schema reader for mapping Maria DB type + */ +public class MariadbSchemaReader extends MysqlSchemaReader { + + public MariadbSchemaReader (String sessionID) { + super(sessionID); + } + + public MariadbSchemaReader (String sessionID, Map connectionArguments) { + super(sessionID, connectionArguments); + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index ab20f3c5d..9dbf0c7d4 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -19,10 +19,14 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.mysql.MysqlDBRecord; import java.util.Map; import javax.annotation.Nullable; @@ -45,6 +49,17 @@ public MariadbSink(MariadbSinkConfig mariadbSinkConfig) { this.mariadbSinkConfig = mariadbSinkConfig; } + @Override + protected DBRecord getDBRecord(StructuredRecord output) { + return new MariadbDBRecord(output, columnTypes); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null); + } + + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index d5ffcb290..4a4d689bb 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -20,9 +20,16 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -53,10 +60,36 @@ protected String createConnectionString() { mariadbSourceConfig.host, mariadbSourceConfig.port, mariadbSourceConfig.database); } + @Override + protected Class getDBRecordType() { + return MariadbDBRecord.class; + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("mariadb", + mariadbSourceConfig.host, + mariadbSourceConfig.port, + mariadbSourceConfig.database, + mariadbSourceConfig.getReferenceName()); + Asset asset = Asset.builder(mariadbSourceConfig.getReferenceName()).setFqn(fqn).build(); + return new LineageRecorder(context, asset); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null, mariadbSourceConfig.getConnectionArguments()); + } + /** * MaraiDB source mariadbSourceConfig. */ public static class MariadbSourceConfig extends DBSpecificSourceConfig { + private static final String JDBC_PROPERTY_CONNECT_TIMEOUT = "connectTimeout"; + private static final String JDBC_PROPERTY_SOCKET_TIMEOUT = "socketTimeout"; + private static final String JDBC_REWRITE_BATCHED_STATEMENTS = "rewriteBatchedStatements"; + + private static final String MARIADB_TINYINT1_IS_BIT = "tinyInt1isBit"; @Name(MariadbConstants.AUTO_RECONNECT) @Description("Should the driver try to re-establish stale and/or dead connections") @@ -116,5 +149,18 @@ public Map getDBSpecificArguments() { public List getInitQueries() { return MariadbUtil.composeDbInitQueries(useAnsiQuotes); } + + @Override + public Map getConnectionArguments() { + Map arguments = new HashMap<>(super.getConnectionArguments()); + // the unit below is millisecond + arguments.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + // MariaDB property to ensure that TINYINT(1) type data is not converted to MariaDB Bit/Boolean type in the + // ResultSet. + arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); + return arguments; + } } } From 0805c0587425f86b9d99f56d19ad49275250a2ae Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 11 Apr 2025 09:53:41 +0530 Subject: [PATCH 17/36] Add MariadbErrorDetailsProvider --- .../java/io/cdap/plugin/util/DBUtils.java | 1 + .../mariadb/MariadbErrorDetailsProvider.java | 33 +++++++++++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 12 ++++++- .../io/cdap/plugin/mariadb/MariadbSource.java | 10 ++++++ 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 34043d513..b125a7214 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -61,6 +61,7 @@ public final class DBUtils { public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + public static final String MARIADB_SUPPORTED_DOC_URL = "https://mariadb.com/kb/en/mariadb-error-codes/"; public static final String MSSQL_SUPPORTED_DOC_URL = "https://docs.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors"; public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java new file mode 100644 index 000000000..38405225d --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for MariaDb plugins. + */ +public class MariadbErrorDetailsProvider extends MysqlErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 9dbf0c7d4..589da2953 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -25,8 +25,8 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.util.DBUtils; -import io.cdap.plugin.mysql.MysqlDBRecord; import java.util.Map; import javax.annotation.Nullable; @@ -60,6 +60,16 @@ protected SchemaReader getSchemaReader() { } + @Override + protected String getErrorDetailsProviderClassName() { + return MariadbErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 4a4d689bb..098145ab9 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -81,6 +81,16 @@ protected SchemaReader getSchemaReader() { return new MariadbSchemaReader(null, mariadbSourceConfig.getConnectionArguments()); } + @Override + protected String getErrorDetailsProviderClassName() { + return MariadbErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + /** * MaraiDB source mariadbSourceConfig. */ From 8777c32c5fe56c52db26b96245f81483799d9b25 Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 16 Apr 2025 14:12:00 +0530 Subject: [PATCH 18/36] Remove snapshot to release 1.12.1 --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index f4251bb32..2eab54191 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 9ca5bef36..8648ca695 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 70e946e3d..fee4ef048 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index b692902e5..b2f41ed7f 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 1bc542518..c5d9997c1 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 58812af4d..7cc5f1622 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0c53ce6a7..8018b783a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index e30dca17d..ba8dd82f5 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index e229e104b..c5e73943f 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index a387ee041..b080138db 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 06491d4d4..3b193c0bd 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 75d967686..72b20d78e 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 7619593e9..08d4fc015 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 232e70336..e884e6d23 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4e9800c7a..378521afd 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 Oracle plugin diff --git a/pom.xml b/pom.xml index c58fec1f3..f52f6bb18 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.1-SNAPSHOT + 1.12.1 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index edf87555d..5d45cfa40 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 7b4382341..57083ad10 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index ab09612a5..f40f39a42 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1-SNAPSHOT + 1.12.1 teradata-plugin From cf99b4db7b299ab2e04ea70785b61563bda50711 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 5 May 2025 15:45:14 +0530 Subject: [PATCH 19/36] Bump version to 1.12.2-SNAPSHOT in pom.xml for all database plugins --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 2eab54191..d1a9c0a96 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 8648ca695..65c6b693f 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index fee4ef048..39e0cb9dc 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index b2f41ed7f..f5798c468 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index c5d9997c1..8ee68aada 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 7cc5f1622..778524126 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 8018b783a..0ef1249cd 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index ba8dd82f5..124260892 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index c5e73943f..5d2d8cc5a 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index b080138db..e82c7db1c 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 3b193c0bd..cd1c9fe57 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 72b20d78e..c00a12e44 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 08d4fc015..bacdd9567 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index e884e6d23..fedd84455 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 378521afd..891b78537 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index f52f6bb18..e8edf4027 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.1 + 1.12.2-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 5d45cfa40..5092392e4 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 57083ad10..2313be26e 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index f40f39a42..0c32586c6 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.1 + 1.12.2-SNAPSHOT teradata-plugin From 1c0c4b689939700fb96c9868cb037e8009f9ab91 Mon Sep 17 00:00:00 2001 From: psainics Date: Fri, 2 May 2025 08:33:23 +0530 Subject: [PATCH 20/36] Skip field validation for compatiblity --- .../plugin/db/source/AbstractDBSource.java | 24 ++++++++++------- .../db/source/AbstractDBSourceTest.java | 12 ++++++--- .../mariadb/MariadbFieldsValidator.java | 25 +++++++++++++++++ .../io/cdap/plugin/mariadb/MariadbSink.java | 7 +++++ .../io/cdap/plugin/mariadb/MariadbSource.java | 27 +++++++++++++++++++ 5 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 3fd490ada..54d1e2ab6 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -529,7 +529,7 @@ public void validateSchema(Schema actualSchema, FailureCollector collector) { } @VisibleForTesting - static void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { + void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { if (configSchema == null) { collector.addFailure("Schema should not be null or empty.", null) .withConfigProperty(SCHEMA); @@ -550,14 +550,20 @@ static void validateSchema(Schema actualSchema, Schema configSchema, FailureColl Schema expectedFieldSchema = field.getSchema().isNullable() ? field.getSchema().getNonNullable() : field.getSchema(); - if (actualFieldSchema.getType() != expectedFieldSchema.getType() || - actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { - collector.addFailure( - String.format("Schema field '%s' has type '%s but found '%s'.", - field.getName(), expectedFieldSchema.getDisplayName(), - actualFieldSchema.getDisplayName()), null) - .withOutputSchemaField(field.getName()); - } + validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } + } + + protected void validateField(FailureCollector collector, Schema.Field field, Schema actualFieldSchema, + Schema expectedFieldSchema) { + if (actualFieldSchema.getType() != expectedFieldSchema.getType() || + actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { + collector.addFailure( + String.format("Schema field '%s' is expected to have type '%s but found '%s'.", field.getName(), + expectedFieldSchema.getDisplayName(), actualFieldSchema.getDisplayName()), + String.format("Change the data type of field %s to %s.", field.getName(), + actualFieldSchema.getDisplayName())) + .withOutputSchemaField(field.getName()); } } diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java index 3dc7a2d1c..a8be38b46 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java @@ -43,11 +43,17 @@ public class AbstractDBSourceTest { Schema.Field.of("double_column", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("boolean_column", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))) ); + private static final AbstractDBSource.DBSourceConfig TEST_CONFIG = new AbstractDBSource.DBSourceConfig() { + @Override + public String getConnectionString() { + return ""; + } + }; @Test public void testValidateSourceSchemaCorrectSchema() { MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(SCHEMA, SCHEMA, collector); + TEST_CONFIG.validateSchema(SCHEMA, SCHEMA, collector); Assert.assertEquals(0, collector.getValidationFailures().size()); } @@ -65,7 +71,7 @@ public void testValidateSourceSchemaMismatchFields() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } @@ -84,7 +90,7 @@ public void testValidateSourceSchemaInvalidFieldType() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java new file mode 100644 index 000000000..71ccb0d06 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.plugin.mysql.MysqlFieldsValidator; + +/** + * Field validator for maraidb + */ +public class MariadbFieldsValidator extends MysqlFieldsValidator { +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 589da2953..52a73344a 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -25,6 +25,8 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.db.sink.FieldsValidator; +import io.cdap.plugin.mysql.MysqlFieldsValidator; import io.cdap.plugin.util.DBUtils; import java.util.Map; @@ -70,6 +72,11 @@ protected String getExternalDocumentationLink() { return DBUtils.MARIADB_SUPPORTED_DOC_URL; } + @Override + protected FieldsValidator getFieldsValidator() { + return new MariadbFieldsValidator(); + } + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 098145ab9..28204100c 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -19,6 +19,8 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.plugin.common.Asset; @@ -172,5 +174,30 @@ public Map getConnectionArguments() { arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); return arguments; } + + @Override + protected void validateField(FailureCollector collector, + Schema.Field field, + Schema actualFieldSchema, + Schema expectedFieldSchema) { + // Backward compatibility changes to support MySQL YEAR to Date type conversion + if (Schema.LogicalType.DATE.equals(expectedFieldSchema.getLogicalType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL MEDIUMINT UNSIGNED to Long type conversion + if (Schema.Type.LONG.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL TINYINT(1) to Bool type conversion + if (Schema.Type.BOOLEAN.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + super.validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } } } From 9e3be1de5e950e79368a0f2ff528ca6e1dbb1f1a Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 21 Apr 2025 15:27:42 +0530 Subject: [PATCH 21/36] Set TRANSACTION_ISOLATION_LEVEL config in MySQL, PostgreSQL & MSSQL plugins Add transaction isolation level to PostgreSQL, MYSQL and MSSQL Plugins. --- .../io/cdap/plugin/db/ConnectionConfig.java | 1 + .../AbstractDBSpecificConnectorConfig.java | 26 ++++++++++++++++++- mssql-plugin/docs/SQL Server-connector.md | 8 ++++++ mssql-plugin/docs/SqlServer-batchsink.md | 8 ++++++ mssql-plugin/docs/SqlServer-batchsource.md | 8 ++++++ .../io/cdap/plugin/mssql/SqlServerSink.java | 5 ++++ .../io/cdap/plugin/mssql/SqlServerSource.java | 5 ++++ .../widgets/SQL Server-connector.json | 14 ++++++++++ mssql-plugin/widgets/SqlServer-batchsink.json | 18 +++++++++++++ .../widgets/SqlServer-batchsource.json | 18 +++++++++++++ mysql-plugin/docs/MySQL-connector.md | 8 ++++++ mysql-plugin/docs/Mysql-batchsink.md | 8 ++++++ mysql-plugin/docs/Mysql-batchsource.md | 8 ++++++ .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 ++++ .../io/cdap/plugin/mysql/MysqlSource.java | 5 ++++ mysql-plugin/widgets/MySQL-connector.json | 14 ++++++++++ mysql-plugin/widgets/Mysql-batchsink.json | 18 +++++++++++++ mysql-plugin/widgets/Mysql-batchsource.json | 18 +++++++++++++ .../plugin/oracle/OracleConnectorConfig.java | 18 ++----------- .../docs/PostgreSQL-connector.md | 8 ++++++ postgresql-plugin/docs/Postgres-batchsink.md | 8 ++++++ .../docs/Postgres-batchsource.md | 8 ++++++ .../io/cdap/plugin/postgres/PostgresSink.java | 5 ++++ .../cdap/plugin/postgres/PostgresSource.java | 5 ++++ .../widgets/PostgreSQL-connector.json | 13 ++++++++++ .../widgets/Postgres-batchsink.json | 17 ++++++++++++ .../widgets/Postgres-batchsource.json | 17 ++++++++++++ 27 files changed, 277 insertions(+), 17 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 588ed78b8..c5320e25e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 5c6b08031..8de0e4d70 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -20,8 +20,9 @@ import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.TransactionIsolationLevel; -import java.util.Collections; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; @@ -42,6 +43,12 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; + @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) + @Description("The transaction isolation level for the database session.") + @Macro + @Nullable + protected String transactionIsolationLevel; + public String getHost() { return host; } @@ -55,4 +62,21 @@ public int getPort() { public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } + + @Override + public Map getAdditionalArguments() { + Map additonalArguments = new HashMap<>(); + if (getTransactionIsolationLevel() != null) { + additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); + } + return additonalArguments; + } + + public String getTransactionIsolationLevel() { + if (transactionIsolationLevel == null) { + return null; + } + return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + } } + diff --git a/mssql-plugin/docs/SQL Server-connector.md b/mssql-plugin/docs/SQL Server-connector.md index cb72161f5..6f0038715 100644 --- a/mssql-plugin/docs/SQL Server-connector.md +++ b/mssql-plugin/docs/SQL Server-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Authentication Type:** Indicates which authentication method will be used for the connection. Use 'SQL Login'. to connect to a SQL Server using username and password properties. Use 'Active Directory Password' to connect to an Azure SQL Database/Data Warehouse using an Azure AD principal name and password. diff --git a/mssql-plugin/docs/SqlServer-batchsink.md b/mssql-plugin/docs/SqlServer-batchsink.md index 5d10b4bb6..b4ca1cbc5 100644 --- a/mssql-plugin/docs/SqlServer-batchsink.md +++ b/mssql-plugin/docs/SqlServer-batchsink.md @@ -46,6 +46,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/docs/SqlServer-batchsource.md b/mssql-plugin/docs/SqlServer-batchsource.md index c8e30f77e..5c917621c 100644 --- a/mssql-plugin/docs/SqlServer-batchsource.md +++ b/mssql-plugin/docs/SqlServer-batchsource.md @@ -56,6 +56,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 223785f6c..dc442d200 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -177,6 +177,11 @@ public Map getDBSpecificArguments() { packetSize, queryTimeout); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public String getConnectionString() { return String.format(SqlServerConstants.SQL_SERVER_CONNECTION_STRING_FORMAT, diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 01dd6c9ca..004532064 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -198,6 +198,11 @@ public List getInitQueries() { return Collections.emptyList(); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index 171076295..c326cd81d 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -64,6 +64,20 @@ "widget-type": "password", "label": "Password", "name": "password" + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 260c66259..fb20cad9d 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -280,6 +294,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index dad5f4708..b3494e485 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -316,6 +330,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index fb5c1fbb8..f586084c1 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index b28a28618..46a763f9d 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 010e08216..552bb5504 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index fee0a31fa..0a9257a0a 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -194,6 +194,11 @@ public Map getDBSpecificArguments() { trustCertificateKeyStorePassword, false); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public MysqlConnectorConfig getConnection() { return connection; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 971b76809..38642468c 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -197,6 +197,11 @@ public MysqlConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 9064d1bf6..f60f5526f 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -30,6 +30,20 @@ "widget-attributes": { "default": "3306" } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index c525ead40..58596aae2 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -225,6 +239,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 9175bd5ed..506e837f7 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -277,6 +291,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 10022364a..c3ce051e5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -81,12 +81,6 @@ public String getConnectionString() { @Macro private String database; - @Name(OracleConstants.TRANSACTION_ISOLATION_LEVEL) - @Description("The transaction isolation level for the database session.") - @Macro - @Nullable - private String transactionIsolationLevel; - @Name(OracleConstants.USE_SSL) @Description("Turns on SSL encryption. Connection will fail if SSL is not available") @Nullable @@ -124,6 +118,7 @@ public Properties getConnectionArgumentsProperties() { return prop; } + @Override public String getTransactionIsolationLevel() { //if null default to the highest isolation level possible if (transactionIsolationLevel == null) { @@ -133,16 +128,7 @@ public String getTransactionIsolationLevel() { //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input //if role is SYSDBA or SYSOP it will map to read_committed. else serialized return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : - TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); - } - - @Override - public Map getAdditionalArguments() { - Map additonalArguments = new HashMap<>(); - if (getTransactionIsolationLevel() != null) { - additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); - } - return additonalArguments; + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } @Override diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index 739c678e3..fe442cbf1 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Database:** The name of the database to connect to. **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index b8a996463..82065e0fd 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index af359022d..559723526 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 7682b8b0b..73430c1e2 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -175,6 +175,11 @@ public Map getDBSpecificArguments() { return ImmutableMap.of(PostgresConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override protected PostgresConnectorConfig getConnection() { return connection; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index 8e3c091f9..b230f3d1e 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -143,6 +143,11 @@ protected PostgresConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 091afc972..9a7a02e14 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -31,6 +31,19 @@ "default": "5432" } }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "textbox", "label": "Database", diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 6aa2dad8a..14e6f8154 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -186,6 +199,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 0e4ba28c1..60de4725f 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -206,6 +219,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" From 829e0e7a58ad5345d5c176f2a9edaad885ceef79 Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 23 Apr 2025 20:02:59 +0530 Subject: [PATCH 22/36] Added changes for New committer to have commit/rollback in commit/abort task methods. --- .../cdap/plugin/db/sink/AbstractDBSink.java | 1 + .../plugin/db/sink/ETLDBOutputFormat.java | 97 +++++++++++++++++-- 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 797abfc23..0bb4bf123 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -228,6 +228,7 @@ public void prepareRun(BatchSinkContext context) { configAccessor.setInitQueries(dbSinkConfig.getInitQueries()); configAccessor.getConfiguration().set(DBConfiguration.DRIVER_CLASS_PROPERTY, driverClass.getName()); configAccessor.getConfiguration().set(DBConfiguration.URL_PROPERTY, connectionString); + configAccessor.getConfiguration().set(ETLDBOutputFormat.STAGE_NAME, context.getStageName()); String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() : dbSinkConfig.getEscapedDbSchemaName() + "." + dbSinkConfig.getEscapedTableName(); configAccessor.getConfiguration().set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, fullyQualifiedTableName); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java index ad2b91ab1..ad196386c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java @@ -25,6 +25,8 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -43,6 +45,7 @@ import java.sql.Statement; import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; import static io.cdap.plugin.db.ConnectionConfigAccessor.OPERATION_NAME; import static io.cdap.plugin.db.ConnectionConfigAccessor.RELATION_TABLE_KEY; @@ -56,15 +59,92 @@ public class ETLDBOutputFormat extends DBOutputFormat { // Batch size before submitting a batch to the SQL engine. If set to 0, no batches will be submitted until commit. public static final String COMMIT_BATCH_SIZE = "io.cdap.plugin.db.output.commit.batch.size"; + public static final String STAGE_NAME = "io.cdap.plugin.db.output.stage_name"; public static final int DEFAULT_COMMIT_BATCH_SIZE = 1000; private static final Character ESCAPE_CHAR = '"'; + // Format for connection map's key will be "taskAttemptId_stageName" + private static final String CONNECTION_MAP_KEY_FORMAT = "%s_%s"; + + // CONNECTION_MAP will be used to store connections with "taskAttemptId_stageName" as key and + // connection object as value. Making it static to be accessed from multiple task attempts within same executor. + private static final Map CONNECTION_MAP = new ConcurrentHashMap<>(); private static final Logger LOG = LoggerFactory.getLogger(ETLDBOutputFormat.class); private Configuration conf; private Driver driver; private JDBCDriverShim driverShim; + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) + throws IOException, InterruptedException { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) throws IOException { + // do nothing + } + + @Override + public void setupTask(TaskAttemptContext taskContext) throws IOException { + // do nothing + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { + return true; + } + + @Override + public void commitTask(TaskAttemptContext taskContext) throws IOException { + conf = context.getConfiguration(); + String stageName = conf.get(STAGE_NAME); + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + Connection connection; + if ((connection = CONNECTION_MAP.remove(connectionId)) != null) { + try { + connection.commit(); + } catch (SQLException e) { + try { + connection.rollback(); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + throw new IOException(e); + } finally { + try { + connection.close(); + LOG.debug("Connection Closed after committing the task with taskAttemptId {}", connectionId); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + } + } + + @Override + public void abortTask(TaskAttemptContext taskContext) throws IOException { + conf = context.getConfiguration(); + String stageName = conf.get(STAGE_NAME); + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + Connection connection; + if ((connection = CONNECTION_MAP.remove(connectionId)) != null) { + try { + connection.rollback(); + } catch (SQLException e) { + throw new IOException(e); + } finally { + try { + connection.close(); + LOG.debug("Connection Closed after rollback the task with taskAttemptId {}", connectionId); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + } + } + }; + } + @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException { conf = context.getConfiguration(); @@ -81,6 +161,11 @@ public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOE try { Connection connection = getConnection(conf); + String stageName = conf.get(STAGE_NAME); + // If using multiple sinks, task attemptID can be same in that case, appending stage in the end for uniqueness. + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + CONNECTION_MAP.put(connectionId, connection); + LOG.debug("Connection Added to the map with connectionId : {}", connectionId); PreparedStatement statement = connection.prepareStatement(constructQueryOnOperation(tableName, fieldNames, operationName, listKeys)); return new DBRecordWriter(connection, statement) { @@ -98,23 +183,15 @@ public void close(TaskAttemptContext context) throws IOException { if (!emptyData) { getStatement().executeBatch(); } - getConnection().commit(); } catch (SQLException e) { - try { - getConnection().rollback(); - } catch (SQLException ex) { - LOG.warn(StringUtils.stringifyException(ex)); - } throw new IOException(e); } finally { try { getStatement().close(); - getConnection().close(); } catch (SQLException ex) { throw new IOException(ex); } } - try { DriverManager.deregisterDriver(driverShim); } catch (SQLException e) { @@ -298,4 +375,8 @@ public String constructUpdateQuery(String table, String[] fieldNames, String[] l return query.toString(); } } + + private String getConnectionMapKeyFormat(String taskAttemptId, String stageName) { + return String.format(CONNECTION_MAP_KEY_FORMAT, taskAttemptId, stageName); + } } From 213bb8d143c5c489afa88d5ecfdd7f81fc64aceb Mon Sep 17 00:00:00 2001 From: psainics Date: Wed, 7 May 2025 11:02:26 +0530 Subject: [PATCH 23/36] Update version to 1.12.2 in pom.xml for all database plugins --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index d1a9c0a96..2f3aa906d 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 65c6b693f..58bd6fb9e 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 39e0cb9dc..708d55615 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f5798c468..58a30b41a 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 8ee68aada..65f8233ec 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 778524126..3ad3e0704 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 0ef1249cd..579d49f96 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 124260892..6009d73e9 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 5d2d8cc5a..1a17ae6b8 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index e82c7db1c..1335d993e 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index cd1c9fe57..bc8bda62e 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index c00a12e44..1bbfe839c 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index bacdd9567..c0e0a9dc2 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index fedd84455..8c6059dde 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 891b78537..5011b2f84 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 Oracle plugin diff --git a/pom.xml b/pom.xml index e8edf4027..e7cf3f934 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.2-SNAPSHOT + 1.12.2 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 5092392e4..28a801545 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 2313be26e..6996cd8a0 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0c32586c6..125871dcd 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2-SNAPSHOT + 1.12.2 teradata-plugin From a96b7dafd9fd071c6ce32662c4b7d54d36c3cd0f Mon Sep 17 00:00:00 2001 From: sahusanket Date: Tue, 3 Jun 2025 15:08:21 +0530 Subject: [PATCH 24/36] [PLUGIN-1893] Adding fields in Oracle source and connector which acts like a flag for Backward compatibility issues for Timestamp and number (precisionless) --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 28 +++++++++-- .../cdap/plugin/oracle/OracleConstants.java | 2 + .../io/cdap/plugin/oracle/OracleSource.java | 13 ++++-- .../oracle/OracleSourceSchemaReader.java | 44 ++++++++++++------ oracle-plugin/widgets/Oracle-batchsource.json | 46 +++++++++++++++++++ oracle-plugin/widgets/Oracle-connector.json | 38 +++++++++++++++ 7 files changed, 153 insertions(+), 21 deletions(-) diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 3d2f7399a..16371d5c1 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -112,7 +112,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new OracleSourceSchemaReader(sessionID); + return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), + config.getTreatPrecisionlessNumAsDeci()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index c3ce051e5..cbc1e5ed2 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -22,8 +22,6 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import javax.annotation.Nullable; @@ -43,12 +41,14 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { - this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null); + this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, + null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, - String role, Boolean useSSL) { + String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, + @Nullable Boolean treatPrecisionlessNumAsDeci) { this.host = host; this.port = port; @@ -60,6 +60,8 @@ public OracleConnectorConfig(String host, int port, String user, String password this.database = database; this.role = role; this.useSSL = useSSL; + this.treatAsOldTimestamp = treatAsOldTimestamp; + this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; } @Override @@ -86,6 +88,16 @@ public String getConnectionString() { @Nullable public Boolean useSSL; + @Name(OracleConstants.TREAT_AS_OLD_TIMESTAMP) + @Description("A hidden field to handle timestamp as CDAP's timestamp micros or string as per old behavior.") + @Nullable + public Boolean treatAsOldTimestamp; + + @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI) + @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.") + @Nullable + public Boolean treatPrecisionlessNumAsDeci; + @Override protected int getDefaultPort() { return 1521; @@ -108,6 +120,14 @@ public Boolean getSSlMode() { return useSSL != null && useSSL; } + public Boolean getTreatAsOldTimestamp() { + return Boolean.TRUE.equals(treatAsOldTimestamp); + } + + public Boolean getTreatPrecisionlessNumAsDeci() { + return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index dc38f80ac..cbd411175 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -43,6 +43,8 @@ private OracleConstants() { public static final String TNS_CONNECTION_TYPE = "tns"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; public static final String USE_SSL = "useSSL"; + public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; + public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 53f75613b..1488a084b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -63,7 +63,12 @@ protected String createConnectionString() { @Override protected SchemaReader getSchemaReader() { - return new OracleSourceSchemaReader(); + // PLUGIN-1893 : Based on field/properties from Oracle source and Oracle connection we will pass the flag to control + // handle schema to make it backward compatible. + boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); + boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci); } @Override @@ -127,9 +132,11 @@ public OracleSourceConfig(String host, int port, String user, String password, S String connectionArguments, String connectionType, String database, String role, int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, - String boundingQuery, String splitBy, Boolean useSSL) { + String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, + Boolean treatPrecisionlessNumAsDeci) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, - connectionType, database, role, useSSL); + connectionType, database, role, useSSL, treatAsOldTimestamp, + treatPrecisionlessNumAsDeci); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index 7d35f9bc7..dd17d2e84 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -26,6 +26,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Set; +import javax.annotation.Nullable; /** * Oracle Source schema reader. @@ -65,14 +66,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { ); private final String sessionID; + private final Boolean isTimestampOldBehavior; + private final Boolean isPrecisionlessNumAsDecimal; public OracleSourceSchemaReader() { - this(null); + this(null, false, false); } - - public OracleSourceSchemaReader(String sessionID) { - super(); + public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, + boolean isPrecisionlessNumAsDecimal) { this.sessionID = sessionID; + this.isTimestampOldBehavior = isTimestampOldBehavior; + this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; } @Override @@ -81,10 +85,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti switch (sqlType) { case TIMESTAMP_TZ: - return Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); - case Types.TIMESTAMP: + return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return Schema.of(Schema.LogicalType.DATETIME); + return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + case Types.TIMESTAMP: + return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: return Schema.of(Schema.Type.FLOAT); case BINARY_DOUBLE: @@ -107,12 +113,24 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti // For a Number type without specified precision and scale, precision will be 0 and scale will be -127 if (precision == 0) { // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 - LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " - + "converting into STRING type to avoid any precision loss.", - metadata.getColumnName(index), - metadata.getColumnTypeName(index), - metadata.getColumnName(index))); - return Schema.of(Schema.Type.STRING); + if (isPrecisionlessNumAsDecimal) { + precision = 38; + scale = 0; + LOG.warn(String.format("%s type with undefined precision and scale is detected, " + + "there may be a precision loss while running the pipeline. " + + "Please define an output precision and scale for field '%s' to avoid " + + "precision loss.", + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.decimalOf(precision, scale); + } else { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.of(Schema.Type.STRING); + } } return Schema.decimalOf(precision, scale); } diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 5eca20cc4..404262fb2 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -120,6 +120,44 @@ ] } }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", @@ -326,6 +364,14 @@ { "type": "property", "name": "transactionIsolationLevel" + }, + { + "type": "property", + "name": "getTreatAsOldTimestampConn" + }, + { + "type": "property", + "name": "treatPrecisionlessNumAsDeci" } ] }, diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 628027caf..013f3b240 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -129,6 +129,44 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From 13520c415fe7e163bcfc3632dc28ae59ee5cff67 Mon Sep 17 00:00:00 2001 From: sahusanket Date: Tue, 3 Jun 2025 20:30:45 +0530 Subject: [PATCH 25/36] Bumping Snapshot. --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 2f3aa906d..acad60eb7 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 58bd6fb9e..9047cb04b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 708d55615..70c98fc78 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 58a30b41a..c81b8f71c 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 65f8233ec..f412f5e2d 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 3ad3e0704..2c339b245 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 579d49f96..bffa9056e 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 6009d73e9..d378fbb8d 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 1a17ae6b8..d80e2e96c 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 1335d993e..4cb8acb79 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index bc8bda62e..d7e4a4586 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 1bbfe839c..a3553635b 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index c0e0a9dc2..0c78be65c 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 8c6059dde..a55398341 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 5011b2f84..a6b359529 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index e7cf3f934..44612edfc 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.2 + 1.12.3-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 28a801545..4303d5468 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6996cd8a0..6e9053d36 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 125871dcd..cf962ad21 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.2 + 1.12.3-SNAPSHOT teradata-plugin From 525a91df0e59f0f0ce4afec65188e4815ff6339d Mon Sep 17 00:00:00 2001 From: sahusanket Date: Wed, 4 Jun 2025 16:55:17 +0530 Subject: [PATCH 26/36] removing Snapshot for hub release --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index acad60eb7..17a18caa9 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 9047cb04b..654bde42b 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 70c98fc78..fd508c8fd 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index c81b8f71c..e5d0f896b 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index f412f5e2d..ebe89f33f 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 2c339b245..f7cbe44d5 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index bffa9056e..920ed89c7 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index d378fbb8d..08f979397 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index d80e2e96c..70c4414dc 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 4cb8acb79..682cc153f 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index d7e4a4586..f3ae24c38 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index a3553635b..bf8239beb 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 0c78be65c..763a26b28 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index a55398341..f141c7371 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index a6b359529..59aeeb067 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 Oracle plugin diff --git a/pom.xml b/pom.xml index 44612edfc..9d231e198 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.3-SNAPSHOT + 1.12.3 pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 4303d5468..22ae8b535 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 6e9053d36..c40736e07 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index cf962ad21..0201805c5 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3-SNAPSHOT + 1.12.3 teradata-plugin From 59e94841ce7867cb4c7f4876dcabab4deab99a41 Mon Sep 17 00:00:00 2001 From: dj-smart Date: Fri, 11 Jul 2025 11:00:52 +0000 Subject: [PATCH 27/36] CDAP OSS migration for CDF 6.11 --- pom.xml | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index 9d231e198..579dd8240 100644 --- a/pom.xml +++ b/pom.xml @@ -78,23 +78,12 @@ - - sonatype - https://oss.sonatype.org/content/groups/public - sonatype-snapshots - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots - - - sonatype - https://oss.sonatype.org/content/groups/public/ - - - @@ -349,16 +338,6 @@ - - - sonatype.release - https://oss.sonatype.org/service/local/staging/deploy/maven2 - - - sonatype.snapshots - https://oss.sonatype.org/content/repositories/snapshots - - ${testSourceLocation} @@ -532,14 +511,14 @@ - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.2 + org.sonatype.central + central-publishing-maven-plugin + 0.8.0 true - https://oss.sonatype.org - sonatype.release - 655dc88dc770c3 + sonatype.release + false + true From c78ec825098344410e9b5a3db4ac2dc1cb0cbd1e Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Mon, 21 Jul 2025 09:36:49 +0000 Subject: [PATCH 28/36] mark dependencies as test scope --- cloudsql-mysql-plugin/pom.xml | 3 +++ cloudsql-postgresql-plugin/pom.xml | 3 +++ mssql-plugin/pom.xml | 3 +++ mysql-plugin/pom.xml | 3 +++ oracle-plugin/pom.xml | 3 +++ postgresql-plugin/pom.xml | 1 + 6 files changed, 16 insertions(+) diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index e5d0f896b..f2ccb2442 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -59,6 +59,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -68,6 +69,7 @@ junit junit + test io.cdap.cdap @@ -77,6 +79,7 @@ org.mockito mockito-core + test org.jetbrains diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index ebe89f33f..eeff25572 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -63,6 +63,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -72,6 +73,7 @@ junit junit + test io.cdap.cdap @@ -81,6 +83,7 @@ org.mockito mockito-core + test org.jetbrains diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index bf8239beb..b485279ed 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -57,6 +57,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -65,10 +66,12 @@ junit junit + test org.mockito mockito-core + test com.microsoft.sqlserver diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index 763a26b28..d87b80ccb 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -57,6 +57,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -65,6 +66,7 @@ junit junit + test io.cdap.cdap @@ -74,6 +76,7 @@ org.mockito mockito-core + test mysql diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 59aeeb067..4d7125448 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -57,6 +57,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap @@ -65,6 +66,7 @@ junit junit + test org.hsqldb @@ -80,6 +82,7 @@ org.mockito mockito-core + test io.cdap.cdap diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 22ae8b535..d641628dd 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -63,6 +63,7 @@ io.cdap.cdap hydrator-test + test io.cdap.cdap From 768575e7817168f2e7afeed7c057dcde2ac60110 Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Wed, 6 Aug 2025 02:20:50 +0000 Subject: [PATCH 29/36] Add required for maven central publishing --- cloudsql-mysql-plugin/pom.xml | 44 ++++++++++++++++++++++++---- cloudsql-postgresql-plugin/pom.xml | 44 ++++++++++++++++++++++++---- database-commons/pom.xml | 39 +++++++++++++++++++++++++ mssql-plugin/pom.xml | 46 +++++++++++++++++++++++++---- mysql-plugin/pom.xml | 46 +++++++++++++++++++++++++---- oracle-plugin/pom.xml | 47 ++++++++++++++++++++++++++---- pom.xml | 19 +++++++++++- postgresql-plugin/pom.xml | 46 +++++++++++++++++++++++++---- 8 files changed, 300 insertions(+), 31 deletions(-) diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f2ccb2442..012acc177 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -26,11 +26,45 @@ CloudSQL MySQL plugin cloudsql-mysql-plugin 4.0.0 + CloudSQL MySQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api + ${cdap.version} provided @@ -41,6 +75,7 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} io.cdap.plugin @@ -59,26 +94,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} test junit junit + ${junit.version} test - - io.cdap.cdap - cdap-api - provided - org.mockito mockito-core + ${mockito.version} test diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index eeff25572..d219d5f65 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -26,11 +26,45 @@ CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin 4.0.0 + CloudSQL PostgreSQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api + ${cdap.version} provided @@ -41,6 +75,7 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} io.cdap.plugin @@ -63,26 +98,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} test junit junit + ${junit.version} test - - io.cdap.cdap - cdap-api - provided - org.mockito mockito-core + ${mockito.version} test diff --git a/database-commons/pom.xml b/database-commons/pom.xml index f7cbe44d5..8ee1e295b 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -26,33 +26,70 @@ Database Commons database-commons 4.0.0 + Database Commons + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} + test com.mockrunner @@ -63,6 +100,8 @@ org.mockito mockito-core + ${mockito.version} + test diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index b485279ed..376a6bc3f 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -26,11 +26,45 @@ Microsoft SQL Server plugin mssql-plugin 4.0.0 + Microsoft SQL Server plugin database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,20 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} test org.mockito mockito-core + ${mockito.version} test @@ -79,11 +120,6 @@ 8.2.1.jre8 test - - io.cdap.cdap - cdap-api - provided - org.jetbrains annotations diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index d87b80ccb..d2ea17f41 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -26,11 +26,45 @@ Mysql plugin mysql-plugin 4.0.0 + Mysql database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,25 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} test - - io.cdap.cdap - cdap-api - provided - org.mockito mockito-core + ${mockito.version} test diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 4d7125448..30e4b5911 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -26,11 +26,45 @@ Oracle plugin oracle-plugin 4.0.0 + Oracle database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,20 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} test org.hsqldb hsqldb + ${hsql.version} test @@ -82,13 +123,9 @@ org.mockito mockito-core + ${mockito.version} test - - io.cdap.cdap - cdap-api - provided - org.glassfish jakarta.json diff --git a/pom.xml b/pom.xml index 579dd8240..1d4baa4b4 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,22 @@ + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + + 7 true @@ -591,6 +607,7 @@ src/e2e-test/java TestRunner.java + 31.1-jre @@ -716,7 +733,7 @@ ch.qos.logback logback-classic - 1.2.8 + 1.3.15 runtime diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index d641628dd..2eeb641bf 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -26,11 +26,45 @@ PostgreSQL plugin postgresql-plugin 4.0.0 + PostgreSQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -63,16 +99,14 @@ io.cdap.cdap hydrator-test + ${cdap.version} test io.cdap.cdap cdap-data-pipeline3_2.12 - - - io.cdap.cdap - cdap-api - provided + ${cdap.version} + test org.jetbrains @@ -83,11 +117,13 @@ org.mockito mockito-core + ${mockito.version} test junit junit + ${junit.version} test From c60c68878790a303f9f9d890e1066f31b9f2f367 Mon Sep 17 00:00:00 2001 From: Komal Yadav Date: Wed, 20 Aug 2025 12:39:38 +0000 Subject: [PATCH 30/36] Remove -SNAPSHOT from Version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d4baa4b4..3a50542e0 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ UTF-8 6.11.0 - 2.13.1-SNAPSHOT + 2.13.1 13.0.1 3.3.6 2.2.4 From 86f19de7eeb67ad35a41ea92112aca26691a1fed Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Wed, 10 Sep 2025 20:40:11 +0530 Subject: [PATCH 31/36] Verify Goal Removal --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3a50542e0..a85b645e6 100644 --- a/pom.xml +++ b/pom.xml @@ -665,7 +665,8 @@ integration-test - verify + + From f10bbf9f0a0367a4318ac17ed0e966ee8ecda18c Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Tue, 7 Oct 2025 11:25:02 +0000 Subject: [PATCH 32/36] bump up to 1.12.4-SNAPSHOT --- amazon-redshift-plugin/pom.xml | 2 +- aurora-mysql-plugin/pom.xml | 2 +- aurora-postgresql-plugin/pom.xml | 2 +- cloudsql-mysql-plugin/pom.xml | 2 +- cloudsql-postgresql-plugin/pom.xml | 2 +- database-commons/pom.xml | 2 +- db2-plugin/pom.xml | 2 +- generic-database-plugin/pom.xml | 2 +- generic-db-argument-setter/pom.xml | 2 +- mariadb-plugin/pom.xml | 2 +- memsql-plugin/pom.xml | 2 +- mssql-plugin/pom.xml | 2 +- mysql-plugin/pom.xml | 2 +- netezza-plugin/pom.xml | 2 +- oracle-plugin/pom.xml | 2 +- pom.xml | 2 +- postgresql-plugin/pom.xml | 2 +- saphana-plugin/pom.xml | 2 +- teradata-plugin/pom.xml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml index 17a18caa9..9a545ef6b 100644 --- a/amazon-redshift-plugin/pom.xml +++ b/amazon-redshift-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Amazon Redshift plugin diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 654bde42b..b9a542c3d 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index fd508c8fd..0f31154ca 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index 012acc177..8061b4ca0 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT CloudSQL MySQL plugin diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index d219d5f65..f147961e6 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT CloudSQL PostgreSQL plugin diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 8ee1e295b..ebd4b8bab 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Database Commons diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index 920ed89c7..868269a8a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 08f979397..39cb543d1 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 70c4414dc..912528d4a 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 682cc153f..3f7b9a58b 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Maria DB plugin diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index f3ae24c38..53e10ed78 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 376a6bc3f..768e5d4c6 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Microsoft SQL Server plugin diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index d2ea17f41..7c7dd054a 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Mysql plugin diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index f141c7371..824a7d6ec 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index 30e4b5911..988cd424b 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT Oracle plugin diff --git a/pom.xml b/pom.xml index a85b645e6..f59a221d6 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.12.3 + 1.12.4-SNAPSHOT pom Database Plugins Collection of database plugins diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 2eeb641bf..73912e656 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT PostgreSQL plugin diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index c40736e07..6e541ddf2 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 0201805c5..6fa152ad7 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.12.3 + 1.12.4-SNAPSHOT teradata-plugin From 7a3fca681cf883c9519f0d60236ae13f59ac20f8 Mon Sep 17 00:00:00 2001 From: AMit-Cloudsufi Date: Tue, 30 Sep 2025 06:23:06 +0000 Subject: [PATCH 33/36] Add hidden treatTimestampLTZAsTimestamp --- .../cdap/plugin/oracle/OracleConnector.java | 3 +- .../plugin/oracle/OracleConnectorConfig.java | 15 ++- .../cdap/plugin/oracle/OracleConstants.java | 1 + .../io/cdap/plugin/oracle/OracleSource.java | 8 +- .../oracle/OracleSourceSchemaReader.java | 16 +++- .../plugin/oracle/OracleSchemaReaderTest.java | 94 +++++++++++++++++++ oracle-plugin/widgets/Oracle-batchsource.json | 19 ++++ oracle-plugin/widgets/Oracle-connector.json | 19 ++++ 8 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 16371d5c1..3094e7152 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -113,7 +113,8 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), - config.getTreatPrecisionlessNumAsDeci()); + config.getTreatPrecisionlessNumAsDeci(), + config.getTreatTimestampLTZAsTimestamp()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index cbc1e5ed2..79d14215b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -42,13 +42,14 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, - null); + null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, - @Nullable Boolean treatPrecisionlessNumAsDeci) { + @Nullable Boolean treatPrecisionlessNumAsDeci, + @Nullable Boolean treatTimestampLTZAsTimestamp) { this.host = host; this.port = port; @@ -62,6 +63,7 @@ public OracleConnectorConfig(String host, int port, String user, String password this.useSSL = useSSL; this.treatAsOldTimestamp = treatAsOldTimestamp; this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; + this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp; } @Override @@ -98,6 +100,11 @@ public String getConnectionString() { @Nullable public Boolean treatPrecisionlessNumAsDeci; + @Name(OracleConstants.TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP) + @Description("A hidden field to handle mapping of Oracle Timestamp_LTZ data type to BQ Timestamp.") + @Nullable + public Boolean treatTimestampLTZAsTimestamp; + @Override protected int getDefaultPort() { return 1521; @@ -128,6 +135,10 @@ public Boolean getTreatPrecisionlessNumAsDeci() { return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); } + public Boolean getTreatTimestampLTZAsTimestamp() { + return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index cbd411175..31c7620ff 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -45,6 +45,7 @@ private OracleConstants() { public static final String USE_SSL = "useSSL"; public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; + public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp"; /** * Constructs the Oracle connection string based on the provided connection type, host, port, and database. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 1488a084b..978155fc5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -67,8 +67,10 @@ protected SchemaReader getSchemaReader() { // handle schema to make it backward compatible. boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp(); - return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci); + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci, + treatTimestampLTZAsTimestamp); } @Override @@ -133,10 +135,10 @@ public OracleSourceConfig(String host, int port, String user, String password, S int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, - Boolean treatPrecisionlessNumAsDeci) { + Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, role, useSSL, treatAsOldTimestamp, - treatPrecisionlessNumAsDeci); + treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index dd17d2e84..208b70410 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,15 +69,17 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { private final String sessionID; private final Boolean isTimestampOldBehavior; private final Boolean isPrecisionlessNumAsDecimal; + private final Boolean isTimestampLtzFieldTimestamp; public OracleSourceSchemaReader() { - this(null, false, false); + this(null, false, false, false); } public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, - boolean isPrecisionlessNumAsDecimal) { + boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) { this.sessionID = sessionID; this.isTimestampOldBehavior = isTimestampOldBehavior; this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; + this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp; } @Override @@ -87,8 +90,7 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti case TIMESTAMP_TZ: return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return isTimestampOldBehavior ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) - : Schema.of(Schema.LogicalType.DATETIME); + return getTimestampLtzSchema(); case Types.TIMESTAMP: return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: @@ -139,6 +141,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti } } + private @NotNull Schema getTimestampLtzSchema() { + return isTimestampOldBehavior || isTimestampLtzFieldTimestamp + ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { if (sessionID == null) { diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java new file mode 100644 index 000000000..1ff77c533 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; + +public class OracleSchemaReaderTest { + + @Test + public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + + } + + @Test + public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.DATETIME))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + } +} diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 404262fb2..ab35f3e8c 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -158,6 +158,25 @@ ] } }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 013f3b240..005c3ffbd 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -167,6 +167,25 @@ } ] } + }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, From 7a62d47bc3597eccb5950626f677559acabfd5d0 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 30 Mar 2026 10:25:27 +0530 Subject: [PATCH 34/36] oracle sink update upsert --- .../main/java/io/cdap/plugin/db/DBRecord.java | 5 +- .../sink/OracleDesignTimeValidation.feature | 137 +++++++++++ .../features/sink/OracleRunTime.feature | 226 ++++++++++++++++++ .../features/sink/OracleRunTimeMacro.feature | 132 ++++++++++ .../java/io.cdap.plugin/OracleClient.java | 32 ++- .../common.stepsdesign/TestSetupHooks.java | 32 +++ .../resources/errorMessage.properties | 4 +- .../resources/pluginParameters.properties | 8 + .../oracle/OracleETLDBOutputFormat.java | 126 ++++++++++ .../io/cdap/plugin/oracle/OracleSink.java | 13 +- .../plugin/oracle/OracleSinkDBRecord.java | 15 +- .../plugin/oracle/OracleSourceDBRecord.java | 2 +- .../oracle/OracleETLDBOutputFormatTest.java | 129 ++++++++++ oracle-plugin/widgets/Oracle-batchsink.json | 28 ++- 14 files changed, 877 insertions(+), 12 deletions(-) create mode 100644 oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java create mode 100644 oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index a5a9fcf5f..b187c7670 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -314,10 +314,7 @@ protected void upsertOperation(PreparedStatement stmt) throws SQLException { } private boolean fillUpdateParams(List updatedKeyList, ColumnType columnType) { - if (operationName.equals(Operation.UPDATE) && updatedKeyList.contains(columnType.getName())) { - return true; - } - return false; + return operationName.equals(Operation.UPDATE) && updatedKeyList.contains(columnType.getName()); } private Schema getNonNullableSchema(Schema.Field field) { diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature index 936802e66..d9cb71e38 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature @@ -173,3 +173,140 @@ Feature: Oracle sink- Verify Oracle sink plugin design time validation scenarios Then Enter input plugin property: "referenceName" with value: "sourceRef" Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "errorMessageInvalidHost" on the header + + @Oracle_Required + Scenario: Verify the validation error message on header with blank database value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.database.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.HostBlank.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario Outline: To verify Oracle sink plugin validation error message for update, upsert operation name and table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Validate button + Then Verify that the Plugin Property: "operationName" is displaying an in-line error message: "errorMessageUpdateUpsertOperationName" + Then Verify that the Plugin Property: "relationTableKey" is displaying an in-line error message: "errorMessageUpdateUpsertOperationName" + Examples: + | options | + | upsert | + | update | diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index 70b1bdba6..e7cb104c5 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -218,3 +218,229 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @BQ_SOURCE_TEST @ORACLE_TEST_TABLE @CONNECTION + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-Oracle" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Replace input plugin property: "database" with value: "databaseName" + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario Outline: To verify pipeline preview failed with invalid table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | invalidOracleTableKey | + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "Failed" + Examples: + | options | + | upsert | + | update | + + @ORACLE_SOURCE_TEST @ORACLE_UPDATE_TABLE @Oracle_Required + Scenario Outline: To verify data is getting transferred from Oracle to Oracle successfully using update operation with table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | oracleTableKey | + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on the Preview Data link on the Sink plugin node: "Oracle" + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + Examples: + | options | + | update | + + @ORACLE_SOURCE_TEST @ORACLE_UPSERT_TABLE @Oracle_Required + Scenario Outline: To verify data is getting transferred from Oracle to Oracle successfully using upsert operation with table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | oracleTableKey | + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on the Preview Data link on the Sink plugin node: "Oracle" + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + Examples: + | options | + | upsert | diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature index 78130655f..74e5302c6 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature @@ -88,3 +88,135 @@ Feature: Oracle - Verify data transfer to Oracle sink with macro arguments Then Close the pipeline logs Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @ORACLE_SOURCE_TEST @ORACLE_UPSERT_TABLE @Oracle_Required + Scenario: To verify data is getting transferred from Oracle to Oracle successfully with connection argument, transaction isolation, operationName with upsert macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "transactionIsolationLevel" and set the value to: "transactionIsolationLevel" + Then Click on the Macro button of Property: "operationName" and set the value to: "oracleOperationName" + Then Click on the Macro button of Property: "relationTableKey" and set the value to: "oracleTableKey" + Then Validate "Oracle2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "upsertOperationName" for key "oracleOperationName" + Then Enter runtime argument value "upsertRelationTableKey" for key "oracleTableKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "upsertOperationName" for key "oracleOperationName" + Then Enter runtime argument value "upsertRelationTableKey" for key "oracleTableKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target table is equal to the values from source table + + @ORACLE_SOURCE_TEST @ORACLE_UPDATE_TABLE @Oracle_Required + Scenario: To verify data is getting transferred from Oracle source to Oracle sink using macro arguments for operation name with update + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "transactionIsolationLevel" and set the value to: "transactionIsolationLevel" + Then Click on the Macro button of Property: "operationName" and set the value to: "oracleOperationName" + Then Click on the Macro button of Property: "relationTableKey" and set the value to: "oracleTableKey" + Then Validate "Oracle2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "operationName" for key "oracleOperationName" + Then Enter runtime argument value "relationTableKey" for key "oracleTableKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "operationName" for key "oracleOperationName" + Then Enter runtime argument value "relationTableKey" for key "oracleTableKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java index 815d5309c..3976972db 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java @@ -73,8 +73,8 @@ public static int countRecord(String table, String schema) throws SQLException, */ public static boolean validateRecordValues(String schema, String sourceTable, String targetTable) throws SQLException, ClassNotFoundException { - String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable; - String getTargetQuery = "SELECT * FROM " + schema + "." + targetTable; + String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable + " ORDER BY ID"; + String getTargetQuery = "SELECT * FROM " + schema + "." + targetTable + " ORDER BY ID"; try (Connection connect = getOracleConnection()) { connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE, @@ -193,6 +193,34 @@ public static void createTargetTable(String targetTable, String schema) throws S } } + public static void createOracleSinkUpdateTable(String sourceTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + + "(ID number(38), LASTNAME varchar2(100))"; + statement.executeUpdate(createSourceTableQuery); + + // Insert dummy data. + statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" + + " VALUES (1, 'Maria')"); + statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" + + " VALUES (2, 'Shelly')"); + } + } + + public static void createOracleSinkUpsertTable(String sourceTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { + String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + + "(ID number(38), LASTNAME varchar2(100))"; + statement.executeUpdate(createSourceTableQuery); + + // Insert dummy data. + statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" + + " VALUES (1, 'John')"); + } + } + public static void createSourceDatatypesTable(String sourceTable, String schema) throws SQLException, ClassNotFoundException { try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) { diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java index 757b9c9f7..eb53de3cd 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java @@ -292,6 +292,38 @@ public static void dropOracleTargetTestTable() throws SQLException, ClassNotFoun + " deleted successfully"); } + @Before(order = 2, value = "@ORACLE_UPDATE_TABLE") + public static void createOracleTargetUpdateTable() throws SQLException, ClassNotFoundException { + OracleClient.createOracleSinkUpdateTable(PluginPropertyUtils.pluginProp("targetTable"), + PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " created successfully"); + } + + @After(order = 2, value = "@ORACLE_UPDATE_TABLE") + public static void dropOracleTargetUpdateTable() throws SQLException, ClassNotFoundException { + OracleClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " deleted successfully"); + } + + @Before(order = 2, value = "@ORACLE_UPSERT_TABLE") + public static void createOracleTargetUpsertTable() throws SQLException, ClassNotFoundException { + OracleClient.createOracleSinkUpsertTable(PluginPropertyUtils.pluginProp("targetTable"), + PluginPropertyUtils.pluginProp("schema")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " created successfully"); + } + + @After(order = 2, value = "@ORACLE_UPSERT_TABLE") + public static void dropOracleTargetUpsertTable() throws SQLException, ClassNotFoundException { + OracleClient.deleteTable(PluginPropertyUtils.pluginProp("schema"), + PluginPropertyUtils.pluginProp("targetTable")); + BeforeActions.scenario.write("Oracle Target Table - " + PluginPropertyUtils.pluginProp("targetTable") + + " deleted successfully"); + } + @Before(order = 1, value = "@BQ_SINK_TEST") public static void setTempTargetBQTableName() { String bqTargetTableName = "E2E_TARGET_" + UUID.randomUUID().toString().replaceAll("-", "_"); diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index e44f4c00a..2a1cea51e 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -18,4 +18,6 @@ errorMessageInvalidHost=Exception while trying to validate schema of database ta errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. blank.database.message=Required property 'database' has no value. -blank.connection.message=Exception while trying to validate schema of database table +blank.connection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '72000', errorCode: '1005', errorMessage: SQL Exception occurred: [Message='ORA-01005: null password given; logon denied ', SQLState='72000', ErrorCode='1005'].' +blank.HostBlank.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: The Network Adapter could not establish the connection', SQLState='08006', ErrorCode='17002'].' +errorMessageUpdateUpsertOperationName=Table key must be set if the operation is 'Update' or 'Upsert'. diff --git a/oracle-plugin/src/e2e-test/resources/pluginParameters.properties b/oracle-plugin/src/e2e-test/resources/pluginParameters.properties index c71e82b53..d16a7b450 100644 --- a/oracle-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/oracle-plugin/src/e2e-test/resources/pluginParameters.properties @@ -84,6 +84,7 @@ invalidPassword=testPassword invalidBoundingQuery=SELECT MIN(id),MAX(id) FROM table invalidBoundingQueryValue=select; invalidTable=table +invalidOracleTableKey=asdas #ORACLE Valid Properties connectionArgumentsList=[{"key":"queryTimeout","value":"-1"}] @@ -93,6 +94,13 @@ numberOfSplits=2 zeroValue=0 splitByColumn=ID importQuery=where $CONDITIONS +connectionArguments=queryTimeout=50 +transactionIsolationLevel=TRANSACTION_READ_COMMITTED +operationName=update +oracleTableKey=ID +relationTableKey=ID +upsertOperationName=upsert +upsertRelationTableKey=ID #bq properties projectId=cdf-athena diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java new file mode 100644 index 000000000..4a7b18278 --- /dev/null +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.plugin.db.sink.ETLDBOutputFormat; + +/** + * Class that extends {@link ETLDBOutputFormat} to implement the abstract methods + */ +public class OracleETLDBOutputFormat extends ETLDBOutputFormat { + + /** + * This method is used to construct the upsert query for Oracle using MERGE statement. + * Example - MERGE INTO my_table target + * USING (SELECT ? AS id, ? AS name, ? AS age FROM dual) source + * ON (target.id = source.id) + * WHEN MATCHED THEN UPDATE SET target.name = source.name, target.age = source.age + * WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (source.id, source.name, source.age) + * @param table - Name of the table + * @param fieldNames - All the columns of the table + * @param listKeys - The columns used as keys for matching + * @return Upsert query in the form of string + */ + @Override + public String constructUpsertQuery(String table, String[] fieldNames, String[] listKeys) { + if (listKeys == null) { + throw new IllegalArgumentException( + "'Relation Table Key' must be specified for upsert operations. " + + "Please provide the list of key columns used to match records in the target table."); + } else if (fieldNames == null) { + throw new IllegalArgumentException( + "'Field Names' must be specified for upsert operations. " + + "Please provide the list of columns to be written to the target table."); + } else { + StringBuilder query = new StringBuilder(); + + // MERGE INTO target_table target + query.append("MERGE INTO ").append(table).append(" target "); + + // USING (SELECT ? AS col1, ? AS col2, ... FROM dual) source + query.append("USING (SELECT "); + for (int i = 0; i < fieldNames.length; ++i) { + query.append("? AS ").append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(" FROM dual) source "); + + // ON (target.key1 = source.key1 AND target.key2 = source.key2 ...) + query.append("ON ("); + for (int i = 0; i < listKeys.length; ++i) { + query.append("target.").append(listKeys[i]).append(" = source.").append(listKeys[i]); + if (i != listKeys.length - 1) { + query.append(" AND "); + } + } + query.append(") "); + + // WHEN MATCHED THEN UPDATE SET target.col1 = source.col1, target.col2 = source.col2 ... + // Only update non-key columns + query.append("WHEN MATCHED THEN UPDATE SET "); + boolean firstUpdateColumn = true; + for (String fieldName : fieldNames) { + boolean isKeyColumn = false; + for (String listKey : listKeys) { + String listKeyNoQuote = listKey.replace("\"", ""); + if (listKeyNoQuote.equals(fieldName)) { + isKeyColumn = true; + break; + } + } + if (!isKeyColumn) { + if (!firstUpdateColumn) { + query.append(", "); + } + query.append("target.").append(fieldName).append(" = source.").append(fieldName); + firstUpdateColumn = false; + } + } + + // WHEN NOT MATCHED THEN INSERT (col1, col2, ...) VALUES (source.col1, source.col2, ...) + query.append(" WHEN NOT MATCHED THEN INSERT ("); + for (int i = 0; i < fieldNames.length; ++i) { + query.append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(") VALUES ("); + for (int i = 0; i < fieldNames.length; ++i) { + query.append("source.").append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(")"); + + return query.toString(); + } + } + + @Override + public String constructUpdateQuery(String table, String[] fieldNames, String[] listKeys) { + // Oracle JDBC does not accept a trailing semicolon in prepared statements. + String query = super.constructUpdateQuery(table, fieldNames, listKeys); + if (query.endsWith(";")) { + return query.substring(0, query.length() - 1); + } + return query; + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 511281e9d..415cfa660 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -23,6 +23,7 @@ import io.cdap.cdap.api.annotation.MetadataProperty; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Output; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSink; @@ -31,6 +32,8 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSinkConfig; @@ -59,7 +62,8 @@ public OracleSink(OracleSinkConfig oracleSinkConfig) { @Override protected DBRecord getDBRecord(StructuredRecord output) { - return new OracleSinkDBRecord(output, columnTypes); + return new OracleSinkDBRecord(output, columnTypes, oracleSinkConfig.getOperationName(), + oracleSinkConfig.getRelationTableKey()); } @Override @@ -71,6 +75,13 @@ protected FieldsValidator getFieldsValidator() { protected SchemaReader getSchemaReader() { return new OracleSinkSchemaReader(); } + + @Override + protected void addOutputContext(BatchSinkContext context) { + context.addOutput(Output.of(oracleSinkConfig.getReferenceName(), + new SinkOutputFormatProvider(OracleETLDBOutputFormat.class, getConfiguration()))); + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String fqn = DBUtils.constructFQN("oracle", diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java index 01b9a8247..ee77a7fc9 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java @@ -19,6 +19,7 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.db.Operation; import io.cdap.plugin.db.SchemaReader; import java.sql.PreparedStatement; @@ -31,9 +32,12 @@ */ public class OracleSinkDBRecord extends OracleSourceDBRecord { - public OracleSinkDBRecord(StructuredRecord record, List columnTypes) { + public OracleSinkDBRecord(StructuredRecord record, List columnTypes, Operation operationName, + String relationTableKey) { this.record = record; this.columnTypes = columnTypes; + this.operationName = operationName; + this.relationTableKey = relationTableKey; } @Override @@ -50,4 +54,13 @@ protected void insertOperation(PreparedStatement stmt) throws SQLException { writeToDB(stmt, field, fieldIndex); } } + + @Override + protected void upsertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + Schema.Field field = record.getSchema().getField(columnType.getName()); + writeToDB(stmt, field, fieldIndex); + } + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java index 7d7c69d2b..44131a01b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java @@ -116,8 +116,8 @@ protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordB @Override protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, String fieldName, int fieldIndex) throws SQLException { - int sqlType = columnTypes.get(fieldIndex).getType(); int sqlIndex = fieldIndex + 1; + int sqlType = modifiableColumnTypes.get(fieldIndex).getType(); // TIMESTAMP and TIMESTAMPTZ types needs to be handled using the specific oracle types to ensure that the data // inserted matches with the provided value. As Oracle driver internally alters the values provided diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java new file mode 100644 index 000000000..994c86b55 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import org.junit.Assert; +import org.junit.Test; + +public class OracleETLDBOutputFormatTest { + + private final OracleETLDBOutputFormat outputFormat = new OracleETLDBOutputFormat(); + + @Test + public void testConstructUpsertQueryBasic() { + String[] fieldNames = {"id", "name", "age"}; + String[] listKeys = {"id"}; + String table = "my_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO my_table target " + + "USING (SELECT ? AS id, ? AS name, ? AS age FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name, target.age = source.age " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (source.id, source.name, source.age)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQueryMultipleKeys() { + String[] fieldNames = {"id", "code", "name", "value"}; + String[] listKeys = {"id", "code"}; + String table = "composite_key_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO composite_key_table target " + + "USING (SELECT ? AS id, ? AS code, ? AS name, ? AS value FROM dual) source " + + "ON (target.id = source.id AND target.code = source.code) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name, target.value = source.value " + + "WHEN NOT MATCHED THEN INSERT (id, code, name, value) VALUES (source.id, source.code, source.name, source" + + ".value)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQuerySingleField() { + String[] fieldNames = {"id", "name"}; + String[] listKeys = {"id"}; + String table = "single_field_update_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO single_field_update_table target " + + "USING (SELECT ? AS id, ? AS name FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) VALUES (source.id, source.name)"; + + Assert.assertEquals(expected, result); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructUpsertQueryNullListKeys() { + String[] fieldNames = {"id", "name", "age"}; + String table = "my_table"; + + outputFormat.constructUpsertQuery(table, fieldNames, null); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructUpsertQueryNullFieldNames() { + String[] listKeys = {"id"}; + String table = "my_table"; + + outputFormat.constructUpsertQuery(table, null, listKeys); + } + + @Test + public void testConstructUpsertQueryAllFieldsAreKeys() { + String[] fieldNames = {"id", "code"}; + String[] listKeys = {"id", "code"}; + String table = "all_keys_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + // When all fields are keys, the UPDATE SET clause will be empty after "SET " + // Note: There's an extra space before "WHEN NOT MATCHED" due to implementation + String expected = "MERGE INTO all_keys_table target " + + "USING (SELECT ? AS id, ? AS code FROM dual) source " + + "ON (target.id = source.id AND target.code = source.code) " + + "WHEN MATCHED THEN UPDATE SET " + + "WHEN NOT MATCHED THEN INSERT (id, code) VALUES (source.id, source.code)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQueryWithSpecialTableName() { + String[] fieldNames = {"id", "name"}; + String[] listKeys = {"id"}; + String table = "SCHEMA.MY_TABLE"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO SCHEMA.MY_TABLE target " + + "USING (SELECT ? AS id, ? AS name FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) VALUES (source.id, source.name)"; + + Assert.assertEquals(expected, result); + } +} diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 8d6168780..e3537e2ba 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -192,6 +192,29 @@ "label": "Schema Name", "name": "dbSchemaName" }, + { + "widget-type": "radio-group", + "label": "Operation Name", + "name": "operationName", + "widget-attributes": { + "default": "insert", + "layout": "inline", + "options": [ + { + "id": "insert", + "label": "INSERT" + }, + { + "id": "update", + "label": "UPDATE" + }, + { + "id": "upsert", + "label": "UPSERT" + } + ] + } + }, { "widget-type": "hidden", "label": "Operation Name", @@ -201,9 +224,10 @@ } }, { - "widget-type": "hidden", + "name": "relationTableKey", + "widget-type": "csv", "label": "Table Key", - "name": "relationTableKey" + "widget-attributes": {} } ] }, From 99951d337d40aa96629e491ab21297749992571e Mon Sep 17 00:00:00 2001 From: suryakumari Date: Mon, 6 Apr 2026 19:34:03 +0530 Subject: [PATCH 35/36] cdap e2e framework version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f59a221d6..c7259d221 100644 --- a/pom.xml +++ b/pom.xml @@ -721,7 +721,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.4.0 + 0.4.1 test From 081c0d5d8f012f5beee0814fab8d7f854325ff19 Mon Sep 17 00:00:00 2001 From: vedanshugarg04 Date: Thu, 21 May 2026 17:11:30 +0530 Subject: [PATCH 36/36] updated cdap-e2e-framework from 0.4.1 to 0.4.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c7259d221..fa94b54b0 100644 --- a/pom.xml +++ b/pom.xml @@ -721,7 +721,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.4.1 + 0.4.2 test