Cdap e2e framework update

.github/workflows/e2e.yml

@@ -65,6 +65,7 @@ jobs:
         with:
           repository: cdapio/cdap-e2e-tests
           path: e2e
+          ref: release/6.11
 
       - name: Cache
         uses: actions/cache@v4

amazon-redshift-plugin/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>database-plugins-parent</artifactId>
     <groupId>io.cdap.plugin</groupId>
-    <version>1.12.0-SNAPSHOT</version>
+    <version>1.12.4-SNAPSHOT</version>
   </parent>
 
   <name>Amazon Redshift plugin</name>

aurora-mysql-plugin/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>database-plugins-parent</artifactId>
     <groupId>io.cdap.plugin</groupId>
-    <version>1.12.0-SNAPSHOT</version>
+    <version>1.12.4-SNAPSHOT</version>
   </parent>
 
   <name>Aurora DB MySQL plugin</name>

aurora-postgresql-plugin/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>database-plugins-parent</artifactId>
     <groupId>io.cdap.plugin</groupId>
-    <version>1.12.0-SNAPSHOT</version>
+    <version>1.12.4-SNAPSHOT</version>
   </parent>
 
   <name>Aurora DB PostgreSQL plugin</name>

cloudsql-mysql-plugin/pom.xml

@@ -20,17 +20,51 @@
   <parent>
     <artifactId>database-plugins-parent</artifactId>
     <groupId>io.cdap.plugin</groupId>
-    <version>1.12.0-SNAPSHOT</version>
+    <version>1.12.4-SNAPSHOT</version>
   </parent>
 
   <name>CloudSQL MySQL plugin</name>
   <artifactId>cloudsql-mysql-plugin</artifactId>
   <modelVersion>4.0.0</modelVersion>
+  <description>CloudSQL MySQL database plugins</description>
+  <url>https://github.com/data-integrations/database-plugins</url>
+
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+      <comments>A business-friendly OSS license</comments>
+    </license>
+  </licenses>
+
+  <developers>
+    <developer>
+      <name>CDAP</name>
+      <email>cdap-dev@googlegroups.com</email>
+      <organization>CDAP</organization>
+      <organizationUrl>http://cdap.io</organizationUrl>
+    </developer>
+  </developers>
+
+  <scm>
+    <connection>scm:git:https://github.com/cdapio/hydrator-plugins.git</connection>
+    <developerConnection>scm:git:git@github.com:cdapio/hydrator-plugins.git</developerConnection>
+    <url>https://github.com/cdapio/hydrator-plugins.git</url>
+    <tag>HEAD</tag>
+  </scm>
 
   <dependencies>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>cdap-etl-api</artifactId>
+      <version>${cdap.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.cdap.cdap</groupId>
+      <artifactId>cdap-api</artifactId>
+      <version>${cdap.version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
@@ -41,11 +75,12 @@
     <dependency>
       <groupId>io.cdap.plugin</groupId>
       <artifactId>hydrator-common</artifactId>
+      <version>${cdap.plugin.version}</version>
     </dependency>
     <dependency>
       <groupId>io.cdap.plugin</groupId>
       <artifactId>mysql-plugin</artifactId>
-      <version>1.12.0-SNAPSHOT</version>
+      <version>${project.version}</version>
     </dependency>
 
     <!-- test dependencies -->
@@ -59,24 +94,26 @@
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>hydrator-test</artifactId>
+      <version>${cdap.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>cdap-data-pipeline3_2.12</artifactId>
+      <version>${cdap.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.cdap.cdap</groupId>
-      <artifactId>cdap-api</artifactId>
-      <scope>provided</scope>
+      <version>${junit.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
+      <version>${mockito.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.jetbrains</groupId>

cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties

@@ -12,10 +12,12 @@ errorMessageNumberOfSplits=Split-By Field Name must be specified if Number of Sp
 errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. Specify the Bounding Query.
 errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'URLDecoder: Illegal hex characters in escape (%) pattern - For input string: "$^"'
 errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table 'Invalidtable' is set correctly and 
-errorMessageConnectionName=Connection Name must be in the format <PROJECT_ID>:<REGION>:<INSTANCE_NAME> to connect to a public CloudSQL PostgreSQL instance.
+errorMessageConnectionName=Connection Name must be in the format <PROJECT_ID>:<REGION>:<INSTANCE_NAME> to connect to a public CloudSQL MySQL instance.
 validationSuccessMessage=No errors found.
 validationErrorMessage=COUNT ERROR found
-errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \
-  Table 
-errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Errors were encountered during validation.
-errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details.
+errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \
+  Table 'Table123' does not exist.. Please check the system logs for more details.
+errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \
+  Exception while trying to validate schema of database table 
+errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : java.io.IOException: You have an error in your SQL syntax; \
+  check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details.

cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java

@@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
       CloudSQLUtil.checkConnectionName(
           failureCollector,
           cloudsqlMysqlActionConfig.instanceType,
-          cloudsqlMysqlActionConfig.connectionName);
+          cloudsqlMysqlActionConfig.connectionName,
+          CloudSQLUtil.CLOUDSQL_MYSQL);
     }
 
     super.configurePipeline(pipelineConfigurer);

cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java

@@ -74,7 +74,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
       CloudSQLUtil.checkConnectionName(
         failureCollector,
         cloudsqlMysqlSinkConfig.connection.getInstanceType(),
-        cloudsqlMysqlSinkConfig.connection.getConnectionName());
+        cloudsqlMysqlSinkConfig.connection.getConnectionName(),
+        CloudSQLUtil.CLOUDSQL_MYSQL);
     }
 
     super.configurePipeline(pipelineConfigurer);
@@ -108,6 +109,11 @@ protected String getErrorDetailsProviderClassName() {
     return CloudSQLMySQLErrorDetailsProvider.class.getName();
   }
 
+  @Override
+  protected String getExternalDocumentationLink() {
+    return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL;
+  }
+
   @Override
   protected LineageRecorder getLineageRecorder(BatchSinkContext context) {
     String host;

cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java

@@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
       CloudSQLUtil.checkConnectionName(
         failureCollector,
         cloudsqlMysqlSourceConfig.connection.getInstanceType(),
-        cloudsqlMysqlSourceConfig.connection.getConnectionName());
+        cloudsqlMysqlSourceConfig.connection.getConnectionName(),
+        CloudSQLUtil.CLOUDSQL_MYSQL);
     }
 
     super.configurePipeline(pipelineConfigurer);

cloudsql-postgresql-plugin/pom.xml

@@ -20,17 +20,51 @@
   <parent>
     <artifactId>database-plugins-parent</artifactId>
     <groupId>io.cdap.plugin</groupId>
-    <version>1.12.0-SNAPSHOT</version>
+    <version>1.12.4-SNAPSHOT</version>
   </parent>
 
   <name>CloudSQL PostgreSQL plugin</name>
   <artifactId>cloudsql-postgresql-plugin</artifactId>
   <modelVersion>4.0.0</modelVersion>
+  <description>CloudSQL PostgreSQL database plugins</description>
+  <url>https://github.com/data-integrations/database-plugins</url>
+
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+      <comments>A business-friendly OSS license</comments>
+    </license>
+  </licenses>
+
+  <developers>
+    <developer>
+      <name>CDAP</name>
+      <email>cdap-dev@googlegroups.com</email>
+      <organization>CDAP</organization>
+      <organizationUrl>http://cdap.io</organizationUrl>
+    </developer>
+  </developers>
+
+  <scm>
+    <connection>scm:git:https://github.com/cdapio/hydrator-plugins.git</connection>
+    <developerConnection>scm:git:git@github.com:cdapio/hydrator-plugins.git</developerConnection>
+    <url>https://github.com/cdapio/hydrator-plugins.git</url>
+    <tag>HEAD</tag>
+  </scm>
 
   <dependencies>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>cdap-etl-api</artifactId>
+      <version>${cdap.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.cdap.cdap</groupId>
+      <artifactId>cdap-api</artifactId>
+      <version>${cdap.version}</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
@@ -41,6 +75,7 @@
     <dependency>
       <groupId>io.cdap.plugin</groupId>
       <artifactId>hydrator-common</artifactId>
+      <version>${cdap.plugin.version}</version>
     </dependency>
     <dependency>
       <groupId>io.cdap.plugin</groupId>
@@ -63,24 +98,26 @@
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>hydrator-test</artifactId>
+      <version>${cdap.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
       <artifactId>cdap-data-pipeline3_2.12</artifactId>
+      <version>${cdap.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.cdap.cdap</groupId>
-      <artifactId>cdap-api</artifactId>
-      <scope>provided</scope>
+      <version>${junit.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
+      <version>${mockito.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.jetbrains</groupId>

cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java

@@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
       CloudSQLUtil.checkConnectionName(
           failureCollector,
           cloudsqlPostgresqlActionConfig.instanceType,
-          cloudsqlPostgresqlActionConfig.connectionName);
+          cloudsqlPostgresqlActionConfig.connectionName,
+          CloudSQLUtil.CLOUDSQL_POSTGRESQL);
     }
 
     super.configurePipeline(pipelineConfigurer);

cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java

@@ -81,7 +81,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
       CloudSQLUtil.checkConnectionName(
         failureCollector,
         cloudsqlPostgresqlSinkConfig.connection.getInstanceType(),
-        cloudsqlPostgresqlSinkConfig.connection.getConnectionName());
+        cloudsqlPostgresqlSinkConfig.connection.getConnectionName(),
+        CloudSQLUtil.CLOUDSQL_POSTGRESQL);
     }
 
     super.configurePipeline(pipelineConfigurer);
@@ -152,6 +153,11 @@ protected String getErrorDetailsProviderClassName() {
     return CloudSQLPostgreSQLErrorDetailsProvider.class.getName();
   }
 
+  @Override
+  protected String getExternalDocumentationLink() {
+    return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL;
+  }
+
   /** CloudSQL PostgreSQL sink config. */
   public static class CloudSQLPostgreSQLSinkConfig extends AbstractDBSpecificSinkConfig {
 

cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java

@@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) {
       CloudSQLUtil.checkConnectionName(
         failureCollector,
         cloudsqlPostgresqlSourceConfig.connection.getInstanceType(),
-        cloudsqlPostgresqlSourceConfig.connection.getConnectionName());
+        cloudsqlPostgresqlSourceConfig.connection.getConnectionName(),
+        CloudSQLUtil.CLOUDSQL_POSTGRESQL);
     }
 
     super.configurePipeline(pipelineConfigurer);