diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml new file mode 100644 index 000000000..691236ceb --- /dev/null +++ b/.github/workflows/build-report.yml @@ -0,0 +1,51 @@ +# Copyright © 2024 Cask Data, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven +# Note: Any changes to this workflow would be used only after merging into develop +name: Build Unit Tests Report + +on: + workflow_run: + workflows: + - Build with unit tests + types: + - completed + +jobs: + build: + runs-on: ubuntu-latest + + if: ${{ github.event.workflow_run.conclusion != 'skipped' }} + + steps: + # Pinned 1.0.0 version + - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + path: artifacts/ + + - name: Surefire Report + # Pinned 3.5.2 version + uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 + if: always() + with: + report_paths: '**/target/surefire-reports/TEST-*.xml' + github_token: ${{ secrets.GITHUB_TOKEN }} + detailed_summary: true + commit: ${{ github.event.workflow_run.head_sha }} + check_name: Build Test Report + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ce0eb526..55cd4617e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,28 +15,34 @@ name: Build with unit tests on: - workflow_run: - workflows: - - Trigger build - types: - - completed + push: + branches: [ develop, release/** ] + pull_request: + branches: [ develop, release/** ] + types: [opened, synchronize, reopened, labeled] jobs: build: runs-on: k8s-runner-build - if: ${{ github.event.workflow_run.conclusion != 'skipped' }} - + # We allow builds: + # 1) When it's a merge into a branch + # 2) For PRs that are labeled as build and + # - It's a code change + # - A build label was just added + # A bit complex, but prevents builds when other labels are manipulated + if: > + github.event_name == 'push' + || (contains(github.event.pull_request.labels.*.name, 'build') + && (github.event.action != 'labeled' || github.event.label.name == 'build') + ) steps: - # Pinned 1.0.0 version - - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94 - - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.workflow_run.head_sha }} - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} @@ -47,21 +53,12 @@ jobs: run: mvn clean test -fae -T 2 -B -V -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - name: Archive build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: Build debug files + name: reports-${{ github.run_id }} path: | **/target/rat.txt **/target/surefire-reports/* - - name: Surefire Report - # Pinned 3.5.2 version - uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32 - if: always() - with: - report_paths: '**/target/surefire-reports/TEST-*.xml' - github_token: ${{ secrets.GITHUB_TOKEN }} - detailed_summary: true - commit: ${{ github.event.workflow_run.head_sha }} - check_name: Test Report \ No newline at end of file + diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1ced579cb..ab3be13ff 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,9 +16,9 @@ name: Build e2e tests on: push: - branches: [ develop ] + branches: [ develop, release/** ] pull_request: - branches: [ develop ] + branches: [ develop, release/** ] types: [ opened, synchronize, reopened, labeled ] workflow_dispatch: @@ -45,7 +45,7 @@ jobs: steps: # Pinned 1.0.0 version - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: plugin submodules: 'recursive' @@ -61,13 +61,14 @@ jobs: - '${{ matrix.module }}/**/e2e-test/**' - name: Checkout e2e test repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: cdapio/cdap-e2e-tests path: e2e + ref: release/6.11 - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }} @@ -156,24 +157,21 @@ jobs: CLOUDSQL_MYSQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_PASSWORD }} CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }} - - name: Upload report - uses: actions/upload-artifact@v3 - if: always() - with: - name: Cucumber report - ${{ matrix.module }} - path: ./**/target/cucumber-reports - - name: Upload debug files - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: Debug files - ${{ matrix.module }} path: ./**/target/e2e-debug - name: Upload files to GCS - uses: google-github-actions/upload-cloud-storage@v0 + uses: google-github-actions/upload-cloud-storage@v2 if: always() with: path: ./plugin destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }} glob: '**/target/cucumber-reports/**' + + - name: Cucumber Report URL + if: always() + run: echo "https://storage.googleapis.com/e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}/plugin/${{ matrix.module }}/target/cucumber-reports/advanced-reports/cucumber-html-reports/overview-features.html" diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml deleted file mode 100644 index 11db8ac25..000000000 --- a/.github/workflows/trigger.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright © 2022 Cask Data, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -# This workflow will trigger build.yml only when needed. -# This way we don't flood main workflow run list -# Note that build.yml from develop will be used even for PR builds -# Also it will have access to the proper GITHUB_SECRET - -name: Trigger build - -on: - push: - branches: [ develop, release/** ] - pull_request: - branches: [ develop, release/** ] - types: [opened, synchronize, reopened, labeled] - workflow_dispatch: - -jobs: - trigger: - runs-on: ubuntu-latest - - # We allow builds: - # 1) When triggered manually - # 2) When it's a merge into a branch - # 3) For PRs that are labeled as build and - # - It's a code change - # - A build label was just added - # A bit complex, but prevents builds when other labels are manipulated - if: > - github.event_name == 'workflow_dispatch' - || github.event_name == 'push' - || (contains(github.event.pull_request.labels.*.name, 'build') - && (github.event.action != 'labeled' || github.event.label.name == 'build') - ) - steps: - - name: Trigger build - run: echo Maven build will be triggered now \ No newline at end of file diff --git a/amazon-redshift-plugin/docs/Redshift-batchsource.md b/amazon-redshift-plugin/docs/Redshift-batchsource.md new file mode 100644 index 000000000..38873b15a --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-batchsource.md @@ -0,0 +1,102 @@ +# Amazon Redshift Batch Source + +Description +----------- +Reads from an Amazon Redshift database using a configurable SQL query. +Outputs one record for each row returned by the query. + + +Use Case +-------- +The source is used whenever you need to read from an Amazon Redshift database. For example, you may want +to create daily snapshots of a database table by using this source and writing to +a TimePartitionedFileSet. + + +Properties +---------- +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host URL of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Import Query:** The SELECT query to use to import data from the specified table. +You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should +contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'. +The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query. +The '$CONDITIONS' string is not required if numSplits is set to one. + +**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field. +For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one. + +**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one. + +**Number of Splits to Generate:** Number of splits to generate. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. + +**Schema:** The schema of records output by the source. This will be used in place of whatever schema comes +back from the query. However, it must match the schema that comes back from the query, +except it can mark fields as nullable and can contain a subset of the fields. + +**Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import, +with the tradeoff of higher memory usage. + +Example +------ +Suppose you want to read data from an Amazon Redshift database named "prod" that is running on +"redshift.xyz.eu-central-1.redshift.amazonaws.com", port 5439, as "sa" user with "Test11" password. +Ensure that the driver for Redshift is installed (you can also provide driver name for some specific driver, +otherwise "redshift" will be used), then configure the plugin with:then configure plugin with: + +``` +Reference Name: "src1" +Driver Name: "redshift" +Host: "redshift.xyz.eu-central-1.redshift.amazonaws.com" +Port: 5439 +Database: "prod" +Import Query: "select id, name, email, phone from users;" +Number of Splits to Generate: 1 +Username: "sa" +Password: "Test11" +``` + +Data Types Mapping +------------------ + +Mapping of Redshift types to CDAP schema: + +| Redshift Data Type | CDAP Schema Data Type | Comment | +|-----------------------------------------------------|-----------------------|----------------------------------| +| bigint | long | | +| boolean | boolean | | +| character | string | | +| character varying | string | | +| double precision | double | | +| integer | int | | +| numeric(precision, scale)/decimal(precision, scale) | decimal | | +| numeric(with 0 precision) | string | | +| real | float | | +| smallint | int | | +| smallserial | int | | +| text | string | | +| date | date | | +| time [ (p) ] [ without time zone ] | time | | +| time [ (p) ] with time zone | string | | +| timestamp [ (p) ] [ without time zone ] | timestamp | | +| timestamp [ (p) ] with time zone | timestamp | stored in UTC format in database | +| xml | string | | +| json | string | | +| super | string | | +| geometry | bytes | | +| hllsketch | string | | diff --git a/amazon-redshift-plugin/docs/Redshift-connector.md b/amazon-redshift-plugin/docs/Redshift-connector.md new file mode 100644 index 000000000..368d9e09f --- /dev/null +++ b/amazon-redshift-plugin/docs/Redshift-connector.md @@ -0,0 +1,26 @@ +# Amazon Redshift Connection + +Description +----------- +Use this connection to access data in an Amazon Redshift database using JDBC. + +Properties +---------- +**Name:** Name of the connection. Connection names must be unique in a namespace. + +**Description:** Description of the connection. + +**JDBC Driver name:** Name of the JDBC driver to use. + +**Host:** Host of the current master instance of Redshift cluster. + +**Port:** Port that Redshift master instance is listening to. + +**Database:** Redshift database name. + +**Username:** User identity for connecting to the specified database. + +**Password:** Password to use to connect to the specified database. + +**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments +will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/amazon-redshift-plugin/icons/Redshift-batchsource.png b/amazon-redshift-plugin/icons/Redshift-batchsource.png new file mode 100644 index 000000000..11c334799 Binary files /dev/null and b/amazon-redshift-plugin/icons/Redshift-batchsource.png differ diff --git a/amazon-redshift-plugin/pom.xml b/amazon-redshift-plugin/pom.xml new file mode 100644 index 000000000..9a545ef6b --- /dev/null +++ b/amazon-redshift-plugin/pom.xml @@ -0,0 +1,139 @@ + + + + + database-plugins-parent + io.cdap.plugin + 1.12.4-SNAPSHOT + + + Amazon Redshift plugin + amazon-redshift-plugin + 4.0.0 + + + 2.1.0.18 + + + + + redshift + http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release + + + + + + io.cdap.cdap + cdap-etl-api + + + io.cdap.plugin + database-commons + ${project.version} + + + io.cdap.plugin + hydrator-common + + + com.google.guava + guava + + + + + com.amazon.redshift + redshift-jdbc42 + ${redshift-jdbc.version} + test + + + io.cdap.plugin + database-commons + ${project.version} + test-jar + test + + + io.cdap.cdap + hydrator-test + + + io.cdap.cdap + cdap-data-pipeline3_2.12 + + + junit + junit + + + org.mockito + mockito-core + test + + + io.cdap.cdap + cdap-api + provided + + + org.jetbrains + annotations + RELEASE + compile + + + + + + io.cdap + cdap-maven-plugin + + + org.apache.felix + maven-bundle-plugin + 5.1.2 + true + + + <_exportcontents> + io.cdap.plugin.amazon.redshift.*; + io.cdap.plugin.db.source.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + + diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java new file mode 100644 index 000000000..fb8cac4a7 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnector.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Category; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.cdap.etl.api.connector.ConnectorSpec; +import io.cdap.cdap.etl.api.connector.ConnectorSpecRequest; +import io.cdap.cdap.etl.api.connector.PluginSpec; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.ReferenceNames; +import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBPath; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Amazon Redshift Database Connector that connects to Amazon Redshift database via JDBC. + */ +@Plugin(type = Connector.PLUGIN_TYPE) +@Name(RedshiftConnector.NAME) +@Description("Connection to access data in Amazon Redshift using JDBC.") +@Category("Database") +public class RedshiftConnector extends AbstractDBSpecificConnector { + public static final String NAME = RedshiftConstants.PLUGIN_NAME; + private final RedshiftConnectorConfig config; + + public RedshiftConnector(RedshiftConnectorConfig config) { + super(config); + this.config = config; + } + + @Override + protected DBConnectorPath getDBConnectorPath(String path) throws IOException { + return new DBPath(path, true); + } + + @Override + public boolean supportSchema() { + return true; + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + public StructuredRecord transform(LongWritable longWritable, RedshiftDBRecord redshiftDBRecord) { + return redshiftDBRecord.getRecord(); + } + + @Override + protected SchemaReader getSchemaReader(String sessionID) { + return new RedshiftSchemaReader(sessionID); + } + + @Override + protected String getTableName(String database, String schema, String table) { + return String.format("\"%s\".\"%s\"", schema, table); + } + + @Override + protected String getRandomQuery(String tableName, int limit) { + return String.format("SELECT * FROM %s\n" + + "TABLESAMPLE BERNOULLI (100.0 * %d / (SELECT COUNT(*) FROM %s))", + tableName, limit, tableName); + } + + @Override + protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath path, + ConnectorSpec.Builder builder) { + Map sourceProperties = new HashMap<>(); + setConnectionProperties(sourceProperties, request); + builder + .addRelatedPlugin(new PluginSpec(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, sourceProperties)); + + String schema = path.getSchema(); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.NUM_SPLITS, "1"); + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.FETCH_SIZE, + RedshiftSource.RedshiftSourceConfig.DEFAULT_FETCH_SIZE); + String table = path.getTable(); + if (table == null) { + return; + } + sourceProperties.put(RedshiftSource.RedshiftSourceConfig.IMPORT_QUERY, + getTableQuery(path.getDatabase(), schema, table)); + sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java new file mode 100644 index 000000000..bae0013b3 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorConfig.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; + +import javax.annotation.Nullable; + +/** + * Configuration for Redshift connector + */ +public class RedshiftConnectorConfig extends AbstractDBConnectorConfig { + + @Name(ConnectionConfig.HOST) + @Description( + "The endpoint of the Amazon Redshift cluster.") + @Macro + private String host; + + @Name(ConnectionConfig.PORT) + @Description("Database port number") + @Macro + @Nullable + private Integer port; + + @Name(ConnectionConfig.DATABASE) + @Description("Database name to connect to") + @Macro + private String database; + + public RedshiftConnectorConfig(String username, String password, String jdbcPluginName, + String connectionArguments, String host, + String database, @Nullable Integer port) { + this.user = username; + this.password = password; + this.jdbcPluginName = jdbcPluginName; + this.connectionArguments = connectionArguments; + this.host = host; + this.database = database; + this.port = port; + } + + public String getDatabase() { + return database; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port == null ? 5439 : port; + } + + @Override + public String getConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + host, + getPort(), + database); + } + + @Override + public boolean canConnect() { + return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + } +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java new file mode 100644 index 000000000..081052fb1 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftConstants.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +/** Amazon Redshift constants. */ +public final class RedshiftConstants { + + private RedshiftConstants() { + } + + public static final String PLUGIN_NAME = "Redshift"; + public static final String REDSHIFT_CONNECTION_STRING_FORMAT = "jdbc:redshift://%s:%s/%s"; +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java new file mode 100644 index 000000000..38e9140d8 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecord.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.util.DBUtils; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +/** + * Writable class for Redshift Source + */ +public class RedshiftDBRecord extends DBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public RedshiftDBRecord() { + } + + @Override + protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordBuilder, Schema.Field field, + int columnIndex, int sqlType, int sqlPrecision, int sqlScale) throws SQLException { + ResultSetMetaData metadata = resultSet.getMetaData(); + String columnTypeName = metadata.getColumnTypeName(columnIndex); + if (isUseSchema(metadata, columnIndex)) { + setFieldAccordingToSchema(resultSet, recordBuilder, field, columnIndex); + return; + } + + // HandleTimestamp + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamp")) { + Timestamp timestamp = resultSet.getTimestamp(columnIndex, DBUtils.PURE_GREGORIAN_CALENDAR); + if (timestamp != null) { + ZonedDateTime zonedDateTime = OffsetDateTime.of(timestamp.toLocalDateTime(), OffsetDateTime.now().getOffset()) + .atZoneSameInstant(ZoneId.of("UTC")); + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + setZonedDateTimeBasedOnOutputSchema(recordBuilder, nonNullableSchema.getLogicalType(), + field.getName(), zonedDateTime); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleTimestampTZ + if (sqlType == Types.TIMESTAMP && columnTypeName.equalsIgnoreCase("timestamptz")) { + OffsetDateTime timestamp = resultSet.getObject(columnIndex, OffsetDateTime.class); + if (timestamp != null) { + recordBuilder.setTimestamp(field.getName(), timestamp.atZoneSameInstant(ZoneId.of("UTC"))); + } else { + recordBuilder.set(field.getName(), null); + } + return; + } + + // HandleNumeric + int columnType = metadata.getColumnType(columnIndex); + if (columnType == Types.NUMERIC) { + Schema nonNullableSchema = field.getSchema().isNullable() ? + field.getSchema().getNonNullable() : field.getSchema(); + int precision = metadata.getPrecision(columnIndex); + if (precision == 0 && Schema.Type.STRING.equals(nonNullableSchema.getType())) { + // When output schema is set to String for precision less numbers + recordBuilder.set(field.getName(), resultSet.getString(columnIndex)); + } else if (Schema.LogicalType.DECIMAL.equals(nonNullableSchema.getLogicalType())) { + BigDecimal originalDecimalValue = resultSet.getBigDecimal(columnIndex); + if (originalDecimalValue != null) { + BigDecimal newDecimalValue = new BigDecimal(originalDecimalValue.toPlainString()) + .setScale(nonNullableSchema.getScale(), RoundingMode.HALF_EVEN); + recordBuilder.setDecimal(field.getName(), newDecimalValue); + } + } + return; + } + setField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); + } + + private void setZonedDateTimeBasedOnOutputSchema(StructuredRecord.Builder recordBuilder, + Schema.LogicalType logicalType, + String fieldName, + ZonedDateTime zonedDateTime) { + if (Schema.LogicalType.DATETIME.equals(logicalType)) { + recordBuilder.setDateTime(fieldName, zonedDateTime.toLocalDateTime()); + } else if (Schema.LogicalType.TIMESTAMP_MICROS.equals(logicalType)) { + recordBuilder.setTimestamp(fieldName, zonedDateTime); + } + } + + private static boolean isUseSchema(ResultSetMetaData metadata, int columnIndex) throws SQLException { + String columnTypeName = metadata.getColumnTypeName(columnIndex); + // If the column Type Name is present in the String mapped Redshift types then return true. + return RedshiftSchemaReader.STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(columnTypeName); + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java new file mode 100644 index 000000000..df9938a45 --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReader.java @@ -0,0 +1,117 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.db.CommonSchemaReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import java.util.Set; + +/** + * Redshift Schema Reader class + */ +public class RedshiftSchemaReader extends CommonSchemaReader { + + private static final Logger LOG = LoggerFactory.getLogger(RedshiftSchemaReader.class); + + public static final Set STRING_MAPPED_REDSHIFT_TYPES_NAMES = ImmutableSet.of( + "timetz", "money" + ); + + private final String sessionID; + + public RedshiftSchemaReader() { + this(null); + } + + public RedshiftSchemaReader(String sessionID) { + super(); + this.sessionID = sessionID; + } + + @Override + public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLException { + String typeName = metadata.getColumnTypeName(index); + int columnType = metadata.getColumnType(index); + + if (STRING_MAPPED_REDSHIFT_TYPES_NAMES.contains(typeName)) { + return Schema.of(Schema.Type.STRING); + } + if (typeName.equalsIgnoreCase("INT")) { + return Schema.of(Schema.Type.INT); + } + if (typeName.equalsIgnoreCase("BIGINT")) { + return Schema.of(Schema.Type.LONG); + } + + // If it is a numeric type without precision then use the Schema of String to avoid any precision loss + if (Types.NUMERIC == columnType) { + int precision = metadata.getPrecision(index); + if (precision == 0) { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index))); + return Schema.of(Schema.Type.STRING); + } + } + + if (typeName.equalsIgnoreCase("timestamp")) { + return Schema.of(Schema.LogicalType.DATETIME); + } + + return super.getSchema(metadata, index); + } + + @Override + public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { + if (sessionID == null) { + return false; + } + return metadata.getColumnName(index).equals("c_" + sessionID) || + metadata.getColumnName(index).equals("sqn_" + sessionID); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + // Setting up schema as nullable as cdata driver doesn't provide proper information about isNullable. + columnSchema = Schema.nullableOf(columnSchema); + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + +} diff --git a/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java new file mode 100644 index 000000000..6a0df3a2d --- /dev/null +++ b/amazon-redshift-plugin/src/main/java/io/cdap/plugin/amazon/redshift/RedshiftSource.java @@ -0,0 +1,136 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.annotations.VisibleForTesting; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Metadata; +import io.cdap.cdap.api.annotation.MetadataProperty; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.connector.Connector; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; +import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; +import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; + +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Batch source to read from an Amazon Redshift database. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(RedshiftConstants.PLUGIN_NAME) +@Description( + "Reads from a Amazon Redshift database table(s) using a configurable SQL query." + + " Outputs one record for each row returned by the query.") +@Metadata(properties = {@MetadataProperty(key = Connector.PLUGIN_TYPE, value = RedshiftConnector.NAME)}) +public class RedshiftSource + extends AbstractDBSource { + + private final RedshiftSourceConfig redshiftSourceConfig; + + public RedshiftSource(RedshiftSourceConfig redshiftSourceConfig) { + super(redshiftSourceConfig); + this.redshiftSourceConfig = redshiftSourceConfig; + } + + @Override + protected SchemaReader getSchemaReader() { + return new RedshiftSchemaReader(); + } + + @Override + protected Class getDBRecordType() { + return RedshiftDBRecord.class; + } + + @Override + protected String createConnectionString() { + return String.format( + RedshiftConstants.REDSHIFT_CONNECTION_STRING_FORMAT, + redshiftSourceConfig.connection.getHost(), + redshiftSourceConfig.connection.getPort(), + redshiftSourceConfig.connection.getDatabase()); + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("redshift", redshiftSourceConfig.getConnection().getHost(), + redshiftSourceConfig.getConnection().getPort(), + redshiftSourceConfig.getConnection().getDatabase(), + redshiftSourceConfig.getReferenceName()); + Asset.Builder assetBuilder = Asset.builder(redshiftSourceConfig.getReferenceName()).setFqn(fqn); + return new LineageRecorder(context, assetBuilder.build()); + } + + /** + * Redshift source config. + */ + public static class RedshiftSourceConfig extends AbstractDBSpecificSourceConfig { + + @Name(ConfigUtil.NAME_USE_CONNECTION) + @Nullable + @Description("Whether to use an existing connection.") + private Boolean useConnection; + + @Name(ConfigUtil.NAME_CONNECTION) + @Macro + @Nullable + @Description("The existing connection to use.") + private RedshiftConnectorConfig connection; + + @Override + public Map getDBSpecificArguments() { + return Collections.emptyMap(); + } + + @VisibleForTesting + public RedshiftSourceConfig(@Nullable Boolean useConnection, + @Nullable RedshiftConnectorConfig connection) { + this.useConnection = useConnection; + this.connection = connection; + } + + @Override + public Integer getFetchSize() { + Integer fetchSize = super.getFetchSize(); + return fetchSize == null ? Integer.parseInt(DEFAULT_FETCH_SIZE) : fetchSize; + } + + @Override + protected RedshiftConnectorConfig getConnection() { + return connection; + } + + @Override + public void validate(FailureCollector collector) { + ConfigUtil.validateConnection(this, useConnection, connection, collector); + super.validate(collector); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java new file mode 100644 index 000000000..a43eb4302 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorTest.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.plugin.db.connector.DBSpecificConnectorBaseTest; +import org.junit.Test; + +import java.io.IOException; + +/** + * Unit tests for {@link RedshiftConnector} + */ +public class RedshiftConnectorTest extends DBSpecificConnectorBaseTest { + + private static final String JDBC_DRIVER_CLASS_NAME = "com.amazon.redshift.Driver"; + + @Test + public void test() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { + test(new RedshiftConnector( + new RedshiftConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments, host, database, + port)), + JDBC_DRIVER_CLASS_NAME, RedshiftConstants.PLUGIN_NAME); + } +} + diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java new file mode 100644 index 000000000..47e8b0a52 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Unit tests for {@link RedshiftConnector} + */ +public class RedshiftConnectorUnitTest { + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + private static final RedshiftConnector CONNECTOR = new RedshiftConnector(null); + + /** + * Unit test for getTableName() + */ + @Test + public void getTableNameTest() { + Assert.assertEquals("\"schema\".\"table\"", + CONNECTOR.getTableName("db", "schema", "table")); + } + + @Test + public void getRandomQuery() { + Assert.assertEquals("SELECT * FROM TestData\n" + + "TABLESAMPLE BERNOULLI (100.0 * 10 / (SELECT COUNT(*) FROM TestData))", + CONNECTOR.getRandomQuery("TestData", 10)); + } + + @Test + public void getDBRecordType() { + Assert.assertEquals("class io.cdap.plugin.amazon.redshift.RedshiftDBRecord", + CONNECTOR.getDBRecordType().toString()); + } + + /** + * Unit tests for getTableQuery() + */ + @Test + public void getTableQueryTest() { + String tableName = CONNECTOR.getTableName("db", "schema", "table"); + + // random query + Assert.assertEquals(String.format("SELECT * FROM %s\n" + + "TABLESAMPLE BERNOULLI (100.0 * %d / (SELECT COUNT(*) FROM %s))", + tableName, 100, tableName), + CONNECTOR.getRandomQuery(tableName, 100)); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java new file mode 100644 index 000000000..4d11004e4 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftDBRecordUnitTest.java @@ -0,0 +1,155 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.util.DBUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +/** + * Unit Test class for the PostgresDBRecord + */ +@RunWith(MockitoJUnitRunner.class) +public class RedshiftDBRecordUnitTest { + + private static final int DEFAULT_PRECISION = 38; + + /** + * Validate the precision less Numbers handling against following use cases. + * 1. Ensure that the numeric type with [p,s] set as [38,4] detect as BigDecimal(38,4) in cdap. + * 2. Ensure that the numeric type without [p,s] detect as String type in cdap. + * + * @throws Exception + */ + @Test + public void validatePrecisionLessDecimalParsing() throws Exception { + Schema.Field field1 = Schema.Field.of("ID1", Schema.decimalOf(DEFAULT_PRECISION, 4)); + Schema.Field field2 = Schema.Field.of("ID2", Schema.of(Schema.Type.STRING)); + + Schema schema = Schema.recordOf( + "dbRecord", + field1, + field2 + ); + + ResultSetMetaData resultSetMetaData = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSetMetaData.getColumnType(Mockito.eq(1))).thenReturn(Types.NUMERIC); + Mockito.when(resultSetMetaData.getPrecision(Mockito.eq(1))).thenReturn(DEFAULT_PRECISION); + Mockito.when(resultSetMetaData.getColumnType(eq(2))).thenReturn(Types.NUMERIC); + when(resultSetMetaData.getPrecision(eq(2))).thenReturn(0); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSet.getBigDecimal(eq(1))).thenReturn(BigDecimal.valueOf(123.4568)); + when(resultSet.getString(eq(2))).thenReturn("123.4568"); + + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 1, Types.NUMERIC, DEFAULT_PRECISION, 4); + dbRecord.handleField(resultSet, builder, field2, 2, Types.NUMERIC, 0, -127); + + StructuredRecord record = builder.build(); + Assert.assertTrue(record.getDecimal("ID1") instanceof BigDecimal); + Assert.assertEquals(record.getDecimal("ID1"), BigDecimal.valueOf(123.4568)); + Assert.assertTrue(record.get("ID2") instanceof String); + Assert.assertEquals(record.get("ID2"), "123.4568"); + } + + @Test + public void validateTimestampType() throws SQLException { + OffsetDateTime offsetDateTime = OffsetDateTime.of(2023, 1, 1, 1, 0, 0, 0, ZoneOffset.UTC); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + when(metaData.getColumnTypeName(eq(0))).thenReturn("timestamp"); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getTimestamp(eq(0), eq(DBUtils.PURE_GREGORIAN_CALENDAR))) + .thenReturn(Timestamp.from(offsetDateTime.toInstant())); + + Schema.Field field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.DATETIME)); + Schema schema = Schema.recordOf( + "dbRecord", + field1 + ); + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + StructuredRecord record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getDateTime("field1")); + Assert.assertEquals(record.getDateTime("field1").toInstant(ZoneOffset.UTC), offsetDateTime.toInstant()); + + // Validate backward compatibility + + field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)); + schema = Schema.recordOf( + "dbRecord", + field1 + ); + builder = StructuredRecord.builder(schema); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getTimestamp("field1")); + Assert.assertEquals(record.getTimestamp("field1").toInstant(), offsetDateTime.toInstant()); + } + + @Test + public void validateTimestampTZType() throws SQLException { + OffsetDateTime offsetDateTime = OffsetDateTime.of(2023, 1, 1, 1, 0, 0, 0, ZoneOffset.UTC); + ResultSetMetaData metaData = Mockito.mock(ResultSetMetaData.class); + when(metaData.getColumnTypeName(eq(0))).thenReturn("timestamptz"); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + when(resultSet.getMetaData()).thenReturn(metaData); + when(resultSet.getObject(eq(0), eq(OffsetDateTime.class))).thenReturn(offsetDateTime); + + Schema.Field field1 = Schema.Field.of("field1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS)); + Schema schema = Schema.recordOf( + "dbRecord", + field1 + ); + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + + RedshiftDBRecord dbRecord = new RedshiftDBRecord(); + dbRecord.handleField(resultSet, builder, field1, 0, Types.TIMESTAMP, 0, 0); + StructuredRecord record = builder.build(); + Assert.assertNotNull(record); + Assert.assertNotNull(record.getTimestamp("field1", ZoneId.of("UTC"))); + Assert.assertEquals(record.getTimestamp("field1", ZoneId.of("UTC")).toInstant(), offsetDateTime.toInstant()); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java new file mode 100644 index 000000000..2d21c4478 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.plugin.db.connector.DBSpecificFailedConnectionTest; +import org.junit.Test; + +import java.io.IOException; + +public class RedshiftFailedConnectionTest extends DBSpecificFailedConnectionTest { + private static final String JDBC_DRIVER_CLASS_NAME = "com.amazon.redshift.Driver"; + + @Test + public void test() throws ClassNotFoundException, IOException { + + RedshiftConnector connector = new RedshiftConnector( + new RedshiftConnectorConfig("username", "password", "jdbc", "", "localhost", "db", 5432)); + + super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + + "jdbc:redshift://localhost:5432/db and arguments: " + + "{user=username}. Error: ConnectException: Connection refused " + + "(Connection refused)."); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java new file mode 100644 index 000000000..5df4fb300 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestBase.java @@ -0,0 +1,218 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.common.collect.Sets; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.plugin.PluginClass; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.batch.DatabasePluginTestBase; +import io.cdap.plugin.db.sink.ETLDBOutputFormat; +import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +/** + * Base test class for Redshift plugins. + */ +public abstract class RedshiftPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftPluginTestBase.class); + protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); + protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + protected static final long CURRENT_TS = System.currentTimeMillis(); + + protected static final String JDBC_DRIVER_NAME = "redshift"; + protected static final Map BASE_PROPS = new HashMap<>(); + + protected static String connectionUrl; + protected static int year; + protected static final int PRECISION = 10; + protected static final int SCALE = 6; + private static int startCount; + + @BeforeClass + public static void setupTest() throws Exception { + if (startCount++ > 0) { + return; + } + + getProperties(); + + Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(CURRENT_TS)); + year = calendar.get(Calendar.YEAR); + + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); + + addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + RedshiftSource.class, DBRecord.class, + ETLDBOutputFormat.class, DataDrivenETLDBInputFormat.class, DBRecord.class); + + // add mysql 3rd party plugin + PluginClass mysqlDriver = new PluginClass(ConnectionConfig.JDBC_PLUGIN_TYPE, JDBC_DRIVER_NAME, + "redshift driver class", Driver.class.getName(), + null, Collections.emptyMap()); + addPluginArtifact(NamespaceId.DEFAULT.artifact("redshift-jdbc-connector", "1.0.0"), + DATAPIPELINE_ARTIFACT_ID, + Sets.newHashSet(mysqlDriver), Driver.class); + + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + + connectionUrl = "jdbc:redshift://" + BASE_PROPS.get(ConnectionConfig.HOST) + ":" + + BASE_PROPS.get(ConnectionConfig.PORT) + "/" + BASE_PROPS.get(ConnectionConfig.DATABASE); + Connection conn = createConnection(); + createTestTables(conn); + prepareTestData(conn); + } + + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("redshift.clusterEndpoint")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("redshift.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("redshift.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("redshift.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("redshift.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + + protected static void createTestTables(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"dbActionTest\" (x int, day varchar(10))"); + // create a table that the action will truncate at the end of the run + stmt.execute("CREATE TABLE \"postActionTest\" (x int, day varchar(10))"); + + stmt.execute("CREATE TABLE my_table" + + "(" + + "\"ID\" INT NOT NULL," + + "\"NAME\" VARCHAR(40) NOT NULL," + + "\"SCORE\" REAL," + + "\"GRADUATED\" BOOLEAN," + + "\"NOT_IMPORTED\" VARCHAR(30)," + + "\"SMALLINT_COL\" SMALLINT," + + "\"BIG\" BIGINT," + + "\"NUMERIC_COL\" NUMERIC(" + PRECISION + "," + SCALE + ")," + + "\"DECIMAL_COL\" DECIMAL(" + PRECISION + "," + SCALE + ")," + + "\"DOUBLE_PREC_COL\" DOUBLE PRECISION," + + "\"DATE_COL\" DATE," + + "\"TIME_COL\" TIME," + + "\"TIMESTAMP_COL\" TIMESTAMP(3)," + + "\"TEXT_COL\" TEXT," + + "\"CHAR_COL\" CHAR(100)," + + "\"BYTEA_COL\" BYTEA" + + ")"); + stmt.execute("CREATE TABLE \"MY_DEST_TABLE\" AS " + + "SELECT * FROM my_table"); + stmt.execute("CREATE TABLE your_table AS " + + "SELECT * FROM my_table"); + } + } + + protected static void prepareTestData(Connection conn) throws SQLException { + try ( + Statement stmt = conn.createStatement(); + PreparedStatement pStmt1 = + conn.prepareStatement("INSERT INTO my_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)"); + PreparedStatement pStmt2 = + conn.prepareStatement("INSERT INTO your_table " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?," + + " ?, ?, ?, ?, ?, ?)")) { + + stmt.execute("insert into \"dbActionTest\" values (1, '1970-01-01')"); + stmt.execute("insert into \"postActionTest\" values (1, '1970-01-01')"); + + populateData(pStmt1, pStmt2); + } + } + + private static void populateData(PreparedStatement... stmts) throws SQLException { + // insert the same data into both tables: my_table and your_table + for (PreparedStatement pStmt : stmts) { + for (int i = 1; i <= 5; i++) { + String name = "user" + i; + pStmt.setInt(1, i); + pStmt.setString(2, name); + pStmt.setDouble(3, 123.45 + i); + pStmt.setBoolean(4, (i % 2 == 0)); + pStmt.setString(5, "random" + i); + pStmt.setShort(6, (short) i); + pStmt.setLong(7, (long) i); + pStmt.setBigDecimal(8, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setBigDecimal(9, new BigDecimal("123.45").add(new BigDecimal(i))); + pStmt.setDouble(10, 123.45 + i); + pStmt.setDate(11, new Date(CURRENT_TS)); + pStmt.setTime(12, new Time(CURRENT_TS)); + pStmt.setTimestamp(13, new Timestamp(CURRENT_TS)); + pStmt.setString(14, name); + pStmt.setString(15, "char" + i); + pStmt.setBytes(16, name.getBytes(Charsets.UTF_8)); + pStmt.executeUpdate(); + } + } + } + + public static Connection createConnection() { + try { + Class.forName(Driver.class.getCanonicalName()); + return DriverManager.getConnection(connectionUrl, BASE_PROPS.get(ConnectionConfig.USER), + BASE_PROPS.get(ConnectionConfig.PASSWORD)); + } catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @AfterClass + public static void tearDownDB() { + try (Connection conn = createConnection(); + Statement stmt = conn.createStatement()) { + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); + } + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java new file mode 100644 index 000000000..95ad0938b --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftPluginTestSuite.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.common.test.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * This is a test suite that runs all the tests for Redshift plugins. + */ +@RunWith(TestSuite.class) +@Suite.SuiteClasses({ + RedshiftSourceTestRun.class, +}) +public class RedshiftPluginTestSuite extends RedshiftPluginTestBase { +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java new file mode 100644 index 000000000..206b4ae9f --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSchemaReaderTest.java @@ -0,0 +1,131 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class RedshiftSchemaReaderTest { + + @Test + public void testGetSchema() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("timetz"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.TIMESTAMP); + + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.STRING), schema); + } + + @Test + public void testGetSchemaWithIntType() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("INT"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC); + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.INT), schema); + } + + @Test + public void testGetSchemaWithNumericTypeWithPrecision() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("STRING"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC); + Mockito.when(metadata.getPrecision(1)).thenReturn(0); + + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.STRING), schema); + } + + @Test + public void testGetSchemaWithOtherTypes() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("BIGINT"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.BIGINT); + Schema schema = schemaReader.getSchema(metadata, 1); + + Assert.assertEquals(Schema.of(Schema.Type.LONG), schema); + + Mockito.when(metadata.getColumnTypeName(2)).thenReturn("timestamp"); + Mockito.when(metadata.getColumnType(2)).thenReturn(Types.TIMESTAMP); + + schema = schemaReader.getSchema(metadata, 2); + + Assert.assertEquals(Schema.of(Schema.LogicalType.DATETIME), schema); + } + + @Test + public void testShouldIgnoreColumn() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader("sessionID"); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(metadata.getColumnName(1)).thenReturn("c_sessionID"); + Assert.assertTrue(schemaReader.shouldIgnoreColumn(metadata, 1)); + Mockito.when(metadata.getColumnName(2)).thenReturn("sqn_sessionID"); + Assert.assertTrue(schemaReader.shouldIgnoreColumn(metadata, 2)); + Mockito.when(metadata.getColumnName(3)).thenReturn("columnName"); + Assert.assertFalse(schemaReader.shouldIgnoreColumn(metadata, 3)); + } + + @Test + public void testGetSchemaFields() throws SQLException { + RedshiftSchemaReader schemaReader = new RedshiftSchemaReader(); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + // Mock two columns with different types + Mockito.when(metadata.getColumnCount()).thenReturn(2); + Mockito.when(metadata.getColumnTypeName(1)).thenReturn("INT"); + Mockito.when(metadata.getColumnType(1)).thenReturn(Types.NUMERIC); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + Mockito.when(metadata.getColumnTypeName(2)).thenReturn("BIGINT"); + Mockito.when(metadata.getColumnType(2)).thenReturn(Types.BIGINT); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.nullableOf(Schema.of(Schema.Type.LONG)))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java new file mode 100644 index 000000000..d09de8f0d --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTest.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class RedshiftSourceTest { + + @Test + public void testGetDBSpecificArguments() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + Map dbSpecificArguments = config.getDBSpecificArguments(); + Assert.assertEquals(0, dbSpecificArguments.size()); + } + + @Test + public void testGetFetchSize() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + Integer fetchSize = config.getFetchSize(); + Assert.assertEquals(1000, fetchSize.intValue()); + } + + @Test + public void testGetSchemaReader() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource source = new RedshiftSource(new RedshiftSource.RedshiftSourceConfig(false, connectorConfig)); + SchemaReader schemaReader = source.getSchemaReader(); + Assert.assertTrue(schemaReader instanceof RedshiftSchemaReader); + } + + @Test + public void testGetDBRecordType() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource source = new RedshiftSource(new RedshiftSource.RedshiftSourceConfig(false, connectorConfig)); + Class dbRecordType = source.getDBRecordType(); + Assert.assertEquals(RedshiftDBRecord.class, dbRecordType); + } + + @Test + public void testCreateConnectionString() { + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "localhost", "test", 5439); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + + RedshiftSource source = new RedshiftSource(config); + String connectionString = source.createConnectionString(); + Assert.assertEquals("jdbc:redshift://localhost:5439/test", connectionString); + } + + @Test + public void testGetLineageRecorder() { + BatchSourceContext context = Mockito.mock(BatchSourceContext.class); + RedshiftConnectorConfig connectorConfig = new RedshiftConnectorConfig("username", "password", + "jdbcPluginName", "connectionArguments", + "host", "database", 1101); + RedshiftSource.RedshiftSourceConfig config = new RedshiftSource.RedshiftSourceConfig(false, connectorConfig); + RedshiftSource source = new RedshiftSource(config); + + LineageRecorder lineageRecorder = source.getLineageRecorder(context); + Assert.assertNotNull(lineageRecorder); + } +} diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java new file mode 100644 index 000000000..1ac41bcd0 --- /dev/null +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftSourceTestRun.java @@ -0,0 +1,332 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.amazon.redshift; + +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.common.Bytes; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.DBConfig; +import io.cdap.plugin.db.source.AbstractDBSource; +import org.junit.Assert; +import org.junit.Test; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.nio.ByteBuffer; +import java.sql.Date; +import java.sql.Time; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test for Redshift source plugin. + */ +public class RedshiftSourceTestRun extends RedshiftPluginTestBase { + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBMacroSupport() throws Exception { + String importQuery = "SELECT * FROM my_table WHERE \"DATE_COL\" <= '${logicalStartTime(yyyy-MM-dd,1d)}' " + + "AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(ID),MAX(ID) from my_table"; + String splitBy = "ID"; + + ImmutableMap sourceProps = ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBTestSource").build(); + + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + sourceProps + ); + + ETLPlugin sinkConfig = MockSink.getPlugin("macroOutputTable"); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBMacro"); + runETLOnce(appManager, ImmutableMap.of("logical.start.time", String.valueOf(CURRENT_TS))); + + DataSetManager outputManager = getDataset("macroOutputTable"); + Assert.assertTrue(MockSink.readOutput(outputManager).isEmpty()); + } + + @Test + @SuppressWarnings("ConstantConditions") + public void testDBSource() throws Exception { + String importQuery = "SELECT \"ID\", \"NAME\", \"SCORE\", \"GRADUATED\", \"SMALLINT_COL\", \"BIG\", " + + "\"NUMERIC_COL\", \"CHAR_COL\", \"DECIMAL_COL\", \"BYTEA_COL\", \"DATE_COL\", \"TIME_COL\", \"TIMESTAMP_COL\", " + + "\"TEXT_COL\", \"DOUBLE_PREC_COL\" FROM my_table " + + "WHERE \"ID\" < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from my_table"; + String splitBy = "ID"; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBSourceTest") + .build(), + null + ); + + String outputDatasetName = "output-dbsourcetest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSource"); + runETLOnce(appManager); + + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals("user1", row1.get("TEXT_COL")); + Assert.assertEquals("user2", row2.get("TEXT_COL")); + Assert.assertEquals("char1", ((String) row1.get("CHAR_COL")).trim()); + Assert.assertEquals("char2", ((String) row2.get("CHAR_COL")).trim()); + Assert.assertEquals(124.45f, ((Float) row1.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(125.45f, ((Float) row2.get("SCORE")).doubleValue(), 0.000001); + Assert.assertEquals(false, row1.get("GRADUATED")); + Assert.assertEquals(true, row2.get("GRADUATED")); + Assert.assertNull(row1.get("NOT_IMPORTED")); + Assert.assertNull(row2.get("NOT_IMPORTED")); + + Assert.assertEquals(1, (int) row1.get("SMALLINT_COL")); + Assert.assertEquals(2, (int) row2.get("SMALLINT_COL")); + Assert.assertEquals(1, (long) row1.get("BIG")); + Assert.assertEquals(2, (long) row2.get("BIG")); + + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("125.45", new MathContext(PRECISION)).setScale(SCALE), + row2.getDecimal("NUMERIC_COL")); + Assert.assertEquals(new BigDecimal("124.45", new MathContext(PRECISION)).setScale(SCALE), + row1.getDecimal("DECIMAL_COL")); + + Assert.assertEquals(124.45, (double) row1.get("DOUBLE_PREC_COL"), 0.000001); + Assert.assertEquals(125.45, (double) row2.get("DOUBLE_PREC_COL"), 0.000001); + // Verify time columns + java.util.Date date = new java.util.Date(CURRENT_TS); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + LocalDate expectedDate = Date.valueOf(sdf.format(date)).toLocalDate(); + sdf = new SimpleDateFormat("H:mm:ss"); + LocalTime expectedTime = Time.valueOf(sdf.format(date)).toLocalTime(); + ZonedDateTime expectedTs = date.toInstant().atZone(ZoneId.ofOffset("UTC", ZoneOffset.UTC)); + Assert.assertEquals(expectedDate, row1.getDate("DATE_COL")); + Assert.assertEquals(expectedTime, row1.getTime("TIME_COL")); + Assert.assertEquals(expectedTs, row1.getTimestamp("TIMESTAMP_COL", ZoneId.ofOffset("UTC", ZoneOffset.UTC))); + + // verify binary columns + Assert.assertEquals("user1", Bytes.toString(((ByteBuffer) row1.get("BYTEA_COL")).array(), 0, 5)); + Assert.assertEquals("user2", Bytes.toString(((ByteBuffer) row2.get("BYTEA_COL")).array(), 0, 5)); + } + + @Test + public void testDbSourceMultipleTables() throws Exception { + String importQuery = "SELECT \"my_table\".\"ID\", \"your_table\".\"NAME\" FROM \"my_table\", \"your_table\"" + + "WHERE \"my_table\".\"ID\" < 3 and \"my_table\".\"ID\" = \"your_table\".\"ID\" and $CONDITIONS"; + String boundingQuery = "SELECT MIN(MIN(\"my_table\".\"ID\"), MIN(\"your_table\".\"ID\")), " + + "MAX(MAX(\"my_table\".\"ID\"), MAX(\"your_table\".\"ID\"))"; + String splitBy = "\"my_table\".\"ID\""; + ETLPlugin sourceConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBMultipleTest") + .build(), + null + ); + + String outputDatasetName = "output-multitabletest"; + ETLPlugin sinkConfig = MockSink.getPlugin(outputDatasetName); + + ApplicationManager appManager = deployETL(sourceConfig, sinkConfig, + DATAPIPELINE_ARTIFACT, "testDBSourceWithMultipleTables"); + runETLOnce(appManager); + + // records should be written + DataSetManager
outputManager = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(outputManager); + Assert.assertEquals(2, outputRecords.size()); + String userid = outputRecords.get(0).get("NAME"); + StructuredRecord row1 = "user1".equals(userid) ? outputRecords.get(0) : outputRecords.get(1); + StructuredRecord row2 = "user1".equals(userid) ? outputRecords.get(1) : outputRecords.get(0); + // Verify data + Assert.assertEquals("user1", row1.get("NAME")); + Assert.assertEquals("user2", row2.get("NAME")); + Assert.assertEquals(1, row1.get("ID").intValue()); + Assert.assertEquals(2, row2.get("ID").intValue()); + } + + @Test + public void testUserNamePasswordCombinations() throws Exception { + String importQuery = "SELECT * FROM \"my_table\" WHERE $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") from \"my_table\""; + String splitBy = "\"ID\""; + + ETLPlugin sinkConfig = MockSink.getPlugin("outputTable"); + + Map baseSourceProps = ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, BASE_PROPS.get(ConnectionConfig.DATABASE)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "UserPassDBTest") + .build(); + + ApplicationId appId = NamespaceId.DEFAULT.app("dbTest"); + + // null user name, null password. Should succeed. + // as source + ETLPlugin dbConfig = new ETLPlugin(RedshiftConstants.PLUGIN_NAME, BatchSource.PLUGIN_TYPE, + baseSourceProps, null); + ETLStage table = new ETLStage("uniqueTableSink", sinkConfig); + ETLStage database = new ETLStage("databaseSource", dbConfig); + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + AppRequest appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + + // null user name, non-null password. Should fail. + // as source + Map noUser = new HashMap<>(baseSourceProps); + noUser.put(DBConfig.PASSWORD, "password"); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, noUser, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + assertDeploymentFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT, + "Deploying DB Source with null username but non-null password should have failed."); + + // non-null username, non-null, but empty password. Should succeed. + // as source + Map emptyPassword = new HashMap<>(baseSourceProps); + emptyPassword.put(DBConfig.USER, "root"); + emptyPassword.put(DBConfig.PASSWORD, ""); + database = new ETLStage("databaseSource", new ETLPlugin(RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, emptyPassword, null)); + etlConfig = ETLBatchConfig.builder() + .addStage(database) + .addStage(table) + .addConnection(database.getName(), table.getName()) + .build(); + appRequest = new AppRequest<>(DATAPIPELINE_ARTIFACT, etlConfig); + deployApplication(appId, appRequest); + } + + @Test + public void testNonExistentDBTable() throws Exception { + // source + String importQuery = "SELECT \"ID\", \"NAME\" FROM \"dummy\" WHERE ID < 3 AND $CONDITIONS"; + String boundingQuery = "SELECT MIN(\"ID\"),MAX(\"ID\") FROM \"dummy\""; + String splitBy = "\"ID\""; + ETLPlugin sinkConfig = MockSink.getPlugin("table"); + ETLPlugin sourceBadNameConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .putAll(BASE_PROPS) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "DBNonExistentTest") + .build(), + null); + ETLStage sink = new ETLStage("sink", sinkConfig); + ETLStage sourceBadName = new ETLStage("sourceBadName", sourceBadNameConfig); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadName) + .addStage(sink) + .addConnection(sourceBadName.getName(), sink.getName()) + .build(); + ApplicationId appId = NamespaceId.DEFAULT.app("dbSourceNonExistingTest"); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + + // Bad connection + ETLPlugin sourceBadConnConfig = new ETLPlugin( + RedshiftConstants.PLUGIN_NAME, + BatchSource.PLUGIN_TYPE, + ImmutableMap.builder() + .put(ConnectionConfig.HOST, BASE_PROPS.get(ConnectionConfig.HOST)) + .put(ConnectionConfig.PORT, BASE_PROPS.get(ConnectionConfig.PORT)) + .put(ConnectionConfig.DATABASE, "dumDB") + .put(ConnectionConfig.USER, BASE_PROPS.get(ConnectionConfig.USER)) + .put(ConnectionConfig.PASSWORD, BASE_PROPS.get(ConnectionConfig.PASSWORD)) + .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) + .put(AbstractDBSource.DBSourceConfig.IMPORT_QUERY, importQuery) + .put(AbstractDBSource.DBSourceConfig.BOUNDING_QUERY, boundingQuery) + .put(AbstractDBSource.DBSourceConfig.SPLIT_BY, splitBy) + .put(Constants.Reference.REFERENCE_NAME, "RedshiftTest") + .build(), + null); + ETLStage sourceBadConn = new ETLStage("sourceBadConn", sourceBadConnConfig); + etlConfig = ETLBatchConfig.builder() + .addStage(sourceBadConn) + .addStage(sink) + .addConnection(sourceBadConn.getName(), sink.getName()) + .build(); + assertDeployAppFailure(appId, etlConfig, DATAPIPELINE_ARTIFACT); + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json new file mode 100644 index 000000000..943e2d24e --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -0,0 +1,240 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Connection", + "properties": [ + { + "widget-type": "toggle", + "label": "Use connection", + "name": "useConnection", + "widget-attributes": { + "on": { + "value": "true", + "label": "YES" + }, + "off": { + "value": "false", + "label": "NO" + }, + "default": "false" + } + }, + { + "widget-type": "connection-select", + "label": "Connection", + "name": "connection", + "widget-attributes": { + "connectionType": "Redshift" + } + }, + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "placeholder": "Redshift endpoint host name." + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + }, + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter" : "=", + "delimiter" : ";" + } + } + ] + }, + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName", + "widget-attributes": { + "placeholder": "Name used to identify this source for lineage. Typically, the name of the table/view." + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + }, + { + "widget-type": "connection-browser", + "widget-category": "plugin", + "widget-attributes": { + "connectionType": "Redshift", + "label": "Browse Database" + } + } + ] + }, + { + "label": "SQL Query", + "properties": [ + { + "widget-type": "textarea", + "label": "Import Query", + "name": "importQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "get-schema", + "widget-category": "plugin" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "textarea", + "label": "Bounding Query", + "name": "boundingQuery", + "widget-attributes": { + "rows": "4" + } + }, + { + "widget-type": "textbox", + "label": "Split-By Field Name", + "name": "splitBy" + }, + { + "widget-type": "textbox", + "label": "Number of Splits", + "name": "numSplits", + "widget-attributes": { + "default": "1" + } + }, + { + "widget-type": "number", + "label": "Fetch Size", + "name": "fetchSize", + "widget-attributes": { + "default": "1000", + "minimum": "0" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "boolean", + "int", + "long", + "float", + "double", + "bytes", + "string" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "showConnectionProperties ", + "condition": { + "expression": "useConnection == false" + }, + "show": [ + { + "type": "property", + "name": "jdbcPluginName" + }, + { + "type": "property", + "name": "instanceType" + }, + { + "type": "property", + "name": "host" + }, + { + "type": "property", + "name": "port" + }, + { + "type": "property", + "name": "user" + }, + { + "type": "property", + "name": "password" + }, + { + "type": "property", + "name": "database" + }, + { + "type": "property", + "name": "connectionArguments" + } + ] + }, + { + "name": "showConnectionId", + "condition": { + "expression": "useConnection == true" + }, + "show": [ + { + "type": "property", + "name": "connection" + } + ] + }, + ], + "jump-config": { + "datasets": [ + { + "ref-property-name": "referenceName" + } + ] + } +} diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json new file mode 100644 index 000000000..3a2af8e01 --- /dev/null +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -0,0 +1,75 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Redshift", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "plugin-list", + "label": "JDBC Driver name", + "name": "jdbcPluginName", + "widget-attributes": { + "plugin-type": "jdbc" + } + }, + { + "widget-type": "textbox", + "label": "Host", + "name": "host", + "widget-attributes": { + "default": "localhost" + } + }, + { + "widget-type": "number", + "label": "Port", + "name": "port", + "widget-attributes": { + "default": "5439" + } + }, + { + "widget-type": "textbox", + "label": "Database", + "name": "database" + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Username", + "name": "user" + }, + { + "widget-type": "password", + "label": "Password", + "name": "password" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "keyvalue", + "label": "Connection Arguments", + "name": "connectionArguments", + "widget-attributes": { + "showDelimiter": "false", + "key-placeholder": "Key", + "value-placeholder": "Value", + "kv-delimiter": "=", + "delimiter": ";" + } + } + ] + } + ], + "outputs": [] +} diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index 51ff6fb06..b9a542c3d 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Aurora DB MySQL plugin diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 28de0db21..0f31154ca 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Aurora DB PostgreSQL plugin diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index d05e592f5..8061b4ca0 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -20,17 +20,51 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT CloudSQL MySQL plugin cloudsql-mysql-plugin 4.0.0 + CloudSQL MySQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api + ${cdap.version} provided @@ -41,11 +75,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} io.cdap.plugin mysql-plugin - 1.11.0-SNAPSHOT + ${project.version} @@ -59,24 +94,26 @@ io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} test junit junit - - - io.cdap.cdap - cdap-api - provided + ${junit.version} + test org.mockito mockito-core + ${mockito.version} + test org.jetbrains diff --git a/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature b/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature index 717f9dcf5..5ab8b4727 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature +++ b/cloudsql-mysql-plugin/src/e2e-test/features/sink/CloudMySqlRunTimeMacro.feature @@ -142,7 +142,9 @@ Feature: CloudMySql Sink - Run time scenarios (macro) Then Enter runtime argument value "invalidTablename" for key "invalidTablename" Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidTableName | @@ -181,7 +183,9 @@ Feature: CloudMySql Sink - Run time scenarios (macro) Then Enter runtime argument value "invalidPassword" for key "Password" Then Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidCredentials | diff --git a/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature b/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature index b884fa7fa..242e53d5d 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature +++ b/cloudsql-mysql-plugin/src/e2e-test/features/source/CloudMySqlRunTime.feature @@ -224,7 +224,9 @@ Feature: CloudMySql Source - Run time scenarios Then Deploy the pipeline Then Run the Pipeline in Runtime Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 5ff3357f2..d4fc1de28 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,10 +12,12 @@ errorMessageNumberOfSplits=Split-By Field Name must be specified if Number of Sp errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. Specify the Bounding Query. errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'URLDecoder: Illegal hex characters in escape (%) pattern - For input string: "$^"' errorMessageInvalidTableName=Table 'Invalidtable' does not exist. Ensure table 'Invalidtable' is set correctly and -errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL PostgreSQL instance. +errorMessageConnectionName=Connection Name must be in the format :: to connect to a public CloudSQL MySQL instance. validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found -errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ - Table -errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Errors were encountered during validation. -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. +errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ + Table 'Table123' does not exist.. Please check the system logs for more details. +errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ + Exception while trying to validate schema of database table +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : java.io.IOException: You have an error in your SQL syntax; \ + check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java index 0608edb75..770dd9030 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLAction.java @@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlActionConfig.instanceType, - cloudsqlMysqlActionConfig.connectionName); + cloudsqlMysqlActionConfig.connectionName, + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index a5ee68787..b4b87c81b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.cloudsql.mysql; +import com.google.common.collect.Maps; import io.cdap.cdap.api.annotation.Category; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; @@ -75,7 +76,7 @@ public StructuredRecord transform(LongWritable longWritable, MysqlDBRecord mysql @Override protected SchemaReader getSchemaReader(String sessionID) { - return new MysqlSchemaReader(sessionID); + return new MysqlSchemaReader(sessionID, Maps.fromProperties(config.getConnectionArgumentsProperties())); } @Override diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java index 1e89d5a95..e763f6235 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java @@ -105,14 +105,14 @@ public String getConnectionString() { @Override public Properties getConnectionArgumentsProperties() { Properties properties = super.getConnectionArgumentsProperties(); - properties.put(JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS, "20000"); - properties.put(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); + properties.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT_MILLIS, "20000"); + properties.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT_MILLIS, "20000"); return properties; } @Override public boolean canConnect() { return super.canConnect() && !containsMacro(CloudSQLUtil.CONNECTION_NAME) && - !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); + !containsMacro(ConnectionConfig.PORT) && !containsMacro(ConnectionConfig.DATABASE); } } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java new file mode 100644 index 000000000..fe276a27a --- /dev/null +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLErrorDetailsProvider.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.mysql; + + +import io.cdap.plugin.mysql.MysqlErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for CloudSQL MySQL plugins. + */ +public class CloudSQLMySQLErrorDetailsProvider extends MysqlErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; + } +} diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 271012f7e..6cd1b0031 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -16,6 +16,7 @@ package io.cdap.plugin.cloudsql.mysql; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.annotation.Description; @@ -25,6 +26,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchSink; @@ -40,7 +42,11 @@ import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.StringJoiner; import javax.annotation.Nullable; /** Sink support for a CloudSQL MySQL database. */ @@ -52,6 +58,7 @@ public class CloudSQLMySQLSink extends AbstractDBSink { private final CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig; + private static final Character ESCAPE_CHAR = '`'; public CloudSQLMySQLSink(CloudSQLMySQLSinkConfig cloudsqlMysqlSinkConfig) { super(cloudsqlMysqlSinkConfig); @@ -67,7 +74,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSinkConfig.connection.getInstanceType(), - cloudsqlMysqlSinkConfig.connection.getConnectionName()); + cloudsqlMysqlSinkConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); @@ -78,6 +86,34 @@ protected DBRecord getDBRecord(StructuredRecord output) { return new MysqlDBRecord(output, columnTypes); } + @Override + protected void setColumnsInfo(List fields) { + List columnsList = new ArrayList<>(); + StringJoiner columnsJoiner = new StringJoiner(","); + for (Schema.Field field : fields) { + columnsList.add(field.getName()); + columnsJoiner.add(ESCAPE_CHAR + field.getName() + ESCAPE_CHAR); + } + + super.columns = Collections.unmodifiableList(columnsList); + super.dbColumns = columnsJoiner.toString(); + } + + @VisibleForTesting + String getDbColumns() { + return dbColumns; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLMySQLErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index b8b6fbf27..201360c67 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -31,9 +31,11 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; import io.cdap.plugin.mysql.MysqlDBRecord; +import io.cdap.plugin.mysql.MysqlSchemaReader; import io.cdap.plugin.util.CloudSQLUtil; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.mapreduce.lib.db.DBWritable; @@ -68,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlMysqlSourceConfig.connection.getInstanceType(), - cloudsqlMysqlSourceConfig.connection.getConnectionName()); + cloudsqlMysqlSourceConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_MYSQL); } super.configurePipeline(pipelineConfigurer); @@ -79,6 +82,11 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; + } + @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( @@ -120,6 +128,16 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, assetBuilder.build()); } + @Override + protected SchemaReader getSchemaReader() { + return new MysqlSchemaReader(null, cloudsqlMysqlSourceConfig.getConnectionArguments()); + } + + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLMySQLErrorDetailsProvider.class.getName(); + } + /** CloudSQL MySQL source config. */ public static class CloudSQLMySQLSourceConfig extends AbstractDBSpecificSourceConfig { diff --git a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java new file mode 100644 index 000000000..65a14502e --- /dev/null +++ b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.mysql; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class CloudSQLMySQLSinkTest { + @Test + public void testSetColumnsInfo() { + Schema outputSchema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); + CloudSQLMySQLSink cloudSQLMySQLSink = new CloudSQLMySQLSink(new CloudSQLMySQLSink.CloudSQLMySQLSinkConfig()); + Assert.assertNotNull(outputSchema.getFields()); + cloudSQLMySQLSink.setColumnsInfo(outputSchema.getFields()); + Assert.assertEquals("`id`,`name`,`insert`", cloudSQLMySQLSink.getDbColumns()); + } +} diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 2f974e854..f147961e6 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -20,17 +20,51 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT CloudSQL PostgreSQL plugin cloudsql-postgresql-plugin 4.0.0 + CloudSQL PostgreSQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api + ${cdap.version} provided @@ -41,6 +75,7 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} io.cdap.plugin @@ -63,24 +98,26 @@ io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} test junit junit - - - io.cdap.cdap - cdap-api - provided + ${junit.version} + test org.mockito mockito-core + ${mockito.version} + test org.jetbrains diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java index 1a3f8ad7b..5b13759f6 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLAction.java @@ -55,7 +55,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlActionConfig.instanceType, - cloudsqlPostgresqlActionConfig.connectionName); + cloudsqlPostgresqlActionConfig.connectionName, + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java new file mode 100644 index 000000000..cfb402468 --- /dev/null +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLErrorDetailsProvider.java @@ -0,0 +1,30 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.cloudsql.postgres; + +import io.cdap.plugin.postgres.PostgresErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for CloudSQL PostgreSQL plugin. + */ +public class CloudSQLPostgreSQLErrorDetailsProvider extends PostgresErrorDetailsProvider { + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } +} diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index f2c04e051..060b67f82 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -81,7 +81,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSinkConfig.connection.getInstanceType(), - cloudsqlPostgresqlSinkConfig.connection.getConnectionName()); + cloudsqlPostgresqlSinkConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); @@ -147,6 +148,16 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, assetBuilder.build()); } + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } + /** CloudSQL PostgreSQL sink config. */ public static class CloudSQLPostgreSQLSinkConfig extends AbstractDBSpecificSinkConfig { diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index 6d6ba29f8..db3f2d708 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -70,7 +70,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { CloudSQLUtil.checkConnectionName( failureCollector, cloudsqlPostgresqlSourceConfig.connection.getInstanceType(), - cloudsqlPostgresqlSourceConfig.connection.getConnectionName()); + cloudsqlPostgresqlSourceConfig.connection.getConnectionName(), + CloudSQLUtil.CLOUDSQL_POSTGRESQL); } super.configurePipeline(pipelineConfigurer); @@ -86,6 +87,16 @@ protected Class getDBRecordType() { return PostgresDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); + } + @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 0ecbfb445..ebd4b8bab 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -20,39 +20,76 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Database Commons database-commons 4.0.0 + Database Commons + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + provided io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} + test com.mockrunner @@ -63,6 +100,8 @@ org.mockito mockito-core + ${mockito.version} + test diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 588ed78b8..c5320e25e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,7 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index 31bb78938..b187c7670 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -17,10 +17,12 @@ package io.cdap.plugin.db; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.Lazy; import org.apache.hadoop.conf.Configurable; @@ -305,14 +307,14 @@ protected void updateOperation(PreparedStatement stmt) throws SQLException { * @throws SQLException */ protected void upsertOperation(PreparedStatement stmt) throws SQLException { - throw new UnsupportedOperationException(); + String errorMessage = "Upsert operation is not supported for this plugin."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new UnsupportedOperationException(errorMessage)); + } private boolean fillUpdateParams(List updatedKeyList, ColumnType columnType) { - if (operationName.equals(Operation.UPDATE) && updatedKeyList.contains(columnType.getName())) { - return true; - } - return false; + return operationName.equals(Operation.UPDATE) && updatedKeyList.contains(columnType.getName()); } private Schema getNonNullableSchema(Schema.Field field) { @@ -366,7 +368,9 @@ private void writeToDataOut(DataOutput out, Schema.Field field) throws IOExcepti out.write((byte[]) fieldValue); break; default: - throw new IOException(String.format("Unsupported datatype: %s with value: %s.", fieldType, fieldValue)); + String errorMessage = String.format("Unsupported datatype: %s with value: %s.", fieldType, fieldValue); + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.USER, false, new IOException(errorMessage)); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index a2be9cbf0..0eaac3148 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,12 +16,15 @@ package io.cdap.plugin.db.action; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; import java.sql.Driver; +import java.sql.SQLException; /** * Action that runs a db command. @@ -40,7 +43,18 @@ public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + try { + executeQuery.run(); + } catch (Exception e) { + if (e instanceof SQLException) { + DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); + throw dbe.getProgramFailureException((SQLException) e, null); + } + FailureCollector collector = context.getFailureCollector(); + collector.addFailure("Failed to execute query with message: " + e.getMessage(), null) + .withStacktrace(e.getStackTrace()); + collector.getOrThrowException(); + } } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 5c6b08031..8de0e4d70 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -20,8 +20,9 @@ import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.db.TransactionIsolationLevel; -import java.util.Collections; +import java.util.HashMap; import java.util.Map; import javax.annotation.Nullable; @@ -42,6 +43,12 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; + @Name(ConnectionConfig.TRANSACTION_ISOLATION_LEVEL) + @Description("The transaction isolation level for the database session.") + @Macro + @Nullable + protected String transactionIsolationLevel; + public String getHost() { return host; } @@ -55,4 +62,21 @@ public int getPort() { public boolean canConnect() { return super.canConnect() && !containsMacro(ConnectionConfig.HOST) && !containsMacro(ConnectionConfig.PORT); } + + @Override + public Map getAdditionalArguments() { + Map additonalArguments = new HashMap<>(); + if (getTransactionIsolationLevel() != null) { + additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); + } + return additonalArguments; + } + + public String getTransactionIsolationLevel() { + if (transactionIsolationLevel == null) { + return null; + } + return TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); + } } + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 2deac8ce4..0bb4bf123 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -25,6 +25,10 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -32,6 +36,7 @@ import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.batch.BatchRuntimeContext; import io.cdap.cdap.etl.api.batch.BatchSinkContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.cdap.etl.api.validation.InvalidStageException; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.ReferenceBatchSink; @@ -163,6 +168,26 @@ public void validateOperations(FailureCollector collector, T dbSinkConfig, @Null } } + /** + * Returns the ErrorDetailsProvider class name. + * Override this method to provide a custom ErrorDetailsProvider class name. + * + * @return ErrorDetailsProvider class name + */ + protected String getErrorDetailsProviderClassName() { + return null; + } + + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ + protected String getExternalDocumentationLink() { + return null; + } + @Override public void prepareRun(BatchSinkContext context) { String connectionString = dbSinkConfig.getConnectionString(); @@ -203,6 +228,7 @@ public void prepareRun(BatchSinkContext context) { configAccessor.setInitQueries(dbSinkConfig.getInitQueries()); configAccessor.getConfiguration().set(DBConfiguration.DRIVER_CLASS_PROPERTY, driverClass.getName()); configAccessor.getConfiguration().set(DBConfiguration.URL_PROPERTY, connectionString); + configAccessor.getConfiguration().set(ETLDBOutputFormat.STAGE_NAME, context.getStageName()); String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() : dbSinkConfig.getEscapedDbSchemaName() + "." + dbSinkConfig.getEscapedTableName(); configAccessor.getConfiguration().set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, fullyQualifiedTableName); @@ -227,7 +253,10 @@ public void prepareRun(BatchSinkContext context) { configuration.set(ETLDBOutputFormat.COMMIT_BATCH_SIZE, context.getArguments().get(ETLDBOutputFormat.COMMIT_BATCH_SIZE)); } - + // set error details provider + if (!Strings.isNullOrEmpty(getErrorDetailsProviderClassName())) { + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + } addOutputContext(context); } protected void addOutputContext(BatchSinkContext context) { @@ -283,8 +312,23 @@ private Schema inferSchema(Class driverClass) { inferredFields.addAll(getSchemaReader().getSchemaFields(rs)); } } catch (SQLException e) { - throw new InvalidStageException("Error while reading table metadata", e); - + // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error while reading table metadata." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessageWithDetails, errorMessage); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); } } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java index ad2b91ab1..ad196386c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/ETLDBOutputFormat.java @@ -25,6 +25,8 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.util.DBUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -43,6 +45,7 @@ import java.sql.Statement; import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; import static io.cdap.plugin.db.ConnectionConfigAccessor.OPERATION_NAME; import static io.cdap.plugin.db.ConnectionConfigAccessor.RELATION_TABLE_KEY; @@ -56,15 +59,92 @@ public class ETLDBOutputFormat extends DBOutputFormat { // Batch size before submitting a batch to the SQL engine. If set to 0, no batches will be submitted until commit. public static final String COMMIT_BATCH_SIZE = "io.cdap.plugin.db.output.commit.batch.size"; + public static final String STAGE_NAME = "io.cdap.plugin.db.output.stage_name"; public static final int DEFAULT_COMMIT_BATCH_SIZE = 1000; private static final Character ESCAPE_CHAR = '"'; + // Format for connection map's key will be "taskAttemptId_stageName" + private static final String CONNECTION_MAP_KEY_FORMAT = "%s_%s"; + + // CONNECTION_MAP will be used to store connections with "taskAttemptId_stageName" as key and + // connection object as value. Making it static to be accessed from multiple task attempts within same executor. + private static final Map CONNECTION_MAP = new ConcurrentHashMap<>(); private static final Logger LOG = LoggerFactory.getLogger(ETLDBOutputFormat.class); private Configuration conf; private Driver driver; private JDBCDriverShim driverShim; + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext context) + throws IOException, InterruptedException { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) throws IOException { + // do nothing + } + + @Override + public void setupTask(TaskAttemptContext taskContext) throws IOException { + // do nothing + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException { + return true; + } + + @Override + public void commitTask(TaskAttemptContext taskContext) throws IOException { + conf = context.getConfiguration(); + String stageName = conf.get(STAGE_NAME); + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + Connection connection; + if ((connection = CONNECTION_MAP.remove(connectionId)) != null) { + try { + connection.commit(); + } catch (SQLException e) { + try { + connection.rollback(); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + throw new IOException(e); + } finally { + try { + connection.close(); + LOG.debug("Connection Closed after committing the task with taskAttemptId {}", connectionId); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + } + } + + @Override + public void abortTask(TaskAttemptContext taskContext) throws IOException { + conf = context.getConfiguration(); + String stageName = conf.get(STAGE_NAME); + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + Connection connection; + if ((connection = CONNECTION_MAP.remove(connectionId)) != null) { + try { + connection.rollback(); + } catch (SQLException e) { + throw new IOException(e); + } finally { + try { + connection.close(); + LOG.debug("Connection Closed after rollback the task with taskAttemptId {}", connectionId); + } catch (SQLException ex) { + LOG.warn(StringUtils.stringifyException(ex)); + } + } + } + } + }; + } + @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException { conf = context.getConfiguration(); @@ -81,6 +161,11 @@ public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOE try { Connection connection = getConnection(conf); + String stageName = conf.get(STAGE_NAME); + // If using multiple sinks, task attemptID can be same in that case, appending stage in the end for uniqueness. + String connectionId = getConnectionMapKeyFormat(context.getTaskAttemptID().toString(), stageName); + CONNECTION_MAP.put(connectionId, connection); + LOG.debug("Connection Added to the map with connectionId : {}", connectionId); PreparedStatement statement = connection.prepareStatement(constructQueryOnOperation(tableName, fieldNames, operationName, listKeys)); return new DBRecordWriter(connection, statement) { @@ -98,23 +183,15 @@ public void close(TaskAttemptContext context) throws IOException { if (!emptyData) { getStatement().executeBatch(); } - getConnection().commit(); } catch (SQLException e) { - try { - getConnection().rollback(); - } catch (SQLException ex) { - LOG.warn(StringUtils.stringifyException(ex)); - } throw new IOException(e); } finally { try { getStatement().close(); - getConnection().close(); } catch (SQLException ex) { throw new IOException(ex); } } - try { DriverManager.deregisterDriver(driverShim); } catch (SQLException e) { @@ -298,4 +375,8 @@ public String constructUpdateQuery(String table, String[] fieldNames, String[] l return query.toString(); } } + + private String getConnectionMapKeyFormat(String taskAttemptId, String stageName) { + return String.format(CONNECTION_MAP_KEY_FORMAT, taskAttemptId, stageName); + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 987b5cc17..54d1e2ab6 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -25,6 +25,10 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -32,6 +36,7 @@ import io.cdap.cdap.etl.api.StageConfigurer; import io.cdap.cdap.etl.api.batch.BatchRuntimeContext; import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.cdap.etl.api.exception.ErrorDetailsProviderSpec; import io.cdap.cdap.internal.io.SchemaTypeAdapter; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.ReferenceBatchSource; @@ -119,8 +124,9 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { collector.addFailure("Unable to instantiate JDBC driver: " + e.getMessage(), null) .withStacktrace(e.getStackTrace()); } catch (SQLException e) { - collector.addFailure("SQL error while getting query schema: " + e.getMessage(), null) - .withStacktrace(e.getStackTrace()); + String details = String.format("SQL error while getting query schema: Error: %s, SQLState: %s, ErrorCode: %s", + e.getMessage(), e.getSQLState(), e.getErrorCode()); + collector.addFailure(details, null).withStacktrace(e.getStackTrace()); } catch (Exception e) { collector.addFailure(e.getMessage(), null).withStacktrace(e.getStackTrace()); } @@ -194,7 +200,22 @@ private Schema loadSchemaFromDB(Class driverClass) } catch (SQLException e) { // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath - throw new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode()); + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); } finally { driverCleanup.destroy(); } @@ -212,6 +233,16 @@ protected SchemaReader getSchemaReader() { return new CommonSchemaReader(); } + /** + * Returns the ErrorDetailsProvider class name. + * Override this method to provide a custom ErrorDetailsProvider class name. + * + * @return ErrorDetailsProvider class name + */ + protected String getErrorDetailsProviderClassName() { + return null; + } + private DriverCleanup loadPluginClassAndGetDriver(Class driverClass) throws IllegalAccessException, InstantiationException, SQLException { @@ -268,6 +299,10 @@ public void prepareRun(BatchSourceContext context) throws Exception { lineageRecorder.recordRead("Read", "Read from database plugin", schema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList())); } + // set error details provider + if (!Strings.isNullOrEmpty(getErrorDetailsProviderClassName())) { + context.setErrorDetailsProvider(new ErrorDetailsProviderSpec(getErrorDetailsProviderClassName())); + } context.setInput(Input.of(sourceConfig.getReferenceName(), new SourceInputFormatProvider( DataDrivenETLDBInputFormat.class, connectionConfigAccessor.getConfiguration()))); } @@ -341,6 +376,16 @@ protected Class getDBRecordType() { return DBRecord.class; } + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ + protected String getExternalDocumentationLink() { + return null; + } + @Override public void initialize(BatchRuntimeContext context) throws Exception { super.initialize(context); @@ -484,7 +529,7 @@ public void validateSchema(Schema actualSchema, FailureCollector collector) { } @VisibleForTesting - static void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { + void validateSchema(Schema actualSchema, Schema configSchema, FailureCollector collector) { if (configSchema == null) { collector.addFailure("Schema should not be null or empty.", null) .withConfigProperty(SCHEMA); @@ -505,14 +550,20 @@ static void validateSchema(Schema actualSchema, Schema configSchema, FailureColl Schema expectedFieldSchema = field.getSchema().isNullable() ? field.getSchema().getNonNullable() : field.getSchema(); - if (actualFieldSchema.getType() != expectedFieldSchema.getType() || - actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { - collector.addFailure( - String.format("Schema field '%s' has type '%s but found '%s'.", - field.getName(), expectedFieldSchema.getDisplayName(), - actualFieldSchema.getDisplayName()), null) - .withOutputSchemaField(field.getName()); - } + validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } + } + + protected void validateField(FailureCollector collector, Schema.Field field, Schema actualFieldSchema, + Schema expectedFieldSchema) { + if (actualFieldSchema.getType() != expectedFieldSchema.getType() || + actualFieldSchema.getLogicalType() != expectedFieldSchema.getLogicalType()) { + collector.addFailure( + String.format("Schema field '%s' is expected to have type '%s but found '%s'.", field.getName(), + expectedFieldSchema.getDisplayName(), actualFieldSchema.getDisplayName()), + String.format("Change the data type of field %s to %s.", field.getName(), + actualFieldSchema.getDisplayName())) + .withOutputSchemaField(field.getName()); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java index 11595ac06..f704f2ad5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java @@ -31,6 +31,9 @@ public class CloudSQLUtil { public static final String INSTANCE_TYPE = "instanceType"; public static final String PUBLIC_INSTANCE = "public"; public static final String PRIVATE_INSTANCE = "private"; + public static final String CLOUDSQL_POSTGRESQL = "CloudSQL PostgreSQL"; + public static final String CLOUDSQL_MYSQL = "CloudSQL MySQL"; + /** * Utility method to check the Connection Name format of a CloudSQL instance. @@ -38,9 +41,10 @@ public class CloudSQLUtil { * @param failureCollector {@link FailureCollector} for the pipeline * @param instanceType CloudSQL instance type * @param connectionName Connection Name for the CloudSQL instance + * @param databaseType Type of CloudSQL instance- CloudSQL PostgreSQL, CLoudSQL MySQL */ public static void checkConnectionName( - FailureCollector failureCollector, String instanceType, String connectionName) { + FailureCollector failureCollector, String instanceType, String connectionName, String databaseType) { if (PUBLIC_INSTANCE.equalsIgnoreCase(instanceType)) { Pattern connectionNamePattern = @@ -50,16 +54,16 @@ public static void checkConnectionName( if (!matcher.matches()) { failureCollector .addFailure( - "Connection Name must be in the format :: to connect to " - + "a public CloudSQL PostgreSQL instance.", null) + String.format("Connection Name must be in the format :: to connect to " + + "a public %s instance.", databaseType), null) .withConfigProperty(CONNECTION_NAME); } } else { if (!InetAddresses.isInetAddress(connectionName)) { failureCollector .addFailure( - "Enter the internal IP address of the Compute Engine VM cloudsql proxy " - + "is running on, to connect to a private CloudSQL PostgreSQL instance.", null) + String.format("Enter the internal IP address of the Compute Engine VM cloudsql proxy " + + "is running on, to connect to a private %s instance.", databaseType), null) .withConfigProperty(CONNECTION_NAME); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 584c7bb3f..b125a7214 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -60,6 +60,16 @@ public final class DBUtils { private static final Logger LOG = LoggerFactory.getLogger(DBUtils.class); public static final Calendar PURE_GREGORIAN_CALENDAR = createPureGregorianCalender(); + public static final String MYSQL_SUPPORTED_DOC_URL = "https://dev.mysql.com/doc/mysql-errors/9.0/en/"; + public static final String MARIADB_SUPPORTED_DOC_URL = "https://mariadb.com/kb/en/mariadb-error-codes/"; + public static final String MSSQL_SUPPORTED_DOC_URL = + "https://docs.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors"; + public static final String CLOUDSQLMYSQL_SUPPORTED_DOC_URL = "https://cloud.google.com/sql/docs/mysql/error-messages"; + public static final String POSTGRES_SUPPORTED_DOC_URL = + "https://www.postgresql.org/docs/current/errcodes-appendix.html"; + public static final String ORACLE_SUPPORTED_DOC_URL = "https://docs.oracle.com/en/error-help/db/ora-index.html"; + public static final String CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL = + "https://cloud.google.com/sql/docs/postgres/error-messages"; // Java by default uses October 15, 1582 as a Gregorian cut over date. // Any timestamp created with time less than this cut over date is treated as Julian date. diff --git a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java index 0f5a3ca4a..cbe1361d0 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/CommonSchemaReaderTest.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.exception.ProgramFailureException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -162,49 +163,49 @@ public void testGetSchemaThrowsExceptionOnNumericWithZeroPrecision() throws SQLE reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnArray() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.ARRAY); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnDatalink() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.DATALINK); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnDistinct() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.DISTINCT); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnJavaObject() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.JAVA_OBJECT); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnOther() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.OTHER); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnRef() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.REF); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnSQLXML() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.SQLXML); reader.getSchema(metadata, 1); } - @Test(expected = SQLException.class) + @Test(expected = ProgramFailureException.class) public void testGetSchemaThrowsExceptionOnStruct() throws SQLException { when(metadata.getColumnType(eq(1))).thenReturn(Types.STRUCT); reader.getSchema(metadata, 1); diff --git a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java index 3dc7a2d1c..a8be38b46 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/source/AbstractDBSourceTest.java @@ -43,11 +43,17 @@ public class AbstractDBSourceTest { Schema.Field.of("double_column", Schema.nullableOf(Schema.of(Schema.Type.DOUBLE))), Schema.Field.of("boolean_column", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))) ); + private static final AbstractDBSource.DBSourceConfig TEST_CONFIG = new AbstractDBSource.DBSourceConfig() { + @Override + public String getConnectionString() { + return ""; + } + }; @Test public void testValidateSourceSchemaCorrectSchema() { MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(SCHEMA, SCHEMA, collector); + TEST_CONFIG.validateSchema(SCHEMA, SCHEMA, collector); Assert.assertEquals(0, collector.getValidationFailures().size()); } @@ -65,7 +71,7 @@ public void testValidateSourceSchemaMismatchFields() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } @@ -84,7 +90,7 @@ public void testValidateSourceSchemaInvalidFieldType() { ); MockFailureCollector collector = new MockFailureCollector(MOCK_STAGE); - AbstractDBSource.DBSourceConfig.validateSchema(actualSchema, SCHEMA, collector); + TEST_CONFIG.validateSchema(actualSchema, SCHEMA, collector); assertPropertyValidationFailed(collector, "boolean_column"); } diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index a43bcb92e..868269a8a 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT IBM DB2 plugin diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index dbcd46d47..39cb543d1 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Generic database plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index 8a8dcd1c4..912528d4a 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Generic database argument setter plugin diff --git a/mariadb-plugin/docs/Mariadb-batchsink.md b/mariadb-plugin/docs/Mariadb-batchsink.md index 11176c0db..e4541fe67 100644 --- a/mariadb-plugin/docs/Mariadb-batchsink.md +++ b/mariadb-plugin/docs/Mariadb-batchsink.md @@ -60,41 +60,39 @@ connections. Data Types Mapping ---------- - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------- diff --git a/mariadb-plugin/docs/Mariadb-batchsource.md b/mariadb-plugin/docs/Mariadb-batchsource.md index 2b1fe3944..713af2ee8 100644 --- a/mariadb-plugin/docs/Mariadb-batchsource.md +++ b/mariadb-plugin/docs/Mariadb-batchsource.md @@ -78,43 +78,39 @@ with the tradeoff of higher memory usage. Data Types Mapping ---------- - - +--------------------------------+-----------------------+------------------------------------+ - | MariaDB Data Type | CDAP Schema Data Type | Comment | - +--------------------------------+-----------------------+------------------------------------+ - | TINYINT | int | | - | BOOLEAN, BOOL | boolean | | - | SMALLINT | int | | - | MEDIUMINT | int | | - | INT, INTEGER | int | | - | BIGINT | long | | - | DECIMAL, DEC, NUMERIC, FIXED | decimal | | - | FLOAT | float | | - | DOUBLE, DOUBLE PRECISION, REAL | decimal | | - | BIT | boolean | | - | CHAR | string | | - | VARCHAR | string | | - | BINARY | bytes | | - | CHAR BYTE | bytes | | - | VARBINARY | bytes | | - | TINYBLOB | bytes | | - | BLOB | bytes | | - | MEDIUMBLOB | bytes | | - | LONGBLOB | bytes | | - | TINYTEXT | string | | - | TEXT | string | | - | MEDIUMTEXT | string | | - | LONGTEXT | string | | - | JSON | string | In MariaDB it is alias to LONGTEXT | - | ENUM | string | Mapping to String by default | - | SET | string | | - | DATE | date | | - | TIME | time_micros | | - | DATETIME | timestamp_micros | | - | TIMESTAMP | timestamp_micros | | - | YEAR | date | | - +--------------------------------+-----------------------+------------------------------------+ - + | MariaDB Data Type | CDAP Schema Data Type | Comment | + |--------------------------------|-----------------------|---------------------------------------------------------| + | TINYINT | int | | + | BOOLEAN, BOOL | boolean | | + | SMALLINT | int | | + | MEDIUMINT | int | | + | INT, INTEGER | int | | + | BIGINT | long | | + | DECIMAL, DEC, NUMERIC, FIXED | decimal | | + | FLOAT | float | | + | DOUBLE, DOUBLE PRECISION, REAL | decimal | | + | BIT | boolean | | + | CHAR | string | | + | VARCHAR | string | | + | BINARY | bytes | | + | CHAR BYTE | bytes | | + | VARBINARY | bytes | | + | TINYBLOB | bytes | | + | BLOB | bytes | | + | MEDIUMBLOB | bytes | | + | LONGBLOB | bytes | | + | TINYTEXT | string | | + | TEXT | string | | + | MEDIUMTEXT | string | | + | LONGTEXT | string | | + | JSON | string | In MariaDB it is alias to LONGTEXT | + | ENUM | string | Mapping to String by default | + | SET | string | | + | DATE | date | | + | TIME | time_micros | | + | DATETIME | timestamp_micros | | + | TIMESTAMP | timestamp_micros | | + | YEAR | int | Users can manually set output schema to map it to Date. | Example ------ diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 0e9a09e02..3f7b9a58b 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Maria DB plugin @@ -83,6 +83,11 @@ RELEASE compile + + io.cdap.plugin + mysql-plugin + ${project.version} + diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java new file mode 100644 index 000000000..94498c787 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbDBRecord.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.mysql.MysqlDBRecord; +import java.util.List; + +/** + * Writable class for MariaDB Source/Sink. + */ +public class MariadbDBRecord extends MysqlDBRecord { + + /** + * Used in map-reduce. Do not remove. + */ + @SuppressWarnings("unused") + public MariadbDBRecord() { + // Required by Hadoop DBRecordReader to create an instance + } + + public MariadbDBRecord(StructuredRecord record, List columnTypes) { + super(record, columnTypes); + } +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java new file mode 100644 index 000000000..38405225d --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbErrorDetailsProvider.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for MariaDb plugins. + */ +public class MariadbErrorDetailsProvider extends MysqlErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java new file mode 100644 index 000000000..71ccb0d06 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbFieldsValidator.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + +import io.cdap.plugin.mysql.MysqlFieldsValidator; + +/** + * Field validator for maraidb + */ +public class MariadbFieldsValidator extends MysqlFieldsValidator { +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java new file mode 100644 index 000000000..37ac12a93 --- /dev/null +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSchemaReader.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mariadb; + + +import io.cdap.plugin.mysql.MysqlSchemaReader; +import java.util.Map; + +/** + * Schema reader for mapping Maria DB type + */ +public class MariadbSchemaReader extends MysqlSchemaReader { + + public MariadbSchemaReader (String sessionID) { + super(sessionID); + } + + public MariadbSchemaReader (String sessionID, Map connectionArguments) { + super(sessionID, connectionArguments); + } + +} diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index ab20f3c5d..52a73344a 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -19,9 +19,15 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.plugin.db.DBRecord; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; import io.cdap.plugin.db.sink.AbstractDBSink; +import io.cdap.plugin.db.sink.FieldsValidator; +import io.cdap.plugin.mysql.MysqlFieldsValidator; +import io.cdap.plugin.util.DBUtils; import java.util.Map; import javax.annotation.Nullable; @@ -45,6 +51,32 @@ public MariadbSink(MariadbSinkConfig mariadbSinkConfig) { this.mariadbSinkConfig = mariadbSinkConfig; } + @Override + protected DBRecord getDBRecord(StructuredRecord output) { + return new MariadbDBRecord(output, columnTypes); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null); + } + + + @Override + protected String getErrorDetailsProviderClassName() { + return MariadbErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + + @Override + protected FieldsValidator getFieldsValidator() { + return new MariadbFieldsValidator(); + } + /** * MariaDB Sink Config. */ diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index d5ffcb290..28204100c 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -19,10 +19,19 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.Asset; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; +import io.cdap.plugin.util.DBUtils; +import org.apache.hadoop.mapreduce.lib.db.DBWritable; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -53,10 +62,46 @@ protected String createConnectionString() { mariadbSourceConfig.host, mariadbSourceConfig.port, mariadbSourceConfig.database); } + @Override + protected Class getDBRecordType() { + return MariadbDBRecord.class; + } + + @Override + protected LineageRecorder getLineageRecorder(BatchSourceContext context) { + String fqn = DBUtils.constructFQN("mariadb", + mariadbSourceConfig.host, + mariadbSourceConfig.port, + mariadbSourceConfig.database, + mariadbSourceConfig.getReferenceName()); + Asset asset = Asset.builder(mariadbSourceConfig.getReferenceName()).setFqn(fqn).build(); + return new LineageRecorder(context, asset); + } + + @Override + protected SchemaReader getSchemaReader() { + return new MariadbSchemaReader(null, mariadbSourceConfig.getConnectionArguments()); + } + + @Override + protected String getErrorDetailsProviderClassName() { + return MariadbErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MARIADB_SUPPORTED_DOC_URL; + } + /** * MaraiDB source mariadbSourceConfig. */ public static class MariadbSourceConfig extends DBSpecificSourceConfig { + private static final String JDBC_PROPERTY_CONNECT_TIMEOUT = "connectTimeout"; + private static final String JDBC_PROPERTY_SOCKET_TIMEOUT = "socketTimeout"; + private static final String JDBC_REWRITE_BATCHED_STATEMENTS = "rewriteBatchedStatements"; + + private static final String MARIADB_TINYINT1_IS_BIT = "tinyInt1isBit"; @Name(MariadbConstants.AUTO_RECONNECT) @Description("Should the driver try to re-establish stale and/or dead connections") @@ -116,5 +161,43 @@ public Map getDBSpecificArguments() { public List getInitQueries() { return MariadbUtil.composeDbInitQueries(useAnsiQuotes); } + + @Override + public Map getConnectionArguments() { + Map arguments = new HashMap<>(super.getConnectionArguments()); + // the unit below is millisecond + arguments.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + arguments.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + // MariaDB property to ensure that TINYINT(1) type data is not converted to MariaDB Bit/Boolean type in the + // ResultSet. + arguments.putIfAbsent(MARIADB_TINYINT1_IS_BIT, "false"); + return arguments; + } + + @Override + protected void validateField(FailureCollector collector, + Schema.Field field, + Schema actualFieldSchema, + Schema expectedFieldSchema) { + // Backward compatibility changes to support MySQL YEAR to Date type conversion + if (Schema.LogicalType.DATE.equals(expectedFieldSchema.getLogicalType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL MEDIUMINT UNSIGNED to Long type conversion + if (Schema.Type.LONG.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + + // Backward compatibility change to support MySQL TINYINT(1) to Bool type conversion + if (Schema.Type.BOOLEAN.equals(expectedFieldSchema.getType()) + && Schema.Type.INT.equals(actualFieldSchema.getType())) { + return; + } + super.validateField(collector, field, actualFieldSchema, expectedFieldSchema); + } } } diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index 5c50a857e..53e10ed78 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Memsql plugin diff --git a/mssql-plugin/docs/SQL Server-connector.md b/mssql-plugin/docs/SQL Server-connector.md index cb72161f5..6f0038715 100644 --- a/mssql-plugin/docs/SQL Server-connector.md +++ b/mssql-plugin/docs/SQL Server-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Authentication Type:** Indicates which authentication method will be used for the connection. Use 'SQL Login'. to connect to a SQL Server using username and password properties. Use 'Active Directory Password' to connect to an Azure SQL Database/Data Warehouse using an Azure AD principal name and password. diff --git a/mssql-plugin/docs/SqlServer-batchsink.md b/mssql-plugin/docs/SqlServer-batchsink.md index 5d10b4bb6..b4ca1cbc5 100644 --- a/mssql-plugin/docs/SqlServer-batchsink.md +++ b/mssql-plugin/docs/SqlServer-batchsink.md @@ -46,6 +46,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/docs/SqlServer-batchsource.md b/mssql-plugin/docs/SqlServer-batchsource.md index c8e30f77e..5c917621c 100644 --- a/mssql-plugin/docs/SqlServer-batchsource.md +++ b/mssql-plugin/docs/SqlServer-batchsource.md @@ -56,6 +56,14 @@ an Azure SQL Database/Data Warehouse using an Azure AD principal name and passwo **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in SQL Server, refer to the [SQL Server documentation](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver16) + **Instance Name:** SQL Server instance name to connect to. When it is not specified, a connection is made to the default instance. For the case where both the instanceName and port are specified, see the notes for port. If you specify a Virtual Network Name in the Server connection property, you cannot diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 45e2b9c03..768e5d4c6 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -20,17 +20,51 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Microsoft SQL Server plugin mssql-plugin 4.0.0 + Microsoft SQL Server plugin database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,18 +93,26 @@ io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} + test org.mockito mockito-core + ${mockito.version} + test com.microsoft.sqlserver @@ -76,11 +120,6 @@ 8.2.1.jre8 test - - io.cdap.cdap - cdap-api - provided - org.jetbrains annotations diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 00721f148..c752d6ec1 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -13,11 +13,11 @@ errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. errormessageBlankHost=Exception while trying to validate schema of database table -errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Errors were encountered during validation. \ - Table 'Table123@' does not exist.. Please check the system logs for more details. +errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'SQL Server2' encountered : io.cdap.cdap.etl.api.validation.ValidationException: \ + Errors were encountered during validation. Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ 'connection' is invalid: Failed to assign value -errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Incorrect syntax near the keyword 'table'.. \ - Please check the system logs for more details. -errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Plugin with id SQL \ - Server:source.jdbc.sqlserver does not exist in program phase-1 of application +errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : io.cdap.cdap.api.exception.ProgramFailureException: \ + Error occurred while trying to get schema from database.Error message: 'Incorrect syntax near the keyword 'table'.'. +errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ + Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java new file mode 100644 index 000000000..90d1ce7b7 --- /dev/null +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mssql; + +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for SQL Server plugins. + */ +public class SqlServerErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } +} diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 0fa8991c5..dc442d200 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -88,6 +88,16 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return SqlServerErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } + /** * MSSQL action configuration. */ @@ -167,6 +177,11 @@ public Map getDBSpecificArguments() { packetSize, queryTimeout); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public String getConnectionString() { return String.format(SqlServerConstants.SQL_SERVER_CONNECTION_STRING_FORMAT, diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 9603b24db..004532064 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -75,6 +75,11 @@ protected Class getDBRecordType() { return SqlServerSourceDBRecord.class; } + @Override + protected String getErrorDetailsProviderClassName() { + return SqlServerErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mssql", @@ -85,6 +90,11 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MSSQL_SUPPORTED_DOC_URL; + } + /** * MSSQL source config. */ @@ -188,6 +198,11 @@ public List getInitQueries() { return Collections.emptyList(); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index 171076295..c326cd81d 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -64,6 +64,20 @@ "widget-type": "password", "label": "Password", "name": "password" + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 260c66259..fb20cad9d 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -280,6 +294,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index dad5f4708..b3494e485 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -84,6 +84,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -316,6 +330,10 @@ { "type": "property", "name": "connectionArguments" + }, + { + "type": "property", + "name": "transactionIsolationLevel" } ] }, diff --git a/mysql-plugin/docs/MySQL-connector.md b/mysql-plugin/docs/MySQL-connector.md index fb5c1fbb8..f586084c1 100644 --- a/mysql-plugin/docs/MySQL-connector.md +++ b/mysql-plugin/docs/MySQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments will be passed to the JDBC driver, as connection arguments, for JDBC drivers that may need additional configurations. This is a semicolon-separated list of key-value pairs, where each pair is separated by a equals '=' and specifies diff --git a/mysql-plugin/docs/Mysql-batchsink.md b/mysql-plugin/docs/Mysql-batchsink.md index b28a28618..46a763f9d 100644 --- a/mysql-plugin/docs/Mysql-batchsink.md +++ b/mysql-plugin/docs/Mysql-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/docs/Mysql-batchsource.md b/mysql-plugin/docs/Mysql-batchsource.md index 010e08216..552bb5504 100644 --- a/mysql-plugin/docs/Mysql-batchsource.md +++ b/mysql-plugin/docs/Mysql-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the database connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- TRANSACTION_READ_UNCOMMITTED: Allows dirty reads (reading uncommitted changes from other transactions). Non-repeatable reads and phantom reads are possible. + +For more details on the Transaction Isolation Levels supported in MySQL, refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-transaction-isolation-levels.html) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index f691a15f2..7c7dd054a 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -20,17 +20,51 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Mysql plugin mysql-plugin 4.0.0 + Mysql database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,23 +93,26 @@ io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit - - - io.cdap.cdap - cdap-api - provided + ${junit.version} + test org.mockito mockito-core + ${mockito.version} + test mysql diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature index 839752ae9..2c7050d08 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature @@ -167,3 +167,91 @@ Feature: MySQL Sink - Design time validation scenarios Then Enter input plugin property: "referenceName" with value: "targetRef" Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "invalid.host.message" on the header + + @Mysql_Required + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | tableName | + + @Mysql_Required + Scenario: Verify the validation error message with missing jdbc plugin name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature index 833b6b946..504ba7b77 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTime.feature @@ -147,3 +147,57 @@ Feature: MySQL Sink - Run time scenarios Then Open and capture logs Then Verify the pipeline status is "Succeeded" Then Validate the values of records transferred to target table is equal to the values from source table + + @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @CONNECTION @Mysql_Required + Scenario: To verify data is getting transferred from BigQuery to Mysql successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "MySQL" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + And Select Mysql Connection + And Enter input plugin property: "name" with value: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sinkRef" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Click plugin property: "useCompression" + Then Click plugin property: "autoReconnect" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target MySQL table is equal to the values from source BigQuery table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature index 0c4a5995e..a4cc32d6e 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/RunTimeWithMacros.feature @@ -100,6 +100,7 @@ Feature: MySQL Sink - Run time scenarios (macro) Then Wait till pipeline is in running state Then Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs @BQ_SOURCE_TEST @MYSQL_TARGET_TABLE @Mysql_Required Scenario: Verify that the pipeline fails when user provides invalid Credentials for connection with Macros @@ -135,4 +136,58 @@ Feature: MySQL Sink - Run time scenarios (macro) And Enter runtime argument value "invalid.password" for key "password" And Run the Pipeline in Runtime with runtime arguments And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully with connection arguments, databasename and tablename macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + And Click on the Macro button of Property: "database" and set the value to: "databaseName" + And Click on the Macro button of Property: "tableName" and set the value to: "tableName" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "databaseName" for key "databaseName" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "targetTable" for key "tableName" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + And Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + Then Enter runtime argument value "databaseName" for key "databaseName" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "targetTable" for key "tableName" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature index ee317eac6..49cf4390e 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/DesignTimeValidation.feature @@ -141,3 +141,77 @@ Feature: MySQL Source - Design time validation scenarios | referenceName | | database | | importQuery | + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message with invalid reference test data + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "invalidRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Validate button + Then Verify that the Plugin Property: "referenceName" is displaying an in-line error message: "invalidreferenceName.error.message" + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message when fetch size is changed to zero + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "fetchSize" with value: "zeroValue" + Then Click on the Validate button + Then Verify that the Plugin Property: "fetchSize" is displaying an in-line error message: "errorMessageInvalidFetchSize" + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message when number of Split value is changed to zero + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "zeroValue" + Then Click on the Validate button + Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessageInvalidNumberOfSplits" + + @Mysql_Required + Scenario: To verify MySQL source plugin validation error message with blank bounding query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "invalidImportQuery" + Then Replace input plugin property: "splitBy" with value: "splitBy" + Then Replace input plugin property: "numSplits" with value: "numberOfSplits" + Then Click on the Validate button + Then Verify that the Plugin Property: "boundingQuery" is displaying an in-line error message: "boundingQuery.error.message" + Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "boundingQuery.error.message" + Then Verify that the Plugin Property: "importQuery" is displaying an in-line error message: "errorMessageInvalidImportQuery" diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature index 096de1f62..0ea426da0 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTime.feature @@ -142,7 +142,8 @@ Feature: MySQL Source - Run time scenarios Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs + Then Open and capture pipeline preview logs Then Verify the preview run status of pipeline in the logs is "failed" @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required @@ -187,4 +188,150 @@ Feature: MySQL Source - Run time scenarios Then Wait till pipeline is in running state Then Open and capture logs Then Verify the pipeline status is "Succeeded" - Then Validate the values of records transferred to target table is equal to the values from source table \ No newline at end of file + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when connection arguments are set to read boolean datatype + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsForBooleanDataType" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSourceSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Enter key value pairs for plugin property: "connectionArguments" with values from json: "connectionArgumentsForBooleanDataType" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST + Scenario: To verify data is getting transferred from Mysql to Mysql successfully with bounding query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Enter textarea plugin property: "boundingQuery" with value: "boundingQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @CONNECTION + Scenario: To verify data is getting transferred from Mysql to Mysql successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + And Select Mysql Connection + And Enter input plugin property: "name" with value: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Use new connection + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for MySQL sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature index 2ae314de8..4c6a77576 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsource/RunTimeWithMacros.feature @@ -201,7 +201,9 @@ Feature: MySQL Source - Run time scenarios (macro) And Enter runtime argument value "invalid.query" for key "importQuery" And Run the Pipeline in Runtime with runtime arguments And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST @Mysql_Required Scenario: Verify that pipeline fails when user provides invalid Credentials for connection with Macros @@ -241,4 +243,61 @@ Feature: MySQL Source - Run time scenarios (macro) And Enter runtime argument value "invalid.password" for key "Password" And Run the Pipeline in Runtime with runtime arguments And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs + + @MYSQL_SOURCE_TEST @MYSQL_TARGET_TEST + Scenario: To verify data is getting transferred from Mysql to Mysql successfully when connection arguments,import,bounding query are macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "MySQL" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "MySQL" from the plugins list as: "Sink" + Then Connect plugins: "MySQL" and "MySQL2" to establish connection + Then Navigate to the properties page of plugin: "MySQL" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSource" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Click on the Macro button of Property: "importQuery" and set the value in textarea: "mysqlImportQuery" + Then Replace input plugin property: "database" with value: "databaseName" + Then Click on the Macro button of Property: "boundingQuery" and set the value in textarea: "mysqlBoundingQuery" + Then Click on the Get Schema button + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "MySQL2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "MySQL2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "selectQuery" for key "mysqlImportQuery" + Then Enter runtime argument value "boundingQuery" for key "mysqlBoundingQuery" + And Run the preview of pipeline with runtime arguments + Then Verify the preview of pipeline is "success" + And Close the preview + And Deploy the pipeline + And Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "selectQuery" for key "mysqlImportQuery" + Then Enter runtime argument value "boundingQuery" for key "mysqlBoundingQuery" + And Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java index d80e4dfc4..9bcea8e02 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java @@ -138,7 +138,7 @@ public static void createSourceTable(String sourceTable) throws SQLException, Cl try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) { String createSourceTableQuery = "CREATE TABLE IF NOT EXISTS " + sourceTable + - "(id int, lastName varchar(255), PRIMARY KEY (id))"; + "(id int, lastName varchar(255), PRIMARY KEY (id), is_active BOOLEAN NOT NULL)"; statement.executeUpdate(createSourceTableQuery); // Truncate table to clean the data of last failure run. @@ -146,12 +146,12 @@ public static void createSourceTable(String sourceTable) throws SQLException, Cl statement.executeUpdate(truncateSourceTableQuery); // Insert dummy data. - statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" + - "VALUES (1, 'Simpson')"); - statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" + - "VALUES (2, 'McBeal')"); - statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" + - "VALUES (3, 'Flinstone')"); + statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" + + "VALUES (1, 'Simpson', true)"); + statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" + + "VALUES (2, 'McBeal', true)"); + statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName, is_active)" + + "VALUES (3, 'Flinstone', false)"); } } @@ -159,7 +159,7 @@ public static void createTargetTable(String targetTable) throws SQLException, Cl try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) { String createTargetTableQuery = "CREATE TABLE IF NOT EXISTS " + targetTable + - "(id int, lastName varchar(255), PRIMARY KEY (id))"; + "(id int, lastName varchar(255), PRIMARY KEY (id), is_active BOOLEAN NOT NULL)"; statement.executeUpdate(createTargetTableQuery); // Truncate table to clean the data of last failure run. String truncateTargetTableQuery = "TRUNCATE TABLE " + targetTable; diff --git a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 2a86269b5..01475c75e 100644 --- a/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -17,6 +17,8 @@ package io.cdap.plugin.common.stepsdesign; import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.pages.actions.CdfConnectionActions; +import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions; import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.MysqlClient; @@ -47,7 +49,10 @@ private static void setTableName() { String targetTableName = String.format("TargetTable_%s", randomString); PluginPropertyUtils.addPluginProp("sourceTable", sourceTableName); PluginPropertyUtils.addPluginProp("targetTable", targetTableName); - PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s", sourceTableName)); + PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s " + + "WHERE $CONDITIONS", sourceTableName)); + PluginPropertyUtils.addPluginProp("boundingQuery", String.format("select MIN(id),MAX(id)" + + " from %s", sourceTableName)); } @Before(order = 1) @@ -212,5 +217,19 @@ public static void setNewConnectionName() { PluginPropertyUtils.addPluginProp("connection.name", connectionName); BeforeActions.scenario.write("New Connection name: " + connectionName); } + + private static void deleteConnection(String connectionType, String connectionName) throws IOException { + CdfConnectionActions.openWranglerConnectionsPage(); + CdfConnectionActions.expandConnections(connectionType); + CdfConnectionActions.openConnectionActionMenu(connectionType, connectionName); + CdfConnectionActions.selectConnectionAction(connectionType, connectionName, "Delete"); + CdfPluginPropertiesActions.clickPluginPropertyButton("Delete"); + } + + @After(order = 1, value = "@CONNECTION") + public static void deleteBQConnection() throws IOException { + deleteConnection("MySQL", "connection.name"); + PluginPropertyUtils.removePluginProp("connection.name"); + } } diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 0c1ac0400..4a7188ff8 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -11,3 +11,8 @@ boundingQuery.error.message=Bounding Query must be specified if Number of Splits splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 invalid.sink.database.message=Exception while trying to validate schema blank.username.message=Username is required when password is given. +errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. +errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. +errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. +blank.connection.message=Exception while trying to validate schema of database table +blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. diff --git a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties index 215e4b830..5cf1095c0 100644 --- a/mysql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mysql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -44,10 +44,11 @@ invalid.tableName=table@123 sourceRef=source targetRef=target -outputSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"}] +outputSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"},{"key":"is_active","value":"int"}] invalid.query= select * invalid.password=mysqlroot1 invalid.database=test123 +invalidRef=%$%^_test #bq queries file path @@ -98,4 +99,9 @@ datatypesSchema=[{"key":"ID","value":"string"},{"key":"COL1","value":"long"},{"k {"key":"COL33","value":"string"}] bqOutputDatatypesSchema=[{"key":"id","value":"long"},{"key":"lastName","value":"string"}] - +connectionArguments=queryTimeout=50 +zeroValue=0 +invalidImportQuery=select +numberOfSplits=2 +connectionArgumentsForBooleanDataType=[{"key":"tinyInt1isBit","value":"true"}] +outputSourceSchema=[{"key":"id","value":"int"},{"key":"lastName","value":"string"},{"key":"is_active","value":"boolean"}] diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index 3dede5d49..e7e935135 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.mysql; +import com.google.common.collect.Maps; import io.cdap.cdap.api.annotation.Category; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; @@ -62,7 +63,7 @@ public boolean supportSchema() { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new MysqlSchemaReader(sessionID); + return new MysqlSchemaReader(sessionID, Maps.fromProperties(config.getConnectionArgumentsProperties())); } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java index 9b481e4fe..8c20798d3 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnectorConfig.java @@ -57,9 +57,9 @@ public int getDefaultPort() { public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); // the unit below is milli-second - prop.put(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); - prop.put(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); - prop.put(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); + prop.putIfAbsent(JDBC_PROPERTY_CONNECT_TIMEOUT, "20000"); + prop.putIfAbsent(JDBC_PROPERTY_SOCKET_TIMEOUT, "20000"); + prop.putIfAbsent(JDBC_REWRITE_BATCHED_STATEMENTS, "true"); // MySQL property to ensure that TINYINT(1) type data is not converted to MySQL Bit/Boolean type in the ResultSet. prop.putIfAbsent(MYSQL_TINYINT1_IS_BIT, "false"); return prop; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java index 39c0b8d08..a73e14e9f 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConstants.java @@ -16,12 +16,18 @@ package io.cdap.plugin.mysql; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; + /** * MySQL Constants. */ public final class MysqlConstants { private MysqlConstants() { - throw new AssertionError("Should not instantiate static utility class."); + String errorMessage = "Should not instantiate static utility class."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new AssertionError(errorMessage)); } public static final String PLUGIN_NAME = "Mysql"; @@ -39,6 +45,7 @@ private MysqlConstants() { public static final String TRUST_CERT_KEYSTORE_PASSWORD = "trustCertificateKeyStorePassword"; public static final String MYSQL_CONNECTION_STRING_FORMAT = "jdbc:mysql://%s:%s/%s"; public static final String USE_CURSOR_FETCH = "useCursorFetch"; + public static final String ZERO_DATE_TIME_BEHAVIOR = "zeroDateTimeBehavior"; /** * Query to set SQL_MODE system variable. diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java index 0560b10c3..94b711786 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlDBRecord.java @@ -93,4 +93,13 @@ protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, super.writeNonNullToDB(stmt, fieldSchema, fieldName, fieldIndex); } + + @Override + protected void insertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + Schema.Field field = record.getSchema().getField(columnType.getName(), true); + writeToDB(stmt, field, fieldIndex); + } + } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java new file mode 100644 index 000000000..ca9a2b928 --- /dev/null +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlErrorDetailsProvider.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for MySQL plugins. + */ +public class MysqlErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MYSQL_SUPPORTED_DOC_URL; + } + + @Override + protected ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { + // https://dev.mysql.com/doc/refman/9.0/en/error-message-elements.html#error-code-ranges + if (errorCode >= 1000 && errorCode <= 5999) { + return ErrorType.USER; + } else if (errorCode >= 10000 && errorCode <= 51999) { + // SYSTEM errors: Enterprise and user-defined custom error messages + return ErrorType.SYSTEM; + } else { + // UNKNOWN errors: Anything outside defined range + return ErrorType.UNKNOWN; + } + } +} diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java index a842ba568..50907c063 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSchemaReader.java @@ -16,12 +16,16 @@ package io.cdap.plugin.mysql; +import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.List; +import java.util.Map; /** * Schema reader for mapping Mysql DB type @@ -31,12 +35,42 @@ public class MysqlSchemaReader extends CommonSchemaReader { public static final String YEAR_TYPE_NAME = "YEAR"; public static final String MEDIUMINT_UNSIGNED_TYPE_NAME = "MEDIUMINT UNSIGNED"; private final String sessionID; + private boolean zeroDateTimeToNull; public MysqlSchemaReader(String sessionID) { super(); this.sessionID = sessionID; } + public MysqlSchemaReader(String sessionID, Map connectionArguments) { + super(); + this.sessionID = sessionID; + this.zeroDateTimeToNull = MysqlUtil.isZeroDateTimeToNull(connectionArguments); + } + + @Override + public List getSchemaFields(ResultSet resultSet) throws SQLException { + List schemaFields = Lists.newArrayList(); + ResultSetMetaData metadata = resultSet.getMetaData(); + // ResultSetMetadata columns are numbered starting with 1 + for (int i = 1; i <= metadata.getColumnCount(); i++) { + if (shouldIgnoreColumn(metadata, i)) { + continue; + } + + String columnName = metadata.getColumnName(i); + Schema columnSchema = getSchema(metadata, i); + + if (ResultSetMetaData.columnNullable == metadata.isNullable(i) + || (zeroDateTimeToNull && MysqlUtil.isDateTimeLikeType(metadata.getColumnType(i)))) { + columnSchema = Schema.nullableOf(columnSchema); + } + Schema.Field field = Schema.Field.of(columnName, columnSchema); + schemaFields.add(field); + } + return schemaFields; + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { return metadata.getColumnName(index).equals("c_" + sessionID) || diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index c839cb12b..0a9257a0a 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -16,6 +16,7 @@ package io.cdap.plugin.mysql; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -24,6 +25,7 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSink; import io.cdap.cdap.etl.api.batch.BatchSinkContext; @@ -39,9 +41,12 @@ import io.cdap.plugin.db.sink.FieldsValidator; import io.cdap.plugin.util.DBUtils; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.StringJoiner; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -54,6 +59,7 @@ public class MysqlSink extends AbstractDBSink { private final MysqlSinkConfig mysqlSinkConfig; + private static final Character ESCAPE_CHAR = '`'; public MysqlSink(MysqlSinkConfig mysqlSinkConfig) { super(mysqlSinkConfig); @@ -85,6 +91,34 @@ protected SchemaReader getSchemaReader() { return new MysqlSchemaReader(null); } + @Override + protected void setColumnsInfo(List fields) { + List columnsList = new ArrayList<>(); + StringJoiner columnsJoiner = new StringJoiner(","); + for (Schema.Field field : fields) { + columnsList.add(field.getName()); + columnsJoiner.add(ESCAPE_CHAR + field.getName() + ESCAPE_CHAR); + } + + super.columns = Collections.unmodifiableList(columnsList); + super.dbColumns = columnsJoiner.toString(); + } + + @VisibleForTesting + String getDbColumns() { + return dbColumns; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return MysqlErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MYSQL_SUPPORTED_DOC_URL; + } + /** * MySQL action configuration. */ @@ -160,6 +194,11 @@ public Map getDBSpecificArguments() { trustCertificateKeyStorePassword, false); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public MysqlConnectorConfig getConnection() { return connection; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 71f113436..38642468c 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -69,6 +69,11 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.MYSQL_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mysql", @@ -81,7 +86,12 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { @Override protected SchemaReader getSchemaReader() { - return new MysqlSchemaReader(null); + return new MysqlSchemaReader(null, mysqlSourceConfig.getConnectionArguments()); + } + + @Override + protected String getErrorDetailsProviderClassName() { + return MysqlErrorDetailsProvider.class.getName(); } /** @@ -187,6 +197,11 @@ public MysqlConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java index c1c770c06..3c3cbedcf 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlUtil.java @@ -17,7 +17,11 @@ package io.cdap.plugin.mysql; import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; +import java.sql.Types; import java.util.Map; /** @@ -25,7 +29,9 @@ */ public final class MysqlUtil { private MysqlUtil() { - throw new AssertionError("Should not instantiate static utility class."); + String errorMessage = "Should not instantiate static utility class."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new AssertionError(errorMessage)); } /** @@ -91,4 +97,20 @@ public static Map composeDbSpecificArgumentsMap(Boolean autoReco public static String getConnectionString(String host, Integer port, String database) { return String.format(MysqlConstants.MYSQL_CONNECTION_STRING_FORMAT, host, port, database); } + + public static boolean isDateTimeLikeType(int columnType) { + int[] dateTimeLikeTypes = new int[]{Types.TIMESTAMP, Types.TIMESTAMP_WITH_TIMEZONE, Types.DATE}; + + for (int dttType : dateTimeLikeTypes) { + if (dttType == columnType) { + return true; + } + } + return false; + } + + public static boolean isZeroDateTimeToNull(Map connectionArguments) { + String argValue = connectionArguments.getOrDefault(MysqlConstants.ZERO_DATE_TIME_BEHAVIOR, ""); + return argValue.equals("CONVERT_TO_NULL") || argValue.equals("convertToNull"); + } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java index a1be6a754..5c4f35828 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java @@ -31,10 +31,26 @@ public void test() throws ClassNotFoundException, IOException { new MysqlConnectorConfig("localhost", 3306, "username", "password", "jdbc", "")); super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + - "jdbc:mysql://localhost:3306 and arguments: {user=username, " + - "rewriteBatchedStatements=true, " + - "connectTimeout=20000, tinyInt1isBit=false, " + - "socketTimeout=20000}. Error: " + - "ConnectException: Connection refused (Connection refused)."); + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + + "connectTimeout=20000, tinyInt1isBit=false, " + + "socketTimeout=20000}. Error: " + + "ConnectException: Connection refused (Connection refused)."); } + + @Test + public void testWithUpdatedConnectionArguments() throws ClassNotFoundException, IOException { + + MysqlConnector connector = new MysqlConnector( + new MysqlConnectorConfig("localhost", 3306, "username", "password", "jdbc", + "connectTimeout=30000;socketTimeout=30000")); + + super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + + "jdbc:mysql://localhost:3306 and arguments: {user=username, " + + "rewriteBatchedStatements=true, " + + "connectTimeout=30000, tinyInt1isBit=false, " + + "socketTimeout=30000}. Error: " + + "ConnectException: Connection refused (Connection refused)."); + } + } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java index 28582bc3b..fa7029c8f 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSchemaReaderUnitTest.java @@ -21,9 +21,13 @@ import org.junit.Test; import org.mockito.Mockito; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class MysqlSchemaReaderUnitTest { @@ -37,4 +41,33 @@ public void validateYearTypeToStringTypeConversion() throws SQLException { Schema schema = schemaReader.getSchema(metadata, 1); Assert.assertTrue(Schema.of(Schema.Type.INT).equals(schema)); } + + @Test + public void validateZeroDateTimeBehavior() throws SQLException { + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(1); + Mockito.when(metadata.getColumnName(Mockito.eq(1))).thenReturn("some_date"); + + Mockito.when(metadata.getColumnType(Mockito.eq(1))).thenReturn(Types.DATE); + Mockito.when(metadata.getColumnTypeName(Mockito.eq(1))).thenReturn(MysqlSchemaReader.YEAR_TYPE_NAME); + + // non-nullable column + Mockito.when(metadata.isNullable(Mockito.eq(1))).thenReturn(0); + + // test that non-nullable date remains non-nullable when no conn arg is present + MysqlSchemaReader schemaReader = new MysqlSchemaReader(null); + List schemaFields = schemaReader.getSchemaFields(resultSet); + Assert.assertFalse(schemaFields.get(0).getSchema().isNullable()); + + // test that it converts non-nullable date column to nullable when zeroDateTimeBehavior is convert to null + Map connectionArguments = new HashMap<>(); + connectionArguments.put("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + + schemaReader = new MysqlSchemaReader(null, connectionArguments); + schemaFields = schemaReader.getSchemaFields(resultSet); + Assert.assertTrue(schemaFields.get(0).getSchema().isNullable()); + } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java new file mode 100644 index 000000000..1dd4e809e --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; + +public class MysqlSinkTest { + @Test + public void testSetColumnsInfo() { + Schema outputSchema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); + MysqlSink mySQLSink = new MysqlSink(new MysqlSink.MysqlSinkConfig()); + Assert.assertNotNull(outputSchema.getFields()); + mySQLSink.setColumnsInfo(outputSchema.getFields()); + Assert.assertEquals("`id`,`name`,`insert`", mySQLSink.getDbColumns()); + } +} diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java new file mode 100644 index 000000000..9481068f1 --- /dev/null +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlUtilUnitTest.java @@ -0,0 +1,62 @@ + +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.mysql; + +import org.junit.Test; + +import java.sql.Types; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MysqlUtilUnitTest { + + @Test + public void testIsZeroDateTimeToNull() { + Map connArgsMap = new HashMap<>(1); + + connArgsMap.put("zeroDateTimeBehavior", ""); + assertFalse(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "ROUND"); + assertFalse(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "CONVERT_TO_NULL"); + assertTrue(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + + connArgsMap.put("zeroDateTimeBehavior", "convertToNull"); + assertTrue(MysqlUtil.isZeroDateTimeToNull(connArgsMap)); + } + + @Test + public void testIsDateTimeLikeType() { + int dateType = Types.DATE; + int timestampType = Types.TIMESTAMP; + int timestampWithTimezoneType = Types.TIMESTAMP_WITH_TIMEZONE; + int timeType = Types.TIME; + int stringType = Types.VARCHAR; + + assertTrue(MysqlUtil.isDateTimeLikeType(dateType)); + assertTrue(MysqlUtil.isDateTimeLikeType(timestampType)); + assertTrue(MysqlUtil.isDateTimeLikeType(timestampWithTimezoneType)); + assertFalse(MysqlUtil.isDateTimeLikeType(timeType)); + assertFalse(MysqlUtil.isDateTimeLikeType(stringType)); + } +} diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 9064d1bf6..f60f5526f 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -30,6 +30,20 @@ "widget-attributes": { "default": "3306" } + }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } } ] }, diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index c525ead40..58596aae2 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -225,6 +239,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 9175bd5ed..506e837f7 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -65,6 +65,20 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_UNCOMMITTED", + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -277,6 +291,10 @@ "type": "property", "name": "password" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "host" diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 900e430fe..824a7d6ec 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Netezza plugin diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index e0ed7ff50..988cd424b 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -20,17 +20,51 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT Oracle plugin oracle-plugin 4.0.0 + Oracle database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -57,18 +93,25 @@ io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 + ${cdap.version} + test junit junit + ${junit.version} + test org.hsqldb hsqldb + ${hsql.version} test @@ -80,11 +123,8 @@ org.mockito mockito-core - - - io.cdap.cdap - cdap-api - provided + ${mockito.version} + test org.glassfish diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature index 936802e66..d9cb71e38 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature @@ -173,3 +173,140 @@ Feature: Oracle sink- Verify Oracle sink plugin design time validation scenarios Then Enter input plugin property: "referenceName" with value: "sourceRef" Then Click on the Validate button Then Verify that the Plugin is displaying an error message: "errorMessageInvalidHost" on the header + + @Oracle_Required + Scenario: Verify the validation error message on header with blank database value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.database.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.HostBlank.message" on the header + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario Outline: To verify Oracle sink plugin validation error message for update, upsert operation name and table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Validate button + Then Verify that the Plugin Property: "operationName" is displaying an in-line error message: "errorMessageUpdateUpsertOperationName" + Then Verify that the Plugin Property: "relationTableKey" is displaying an in-line error message: "errorMessageUpdateUpsertOperationName" + Examples: + | options | + | upsert | + | update | diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature index c2b56e8b7..e7cb104c5 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTime.feature @@ -117,3 +117,330 @@ Feature: Oracle - Verify data transfer from BigQuery source to Oracle sink Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @BQ_SOURCE_TEST_SMALL_CASE @ORACLE_TEST_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is coming in small case + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchemaSmallCase" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table with case + + + @BQ_SOURCE_TEST_DATE @ORACLE_DATE_TABLE + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully when schema is having date and timestamp fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesDateTimeSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @BQ_SOURCE_TEST @ORACLE_TEST_TABLE @CONNECTION + Scenario: To verify data is getting transferred from BigQuery source to Oracle sink successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Oracle" to establish connection + Then Navigate to the properties page of plugin: "BigQuery" + Then Replace input plugin property: "project" with value: "projectId" + Then Enter input plugin property: "datasetProject" with value: "projectId" + Then Enter input plugin property: "referenceName" with value: "BQReferenceName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "bqOutputDatatypesSchema" + Then Validate "BigQuery" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-Oracle" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Replace input plugin property: "database" with value: "databaseName" + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Oracle sink + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table with record counts of BigQuery table + Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @ORACLE_SOURCE_DATATYPES_TEST @ORACLE_TARGET_DATATYPES_TEST @Oracle_Required + Scenario Outline: To verify pipeline preview failed with invalid table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | invalidOracleTableKey | + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "Failed" + Examples: + | options | + | upsert | + | update | + + @ORACLE_SOURCE_TEST @ORACLE_UPDATE_TABLE @Oracle_Required + Scenario Outline: To verify data is getting transferred from Oracle to Oracle successfully using update operation with table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | oracleTableKey | + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on the Preview Data link on the Sink plugin node: "Oracle" + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + Examples: + | options | + | update | + + @ORACLE_SOURCE_TEST @ORACLE_UPSERT_TABLE @Oracle_Required + Scenario Outline: To verify data is getting transferred from Oracle to Oracle successfully using upsert operation with table key + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Select radio button plugin property: "operationName" with value: "" + Then Click on the Add Button of the property: "relationTableKey" with value: + | oracleTableKey | + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on the Preview Data link on the Sink plugin node: "Oracle" + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate the values of records transferred to target table is equal to the values from source table + Examples: + | options | + | upsert | diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature b/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature index 78130655f..74e5302c6 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleRunTimeMacro.feature @@ -88,3 +88,135 @@ Feature: Oracle - Verify data transfer to Oracle sink with macro arguments Then Close the pipeline logs Then Validate records transferred to target table with record counts of BigQuery table Then Validate the values of records transferred to target Oracle table is equal to the values from source BigQuery table + + @ORACLE_SOURCE_TEST @ORACLE_UPSERT_TABLE @Oracle_Required + Scenario: To verify data is getting transferred from Oracle to Oracle successfully with connection argument, transaction isolation, operationName with upsert macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "transactionIsolationLevel" and set the value to: "transactionIsolationLevel" + Then Click on the Macro button of Property: "operationName" and set the value to: "oracleOperationName" + Then Click on the Macro button of Property: "relationTableKey" and set the value to: "oracleTableKey" + Then Validate "Oracle2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "upsertOperationName" for key "oracleOperationName" + Then Enter runtime argument value "upsertRelationTableKey" for key "oracleTableKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "upsertOperationName" for key "oracleOperationName" + Then Enter runtime argument value "upsertRelationTableKey" for key "oracleTableKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target table is equal to the values from source table + + @ORACLE_SOURCE_TEST @ORACLE_UPDATE_TABLE @Oracle_Required + Scenario: To verify data is getting transferred from Oracle source to Oracle sink using macro arguments for operation name with update + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "Oracle" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Oracle" from the plugins list as: "Sink" + Then Connect plugins: "Oracle" and "Oracle2" to establish connection + Then Navigate to the properties page of plugin: "Oracle" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "Oracle" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Oracle2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Select radio button plugin property: "connectionType" with value: "service" + Then Select radio button plugin property: "role" with value: "normal" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "transactionIsolationLevel" and set the value to: "transactionIsolationLevel" + Then Click on the Macro button of Property: "operationName" and set the value to: "oracleOperationName" + Then Click on the Macro button of Property: "relationTableKey" and set the value to: "oracleTableKey" + Then Validate "Oracle2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "operationName" for key "oracleOperationName" + Then Enter runtime argument value "relationTableKey" for key "oracleTableKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "transactionIsolationLevel" for key "transactionIsolationLevel" + Then Enter runtime argument value "operationName" for key "oracleOperationName" + Then Enter runtime argument value "relationTableKey" for key "oracleTableKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target table is equal to the values from source table diff --git a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature index 2d1ca9ad1..d6ad85cd4 100644 --- a/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature +++ b/oracle-plugin/src/e2e-test/features/source/OracleRunTime.feature @@ -296,7 +296,7 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline - Then Wait till pipeline preview is in running state + Then Wait till pipeline preview is in running state and check if any error occurs Then Verify the preview run status of pipeline in the logs is "failed" @ORACLE_SOURCE_TEST @BQ_SINK_TEST @@ -338,7 +338,9 @@ Feature: Oracle - Verify data transfer from Oracle source to BigQuery sink And Save and Deploy Pipeline And Run the Pipeline in Runtime And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java index 6edfcc8fd..b5a82e420 100644 --- a/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java +++ b/oracle-plugin/src/e2e-test/java/io.cdap.plugin/BQValidation.java @@ -33,7 +33,12 @@ import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Date; import java.util.List; @@ -44,6 +49,13 @@ public class BQValidation { + private static final List TIMESTAMP_DATE_FORMATS = Arrays.asList( + new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss"), + new SimpleDateFormat("yyyy-MM-dd")); + private static final List TIMESTAMP_TZ_DATE_FORMATS = Arrays.asList( + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX"), + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX")); + /** * Extracts entire data from source and target tables. * @@ -68,11 +80,12 @@ public static boolean validateDBToBQRecordValues(String schema, String sourceTab ResultSet.HOLD_CURSORS_OVER_COMMIT); ResultSet rsSource = statement1.executeQuery(getSourceQuery); - return compareResultSetAndJsonData(rsSource, jsonResponse); + return compareResultSetAndJsonData(rsSource, jsonResponse, false); } } - public static boolean validateBQToDBRecordValues(String schema, String sourceTable, String targetTable) + public static boolean validateBQToDBRecordValues(String schema, String sourceTable, String targetTable, + boolean isSchemaSmallCase) throws SQLException, ClassNotFoundException, ParseException, IOException, InterruptedException { List jsonResponse = new ArrayList<>(); List bigQueryRows = new ArrayList<>(); @@ -88,7 +101,7 @@ public static boolean validateBQToDBRecordValues(String schema, String sourceTab ResultSet.HOLD_CURSORS_OVER_COMMIT); ResultSet rsTarget = statement1.executeQuery(getTargetQuery); - return compareResultSetAndJsonData(rsTarget, jsonResponse); + return compareResultSetAndJsonData(rsTarget, jsonResponse, isSchemaSmallCase); } } @@ -119,7 +132,8 @@ private static void getBigQueryTableData(String table, List bigQueryRows * @throws ParseException If an error occurs while parsing the data. */ - public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData) + public static boolean compareResultSetAndJsonData(ResultSet rsSource, List bigQueryData, + boolean isSchemaSmallCase) throws SQLException, ParseException { ResultSetMetaData mdSource = rsSource.getMetaData(); boolean result = false; @@ -146,7 +160,8 @@ public static boolean compareResultSetAndJsonData(ResultSet rsSource, List getDBSpecificArguments() { return ImmutableMap.of(OracleConstants.DEFAULT_BATCH_VALUE, String.valueOf(defaultBatchValue)); diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index fde72c8ad..3094e7152 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -112,7 +112,9 @@ protected DBConnectorPath getDBConnectorPath(String path) { @Override protected SchemaReader getSchemaReader(String sessionID) { - return new OracleSourceSchemaReader(sessionID); + return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(), + config.getTreatPrecisionlessNumAsDeci(), + config.getTreatTimestampLTZAsTimestamp()); } @Override @@ -125,15 +127,8 @@ protected String getConnectionString(@Nullable String database) { if (database == null) { return config.getConnectionString(); } - if (OracleConstants.TNS_CONNECTION_TYPE.equals(config.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(config.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, config.getHost(), - config.getPort(), database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, - config.getHost(), config.getPort(), database); - } + return OracleConstants.getConnectionString(config.getConnectionType(), + config.getHost(), config.getPort(), database, config.getSSlMode()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java index 73b005243..79d14215b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java @@ -22,8 +22,6 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBSpecificConnectorConfig; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import javax.annotation.Nullable; @@ -43,12 +41,15 @@ public OracleConnectorConfig(String host, int port, String user, String password public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database) { - this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null); + this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null, + null, null); } public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName, String connectionArguments, String connectionType, String database, - String role) { + String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp, + @Nullable Boolean treatPrecisionlessNumAsDeci, + @Nullable Boolean treatTimestampLTZAsTimestamp) { this.host = host; this.port = port; @@ -59,17 +60,15 @@ public OracleConnectorConfig(String host, int port, String user, String password this.connectionType = connectionType; this.database = database; this.role = role; + this.useSSL = useSSL; + this.treatAsOldTimestamp = treatAsOldTimestamp; + this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci; + this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp; } @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, getPort(), database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, getPort(), database); - } + return OracleConstants.getConnectionString(connectionType, host, getPort(), database, useSSL); } @Name(OracleConstants.CONNECTION_TYPE) @@ -86,11 +85,25 @@ public String getConnectionString() { @Macro private String database; - @Name(OracleConstants.TRANSACTION_ISOLATION_LEVEL) - @Description("The transaction isolation level for the database session.") - @Macro + @Name(OracleConstants.USE_SSL) + @Description("Turns on SSL encryption. Connection will fail if SSL is not available") + @Nullable + public Boolean useSSL; + + @Name(OracleConstants.TREAT_AS_OLD_TIMESTAMP) + @Description("A hidden field to handle timestamp as CDAP's timestamp micros or string as per old behavior.") + @Nullable + public Boolean treatAsOldTimestamp; + + @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI) + @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.") @Nullable - private String transactionIsolationLevel; + public Boolean treatPrecisionlessNumAsDeci; + + @Name(OracleConstants.TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP) + @Description("A hidden field to handle mapping of Oracle Timestamp_LTZ data type to BQ Timestamp.") + @Nullable + public Boolean treatTimestampLTZAsTimestamp; @Override protected int getDefaultPort() { @@ -109,6 +122,23 @@ public String getDatabase() { return database; } + public Boolean getSSlMode() { + // return false if useSSL is null, otherwise return its value + return useSSL != null && useSSL; + } + + public Boolean getTreatAsOldTimestamp() { + return Boolean.TRUE.equals(treatAsOldTimestamp); + } + + public Boolean getTreatPrecisionlessNumAsDeci() { + return Boolean.TRUE.equals(treatPrecisionlessNumAsDeci); + } + + public Boolean getTreatTimestampLTZAsTimestamp() { + return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp); + } + @Override public Properties getConnectionArgumentsProperties() { Properties prop = super.getConnectionArgumentsProperties(); @@ -119,6 +149,7 @@ public Properties getConnectionArgumentsProperties() { return prop; } + @Override public String getTransactionIsolationLevel() { //if null default to the highest isolation level possible if (transactionIsolationLevel == null) { @@ -128,16 +159,7 @@ public String getTransactionIsolationLevel() { //This ensures that the role is mapped to the right serialization level, even w/ incorrect user input //if role is SYSDBA or SYSOP it will map to read_committed. else serialized return (!getRole().equals(ROLE_NORMAL)) ? TransactionIsolationLevel.Level.TRANSACTION_READ_COMMITTED.name() : - TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); - } - - @Override - public Map getAdditionalArguments() { - Map additonalArguments = new HashMap<>(); - if (getTransactionIsolationLevel() != null) { - additonalArguments.put(TransactionIsolationLevel.CONF_KEY, getTransactionIsolationLevel()); - } - return additonalArguments; + TransactionIsolationLevel.Level.valueOf(transactionIsolationLevel).name(); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java index 040780a89..31c7620ff 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java @@ -16,6 +16,8 @@ package io.cdap.plugin.oracle; +import javax.annotation.Nullable; + /** * Oracle Constants. */ @@ -27,6 +29,10 @@ private OracleConstants() { public static final String PLUGIN_NAME = "Oracle"; public static final String ORACLE_CONNECTION_STRING_SID_FORMAT = "jdbc:oracle:thin:@%s:%s:%s"; public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT = "jdbc:oracle:thin:@//%s:%s/%s"; + // Connection formats to accept protocol (e.g., jdbc:oracle:thin:@://:/) + public static final String ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL = "jdbc:oracle:thin:@%s:%s:%s/%s"; + public static final String ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL = + "jdbc:oracle:thin:@%s://%s:%s/%s"; public static final String ORACLE_CONNECTION_STRING_TNS_FORMAT = "jdbc:oracle:thin:@%s"; public static final String DEFAULT_BATCH_VALUE = "defaultBatchValue"; public static final String DEFAULT_ROW_PREFETCH = "defaultRowPrefetch"; @@ -34,6 +40,97 @@ private OracleConstants() { public static final String CONNECTION_TYPE = "connectionType"; public static final String ROLE = "role"; public static final String NAME_DATABASE = "database"; - public static final String TNS_CONNECTION_TYPE = "TNS"; + public static final String TNS_CONNECTION_TYPE = "tns"; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; + public static final String USE_SSL = "useSSL"; + public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp"; + public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci"; + public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp"; + + /** + * Constructs the Oracle connection string based on the provided connection type, host, port, and database. + * If SSL is enabled, the connection protocol will be "tcps" instead of "tcp". + * + * @param connectionType TNS/Service/SID + * @param host Host name of the oracle server + * @param port Port of the oracle server + * @param database Database to connect to + * @param useSSL Whether SSL/TLS is required(YES/NO) + * @return Connection String based on the given parameters and connection type. + */ + public static String getConnectionString(String connectionType, + @Nullable String host, + @Nullable int port, + String database, + @Nullable Boolean useSSL) { + // Use protocol as "tcps" when SSL is requested or else use "tcp". + String connectionProtocol; + boolean isSSLEnabled = false; + if (useSSL != null && useSSL) { + connectionProtocol = "tcps"; + isSSLEnabled = true; + } else { + connectionProtocol = "tcp"; + } + + switch (connectionType.toLowerCase()) { + case OracleConstants.TNS_CONNECTION_TYPE: + // TNS connection doesn't require protocol + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); + case OracleConstants.SERVICE_CONNECTION_TYPE: + // Create connection string for SERVICE type. + return getConnectionStringWithService(host, port, database, connectionProtocol, isSSLEnabled); + default: + // Default to SID format if no matching case is found. + return getConnectionStringWithSID(host, port, database, connectionProtocol, isSSLEnabled); + } + } + + /** + * Constructs the connection string for a SERVICE connection type. + * + * @param host Host name of the Oracle server. + * @param port Port of the Oracle server. + * @param database Database name to connect to. + * @param connectionProtocol Protocol to use for the connection ("tcp" or "tcps"). + * @param isSSLEnabled Indicates if SSL is enabled. + * @return Formatted connection string for a SERVICE connection. + */ + private static String getConnectionStringWithService(@Nullable String host, + @Nullable int port, + String database, + String connectionProtocol, + boolean isSSLEnabled) { + // Choose the appropriate format based on whether SSL is enabled. + if (isSSLEnabled) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT_WITH_PROTOCOL, + connectionProtocol, host, port, database); + } + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, + host, port, database); + } + + /** + * Constructs the connection string for a SID connection type. + * + * @param host Host name of the Oracle server. + * @param port Port of the Oracle server. + * @param database Database name to connect to. + * @param connectionProtocol Protocol to use for the connection ("tcp" or "tcps"). + * @param isSSLEnabled Indicates if SSL is enabled. + * @return Formatted connection string for a SID connection. + */ + private static String getConnectionStringWithSID(@Nullable String host, + @Nullable int port, + String database, + String connectionProtocol, + boolean isSSLEnabled) { + // Choose the appropriate format based on whether SSL is enabled. + if (isSSLEnabled) { + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT_WITH_PROTOCOL, + connectionProtocol, host, port, database); + } + return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, + host, port, database); + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java new file mode 100644 index 000000000..4a7b18278 --- /dev/null +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleETLDBOutputFormat.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.plugin.db.sink.ETLDBOutputFormat; + +/** + * Class that extends {@link ETLDBOutputFormat} to implement the abstract methods + */ +public class OracleETLDBOutputFormat extends ETLDBOutputFormat { + + /** + * This method is used to construct the upsert query for Oracle using MERGE statement. + * Example - MERGE INTO my_table target + * USING (SELECT ? AS id, ? AS name, ? AS age FROM dual) source + * ON (target.id = source.id) + * WHEN MATCHED THEN UPDATE SET target.name = source.name, target.age = source.age + * WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (source.id, source.name, source.age) + * @param table - Name of the table + * @param fieldNames - All the columns of the table + * @param listKeys - The columns used as keys for matching + * @return Upsert query in the form of string + */ + @Override + public String constructUpsertQuery(String table, String[] fieldNames, String[] listKeys) { + if (listKeys == null) { + throw new IllegalArgumentException( + "'Relation Table Key' must be specified for upsert operations. " + + "Please provide the list of key columns used to match records in the target table."); + } else if (fieldNames == null) { + throw new IllegalArgumentException( + "'Field Names' must be specified for upsert operations. " + + "Please provide the list of columns to be written to the target table."); + } else { + StringBuilder query = new StringBuilder(); + + // MERGE INTO target_table target + query.append("MERGE INTO ").append(table).append(" target "); + + // USING (SELECT ? AS col1, ? AS col2, ... FROM dual) source + query.append("USING (SELECT "); + for (int i = 0; i < fieldNames.length; ++i) { + query.append("? AS ").append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(" FROM dual) source "); + + // ON (target.key1 = source.key1 AND target.key2 = source.key2 ...) + query.append("ON ("); + for (int i = 0; i < listKeys.length; ++i) { + query.append("target.").append(listKeys[i]).append(" = source.").append(listKeys[i]); + if (i != listKeys.length - 1) { + query.append(" AND "); + } + } + query.append(") "); + + // WHEN MATCHED THEN UPDATE SET target.col1 = source.col1, target.col2 = source.col2 ... + // Only update non-key columns + query.append("WHEN MATCHED THEN UPDATE SET "); + boolean firstUpdateColumn = true; + for (String fieldName : fieldNames) { + boolean isKeyColumn = false; + for (String listKey : listKeys) { + String listKeyNoQuote = listKey.replace("\"", ""); + if (listKeyNoQuote.equals(fieldName)) { + isKeyColumn = true; + break; + } + } + if (!isKeyColumn) { + if (!firstUpdateColumn) { + query.append(", "); + } + query.append("target.").append(fieldName).append(" = source.").append(fieldName); + firstUpdateColumn = false; + } + } + + // WHEN NOT MATCHED THEN INSERT (col1, col2, ...) VALUES (source.col1, source.col2, ...) + query.append(" WHEN NOT MATCHED THEN INSERT ("); + for (int i = 0; i < fieldNames.length; ++i) { + query.append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(") VALUES ("); + for (int i = 0; i < fieldNames.length; ++i) { + query.append("source.").append(fieldNames[i]); + if (i != fieldNames.length - 1) { + query.append(", "); + } + } + query.append(")"); + + return query.toString(); + } + } + + @Override + public String constructUpdateQuery(String table, String[] fieldNames, String[] listKeys) { + // Oracle JDBC does not accept a trailing semicolon in prepared statements. + String query = super.constructUpdateQuery(table, fieldNames, listKeys); + if (query.endsWith(";")) { + return query.substring(0, query.length() - 1); + } + return query; + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java new file mode 100644 index 000000000..e0c770c9f --- /dev/null +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleErrorDetailsProvider.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +/** + * A custom ErrorDetailsProvider for Oracle plugin. + */ +public class OracleErrorDetailsProvider extends DBErrorDetailsProvider { + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } +} diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java index f8ebd9ac2..e11e455c1 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OraclePostAction.java @@ -57,13 +57,7 @@ public static class OracleQueryActionConfig extends DBSpecificQueryActionConfig @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, database); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(this.connectionType)) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, host, port, database); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, host, port, database); - } + return OracleConstants.getConnectionString(this.connectionType, host, port, database, null); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 45afbcb51..415cfa660 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -23,6 +23,7 @@ import io.cdap.cdap.api.annotation.MetadataProperty; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Output; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.batch.BatchSink; @@ -31,6 +32,8 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSinkConfig; @@ -59,7 +62,8 @@ public OracleSink(OracleSinkConfig oracleSinkConfig) { @Override protected DBRecord getDBRecord(StructuredRecord output) { - return new OracleSinkDBRecord(output, columnTypes); + return new OracleSinkDBRecord(output, columnTypes, oracleSinkConfig.getOperationName(), + oracleSinkConfig.getRelationTableKey()); } @Override @@ -71,6 +75,13 @@ protected FieldsValidator getFieldsValidator() { protected SchemaReader getSchemaReader() { return new OracleSinkSchemaReader(); } + + @Override + protected void addOutputContext(BatchSinkContext context) { + context.addOutput(Output.of(oracleSinkConfig.getReferenceName(), + new SinkOutputFormatProvider(OracleETLDBOutputFormat.class, getConfiguration()))); + } + @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String fqn = DBUtils.constructFQN("oracle", @@ -82,6 +93,15 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return OracleErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } /** * Oracle action configuration. diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java index 7bbd25f22..ee77a7fc9 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSinkDBRecord.java @@ -17,9 +17,13 @@ package io.cdap.plugin.oracle; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.ColumnType; +import io.cdap.plugin.db.Operation; import io.cdap.plugin.db.SchemaReader; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.List; /** @@ -28,13 +32,35 @@ */ public class OracleSinkDBRecord extends OracleSourceDBRecord { - public OracleSinkDBRecord(StructuredRecord record, List columnTypes) { + public OracleSinkDBRecord(StructuredRecord record, List columnTypes, Operation operationName, + String relationTableKey) { this.record = record; this.columnTypes = columnTypes; + this.operationName = operationName; + this.relationTableKey = relationTableKey; } @Override protected SchemaReader getSchemaReader() { return new OracleSinkSchemaReader(); } + + @Override + protected void insertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + // Get the field from the schema using the column name with ignoring case. + Schema.Field field = record.getSchema().getField(columnType.getName(), true); + writeToDB(stmt, field, fieldIndex); + } + } + + @Override + protected void upsertOperation(PreparedStatement stmt) throws SQLException { + for (int fieldIndex = 0; fieldIndex < columnTypes.size(); fieldIndex++) { + ColumnType columnType = columnTypes.get(fieldIndex); + Schema.Field field = record.getSchema().getField(columnType.getName()); + writeToDB(stmt, field, fieldIndex); + } + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 9a554a4a4..978155fc5 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -63,7 +63,14 @@ protected String createConnectionString() { @Override protected SchemaReader getSchemaReader() { - return new OracleSourceSchemaReader(); + // PLUGIN-1893 : Based on field/properties from Oracle source and Oracle connection we will pass the flag to control + // handle schema to make it backward compatible. + boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp(); + boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci(); + boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp(); + + return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci, + treatTimestampLTZAsTimestamp); } @Override @@ -71,6 +78,16 @@ protected Class getDBRecordType() { return OracleSourceDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.ORACLE_SUPPORTED_DOC_URL; + } + + @Override + protected String getErrorDetailsProviderClassName() { + return OracleErrorDetailsProvider.class.getName(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("oracle", @@ -117,9 +134,11 @@ public OracleSourceConfig(String host, int port, String user, String password, S String connectionArguments, String connectionType, String database, String role, int defaultBatchValue, int defaultRowPrefetch, String importQuery, Integer numSplits, int fetchSize, - String boundingQuery, String splitBy) { + String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp, + Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) { this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments, - connectionType, database, role); + connectionType, database, role, useSSL, treatAsOldTimestamp, + treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp); this.defaultBatchValue = defaultBatchValue; this.defaultRowPrefetch = defaultRowPrefetch; this.fetchSize = fetchSize; @@ -131,15 +150,8 @@ public OracleSourceConfig(String host, int port, String user, String password, S @Override public String getConnectionString() { - if (OracleConstants.TNS_CONNECTION_TYPE.equals(connection.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_TNS_FORMAT, connection.getDatabase()); - } else if (OracleConstants.SERVICE_CONNECTION_TYPE.equals(connection.getConnectionType())) { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SERVICE_NAME_FORMAT, connection.getHost(), - connection.getPort(), connection.getDatabase()); - } else { - return String.format(OracleConstants.ORACLE_CONNECTION_STRING_SID_FORMAT, - connection.getHost(), connection.getPort(), connection.getDatabase()); - } + return OracleConstants.getConnectionString(connection.getConnectionType(), connection.getHost(), + connection.getPort(), connection.getDatabase(), connection.getSSlMode()); } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java index 3f7c2a20a..44131a01b 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceDBRecord.java @@ -116,8 +116,8 @@ protected void handleField(ResultSet resultSet, StructuredRecord.Builder recordB @Override protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, String fieldName, int fieldIndex) throws SQLException { - int sqlType = columnTypes.get(fieldIndex).getType(); int sqlIndex = fieldIndex + 1; + int sqlType = modifiableColumnTypes.get(fieldIndex).getType(); // TIMESTAMP and TIMESTAMPTZ types needs to be handled using the specific oracle types to ensure that the data // inserted matches with the provided value. As Oracle driver internally alters the values provided @@ -160,8 +160,8 @@ protected void writeNonNullToDB(PreparedStatement stmt, Schema fieldSchema, String timestampString = Timestamp.valueOf(localDateTime).toString(); Object timestampWithTimeZone = createOracleTimestamp(stmt.getConnection(), timestampString); stmt.setObject(sqlIndex, timestampWithTimeZone); - } else if (Schema.LogicalType.TIMESTAMP_MICROS.equals(fieldSchema.getLogicalType())) { - // Deprecated: Handle the case when the Timestamp is mapped to CDAP Timestamp type + } else { + // Handle the case when the Timestamp is mapped to CDAP Timestamp type or CDAP Date type. super.writeNonNullToDB(stmt, fieldSchema, fieldName, fieldIndex); } } else { diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java index 7d35f9bc7..208b70410 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.db.CommonSchemaReader; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +27,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Set; +import javax.annotation.Nullable; /** * Oracle Source schema reader. @@ -65,14 +67,19 @@ public class OracleSourceSchemaReader extends CommonSchemaReader { ); private final String sessionID; + private final Boolean isTimestampOldBehavior; + private final Boolean isPrecisionlessNumAsDecimal; + private final Boolean isTimestampLtzFieldTimestamp; public OracleSourceSchemaReader() { - this(null); + this(null, false, false, false); } - - public OracleSourceSchemaReader(String sessionID) { - super(); + public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior, + boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) { this.sessionID = sessionID; + this.isTimestampOldBehavior = isTimestampOldBehavior; + this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal; + this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp; } @Override @@ -81,10 +88,11 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti switch (sqlType) { case TIMESTAMP_TZ: - return Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); - case Types.TIMESTAMP: + return isTimestampOldBehavior ? Schema.of(Schema.Type.STRING) : Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); case TIMESTAMP_LTZ: - return Schema.of(Schema.LogicalType.DATETIME); + return getTimestampLtzSchema(); + case Types.TIMESTAMP: + return isTimestampOldBehavior ? super.getSchema(metadata, index) : Schema.of(Schema.LogicalType.DATETIME); case BINARY_FLOAT: return Schema.of(Schema.Type.FLOAT); case BINARY_DOUBLE: @@ -107,12 +115,24 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti // For a Number type without specified precision and scale, precision will be 0 and scale will be -127 if (precision == 0) { // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832 - LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " - + "converting into STRING type to avoid any precision loss.", - metadata.getColumnName(index), - metadata.getColumnTypeName(index), - metadata.getColumnName(index))); - return Schema.of(Schema.Type.STRING); + if (isPrecisionlessNumAsDecimal) { + precision = 38; + scale = 0; + LOG.warn(String.format("%s type with undefined precision and scale is detected, " + + "there may be a precision loss while running the pipeline. " + + "Please define an output precision and scale for field '%s' to avoid " + + "precision loss.", + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.decimalOf(precision, scale); + } else { + LOG.warn(String.format("Field '%s' is a %s type without precision and scale, " + + "converting into STRING type to avoid any precision loss.", + metadata.getColumnName(index), + metadata.getColumnTypeName(index), + metadata.getColumnName(index))); + return Schema.of(Schema.Type.STRING); + } } return Schema.decimalOf(precision, scale); } @@ -121,6 +141,12 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti } } + private @NotNull Schema getTimestampLtzSchema() { + return isTimestampOldBehavior || isTimestampLtzFieldTimestamp + ? Schema.of(Schema.LogicalType.TIMESTAMP_MICROS) + : Schema.of(Schema.LogicalType.DATETIME); + } + @Override public boolean shouldIgnoreColumn(ResultSetMetaData metadata, int index) throws SQLException { if (sessionID == null) { diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java new file mode 100644 index 000000000..994c86b55 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleETLDBOutputFormatTest.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import org.junit.Assert; +import org.junit.Test; + +public class OracleETLDBOutputFormatTest { + + private final OracleETLDBOutputFormat outputFormat = new OracleETLDBOutputFormat(); + + @Test + public void testConstructUpsertQueryBasic() { + String[] fieldNames = {"id", "name", "age"}; + String[] listKeys = {"id"}; + String table = "my_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO my_table target " + + "USING (SELECT ? AS id, ? AS name, ? AS age FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name, target.age = source.age " + + "WHEN NOT MATCHED THEN INSERT (id, name, age) VALUES (source.id, source.name, source.age)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQueryMultipleKeys() { + String[] fieldNames = {"id", "code", "name", "value"}; + String[] listKeys = {"id", "code"}; + String table = "composite_key_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO composite_key_table target " + + "USING (SELECT ? AS id, ? AS code, ? AS name, ? AS value FROM dual) source " + + "ON (target.id = source.id AND target.code = source.code) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name, target.value = source.value " + + "WHEN NOT MATCHED THEN INSERT (id, code, name, value) VALUES (source.id, source.code, source.name, source" + + ".value)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQuerySingleField() { + String[] fieldNames = {"id", "name"}; + String[] listKeys = {"id"}; + String table = "single_field_update_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO single_field_update_table target " + + "USING (SELECT ? AS id, ? AS name FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) VALUES (source.id, source.name)"; + + Assert.assertEquals(expected, result); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructUpsertQueryNullListKeys() { + String[] fieldNames = {"id", "name", "age"}; + String table = "my_table"; + + outputFormat.constructUpsertQuery(table, fieldNames, null); + } + + @Test(expected = IllegalArgumentException.class) + public void testConstructUpsertQueryNullFieldNames() { + String[] listKeys = {"id"}; + String table = "my_table"; + + outputFormat.constructUpsertQuery(table, null, listKeys); + } + + @Test + public void testConstructUpsertQueryAllFieldsAreKeys() { + String[] fieldNames = {"id", "code"}; + String[] listKeys = {"id", "code"}; + String table = "all_keys_table"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + // When all fields are keys, the UPDATE SET clause will be empty after "SET " + // Note: There's an extra space before "WHEN NOT MATCHED" due to implementation + String expected = "MERGE INTO all_keys_table target " + + "USING (SELECT ? AS id, ? AS code FROM dual) source " + + "ON (target.id = source.id AND target.code = source.code) " + + "WHEN MATCHED THEN UPDATE SET " + + "WHEN NOT MATCHED THEN INSERT (id, code) VALUES (source.id, source.code)"; + + Assert.assertEquals(expected, result); + } + + @Test + public void testConstructUpsertQueryWithSpecialTableName() { + String[] fieldNames = {"id", "name"}; + String[] listKeys = {"id"}; + String table = "SCHEMA.MY_TABLE"; + + String result = outputFormat.constructUpsertQuery(table, fieldNames, listKeys); + + String expected = "MERGE INTO SCHEMA.MY_TABLE target " + + "USING (SELECT ? AS id, ? AS name FROM dual) source " + + "ON (target.id = source.id) " + + "WHEN MATCHED THEN UPDATE SET target.name = source.name " + + "WHEN NOT MATCHED THEN INSERT (id, name) VALUES (source.id, source.name)"; + + Assert.assertEquals(expected, result); + } +} diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java index a2c9bcd5e..7ec6f3844 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleFailedConnectionTest.java @@ -28,7 +28,8 @@ public class OracleFailedConnectionTest extends DBSpecificFailedConnectionTest { public void test() throws ClassNotFoundException, IOException { OracleConnector connector = new OracleConnector( - new OracleConnectorConfig("localhost", 1521, "username", "password", "jdbc", "", "database")); + new OracleConnectorConfig("localhost", 1521, "username", "password", "jdbc", "", + "SID", "database")); super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string:" + " jdbc:oracle:thin:@localhost:1521:database and arguments: " + diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java new file mode 100644 index 000000000..1ff77c533 --- /dev/null +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.oracle; + +import com.google.common.collect.Lists; +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; + +public class OracleSchemaReaderTest { + + @Test + public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + + } + + @Test + public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException { + OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false); + + ResultSet resultSet = Mockito.mock(ResultSet.class); + ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class); + + Mockito.when(resultSet.getMetaData()).thenReturn(metadata); + + Mockito.when(metadata.getColumnCount()).thenReturn(2); + // -101 is for TIMESTAMP_TZ + Mockito.when(metadata.getColumnType(1)).thenReturn(-101); + Mockito.when(metadata.getColumnName(1)).thenReturn("column1"); + + // -102 is for TIMESTAMP_LTZ + Mockito.when(metadata.getColumnType(2)).thenReturn(-102); + Mockito.when(metadata.getColumnName(2)).thenReturn("column2"); + + List expectedSchemaFields = Lists.newArrayList(); + expectedSchemaFields.add(Schema.Field.of("column1", Schema.of(Schema.LogicalType.TIMESTAMP_MICROS))); + expectedSchemaFields.add(Schema.Field.of("column2", Schema.of(Schema.LogicalType.DATETIME))); + + List actualSchemaFields = schemaReader.getSchemaFields(resultSet); + + Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName()); + Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema()); + Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName()); + Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema()); + } +} diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 30d5b345f..e3537e2ba 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -100,6 +100,26 @@ "default": "TRANSACTION_SERIALIZABLE" } }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", @@ -172,6 +192,29 @@ "label": "Schema Name", "name": "dbSchemaName" }, + { + "widget-type": "radio-group", + "label": "Operation Name", + "name": "operationName", + "widget-attributes": { + "default": "insert", + "layout": "inline", + "options": [ + { + "id": "insert", + "label": "INSERT" + }, + { + "id": "update", + "label": "UPDATE" + }, + { + "id": "upsert", + "label": "UPSERT" + } + ] + } + }, { "widget-type": "hidden", "label": "Operation Name", @@ -181,9 +224,10 @@ } }, { - "widget-type": "hidden", + "name": "relationTableKey", + "widget-type": "csv", "label": "Table Key", - "name": "relationTableKey" + "widget-attributes": {} } ] }, diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 0fc0a5285..ab35f3e8c 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -100,6 +100,83 @@ "default": "TRANSACTION_SERIALIZABLE" } }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, { "name": "connectionType", "label": "Connection Type", @@ -306,6 +383,14 @@ { "type": "property", "name": "transactionIsolationLevel" + }, + { + "type": "property", + "name": "getTreatAsOldTimestampConn" + }, + { + "type": "property", + "name": "treatPrecisionlessNumAsDeci" } ] }, diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 46f006c9c..005c3ffbd 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -109,6 +109,83 @@ ], "default": "TRANSACTION_SERIALIZABLE" } + }, + { + "widget-type": "hidden", + "label": "TLS Encryption", + "name": "useSSL", + "description": "Enable TLS encryption (true/false)", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat as old timestamp", + "name": "treatAsOldTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat precision less number as Decimal(old behavior)", + "name": "treatPrecisionlessNumAsDeci", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } + }, + { + "widget-type": "hidden", + "label": "Treat Timestamp_LTZ as Timestamp", + "name": "treatTimestampLTZAsTimestamp", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "true" + }, + { + "id": "false", + "label": "false" + } + ] + } } ] }, diff --git a/pom.xml b/pom.xml index a6b40960c..fa94b54b0 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ io.cdap.plugin database-plugins-parent - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT pom Database Plugins Collection of database plugins @@ -44,6 +44,7 @@ cloudsql-postgresql-plugin teradata-plugin generic-db-argument-setter + amazon-redshift-plugin @@ -55,15 +56,31 @@ + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + + 7 true UTF-8 - 6.9.1 - 2.11.1 + 6.11.0 + 2.13.1 13.0.1 - 2.3.0 + 3.3.6 2.2.4 4.13 2.0.1 @@ -77,23 +94,12 @@ - - sonatype - https://oss.sonatype.org/content/groups/public - sonatype-snapshots - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots - - - sonatype - https://oss.sonatype.org/content/groups/public/ - - - @@ -348,16 +354,6 @@ - - - sonatype.release - https://oss.sonatype.org/service/local/staging/deploy/maven2 - - - sonatype.snapshots - https://oss.sonatype.org/content/repositories/snapshots - - ${testSourceLocation} @@ -368,8 +364,8 @@ 1.1.0 - system:cdap-data-pipeline[6.9.0-SNAPSHOT,7.0.0-SNAPSHOT) - system:cdap-data-streams[6.9.0-SNAPSHOT,7.0.0-SNAPSHOT) + system:cdap-data-pipeline[6.11.0-SNAPSHOT,7.0.0-SNAPSHOT) + system:cdap-data-streams[6.11.0-SNAPSHOT,7.0.0-SNAPSHOT) @@ -531,14 +527,14 @@ - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.2 + org.sonatype.central + central-publishing-maven-plugin + 0.8.0 true - https://oss.sonatype.org - sonatype.release - 655dc88dc770c3 + sonatype.release + false + true @@ -611,6 +607,7 @@ src/e2e-test/java TestRunner.java + 31.1-jre @@ -668,7 +665,8 @@ integration-test - verify + + @@ -723,7 +721,7 @@ io.cdap.tests.e2e cdap-e2e-framework - 0.3.0-SNAPSHOT + 0.4.2 test @@ -736,7 +734,7 @@ ch.qos.logback logback-classic - 1.2.8 + 1.3.15 runtime diff --git a/postgresql-plugin/docs/PostgreSQL-connector.md b/postgresql-plugin/docs/PostgreSQL-connector.md index 739c678e3..fe442cbf1 100644 --- a/postgresql-plugin/docs/PostgreSQL-connector.md +++ b/postgresql-plugin/docs/PostgreSQL-connector.md @@ -22,6 +22,14 @@ authentication. Optional for databases that do not require authentication. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Database:** The name of the database to connect to. **Connection Arguments:** A list of arbitrary string tag/value pairs as connection arguments. These arguments diff --git a/postgresql-plugin/docs/Postgres-batchsink.md b/postgresql-plugin/docs/Postgres-batchsink.md index b8a996463..82065e0fd 100644 --- a/postgresql-plugin/docs/Postgres-batchsink.md +++ b/postgresql-plugin/docs/Postgres-batchsink.md @@ -39,6 +39,14 @@ You also can use the macro function ${conn(connection-name)}. **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/docs/Postgres-batchsource.md b/postgresql-plugin/docs/Postgres-batchsource.md index af359022d..559723526 100644 --- a/postgresql-plugin/docs/Postgres-batchsource.md +++ b/postgresql-plugin/docs/Postgres-batchsource.md @@ -49,6 +49,14 @@ For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is s **Password:** Password to use to connect to the specified database. +**Transaction Isolation Level** The transaction isolation level of the databse connection +- TRANSACTION_READ_COMMITTED: No dirty reads. Non-repeatable reads and phantom reads are possible. +- TRANSACTION_SERIALIZABLE: No dirty reads. Non-repeatable and phantom reads are prevented. +- TRANSACTION_REPEATABLE_READ: No dirty reads. Prevents non-repeatable reads, but phantom reads are still possible. +- Note: PostgreSQL does not implement `TRANSACTION_READ_UNCOMMITTED` as a distinct isolation level. Instead, this mode behaves identically to`TRANSACTION_READ_COMMITTED`, which is why it is not exposed as a separate option. + +For more details on the Transaction Isolation Levels supported in PostgreSQL, refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/transaction-iso.html#TRANSACTION-ISO) + **Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations. diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index 7f3e6f14c..73912e656 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -20,17 +20,51 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT PostgreSQL plugin postgresql-plugin 4.0.0 + PostgreSQL database plugins + https://github.com/data-integrations/database-plugins + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://cdap.io + + + + + scm:git:https://github.com/cdapio/hydrator-plugins.git + scm:git:git@github.com:cdapio/hydrator-plugins.git + https://github.com/cdapio/hydrator-plugins.git + HEAD + io.cdap.cdap cdap-etl-api + ${cdap.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided io.cdap.plugin @@ -40,10 +74,12 @@ io.cdap.plugin hydrator-common + ${cdap.plugin.version} com.google.guava guava + ${guava.version} @@ -63,15 +99,14 @@ io.cdap.cdap hydrator-test + ${cdap.version} + test io.cdap.cdap cdap-data-pipeline3_2.12 - - - io.cdap.cdap - cdap-api - provided + ${cdap.version} + test org.jetbrains @@ -82,11 +117,13 @@ org.mockito mockito-core + ${mockito.version} test junit junit + ${junit.version} test diff --git a/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature b/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature index ad83a1607..99a6aca3c 100644 --- a/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature +++ b/postgresql-plugin/src/e2e-test/features/postgresql/source/PostgresqlRunTime.feature @@ -147,7 +147,9 @@ Feature: PostgreSQL - Verify data transfer from PostgreSQL source to BigQuery si And Save and Deploy Pipeline And Run the Pipeline in Runtime And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 6e1929245..f793e3be7 100644 --- a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -18,5 +18,5 @@ errorMessageInvalidSourceHost=SQL error while getting query schema: The connecti errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"targettable_ errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection -errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: The column index is out of range: 1, \ - number of columns: 0.. Please check the system logs for more details. +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'PostgreSQL' encountered : \ + java.io.IOException: The column index is out of range: 1, number of columns: 0.. Please check the system logs for more details. diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java index bed7a3ec3..4ed7cf804 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConstants.java @@ -16,13 +16,19 @@ package io.cdap.plugin.postgres; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; + /** * Postgres constants. */ public final class PostgresConstants { private PostgresConstants() { - throw new AssertionError("Should not instantiate static utility class."); + String errorMessage = "Should not instantiate static utility class."; + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessage, ErrorType.SYSTEM, false, new AssertionError(errorMessage)); } public static final String PLUGIN_NAME = "Postgres"; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java new file mode 100644 index 000000000..a7de4e5dc --- /dev/null +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresErrorDetailsProvider.java @@ -0,0 +1,96 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.postgres; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; +import io.cdap.plugin.util.DBUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * A custom ErrorDetailsProvider for Postgres plugins. + */ +public class PostgresErrorDetailsProvider extends DBErrorDetailsProvider { + // https://www.postgresql.org/docs/current/errcodes-appendix.html + private static final Map ERROR_CODE_TO_ERROR_TYPE; + private static final Map ERROR_CODE_TO_ERROR_CATEGORY; + static { + ERROR_CODE_TO_ERROR_TYPE = new HashMap<>(); + ERROR_CODE_TO_ERROR_TYPE.put("01", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("02", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("08", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("0A", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("22", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("23", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("28", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("40", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("42", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("53", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("54", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("55", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("57", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("58", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("P0", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("XX", ErrorType.SYSTEM); + + ErrorCategory.ErrorCategoryEnum plugin = ErrorCategory.ErrorCategoryEnum.PLUGIN; + ERROR_CODE_TO_ERROR_CATEGORY = new HashMap<>(); + ERROR_CODE_TO_ERROR_CATEGORY.put("01", new ErrorCategory(plugin, "Warning")); + ERROR_CODE_TO_ERROR_CATEGORY.put("02", new ErrorCategory(plugin, "No Data")); + ERROR_CODE_TO_ERROR_CATEGORY.put("08", new ErrorCategory(plugin, "Postgres Server Connection Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0A", new ErrorCategory(plugin, "Postgres Server Feature Not Supported")); + ERROR_CODE_TO_ERROR_CATEGORY.put("22", new ErrorCategory(plugin, "Postgres Server Data Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("23", new ErrorCategory(plugin, "Postgres Integrity Constraint Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("28", new ErrorCategory(plugin, "Postgres Invalid Authorization Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("40", new ErrorCategory(plugin, "Transaction Rollback")); + ERROR_CODE_TO_ERROR_CATEGORY.put("42", new ErrorCategory(plugin, "Syntax Error or Access Rule Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("53", new ErrorCategory(plugin, "Postgres Server Insufficient Resources")); + ERROR_CODE_TO_ERROR_CATEGORY.put("54", new ErrorCategory(plugin, "Postgres Program Limit Exceeded")); + ERROR_CODE_TO_ERROR_CATEGORY.put("55", new ErrorCategory(plugin, "Object Not in Prerequisite State")); + ERROR_CODE_TO_ERROR_CATEGORY.put("57", new ErrorCategory(plugin, "Operator Intervention")); + ERROR_CODE_TO_ERROR_CATEGORY.put("58", new ErrorCategory(plugin, "Postgres Server System Error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("P0", new ErrorCategory(plugin, "PL/pgSQL Error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("XX", new ErrorCategory(plugin, "Postgres Server Internal Error")); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + + @Override + protected ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { + if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && + ERROR_CODE_TO_ERROR_TYPE.containsKey(sqlState.substring(0, 2))) { + return ERROR_CODE_TO_ERROR_TYPE.get(sqlState.substring(0, 2)); + } + return ErrorType.UNKNOWN; + } + + @Override + protected ErrorCategory getErrorCategoryFromSqlState(String sqlState) { + if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && + ERROR_CODE_TO_ERROR_CATEGORY.containsKey(sqlState.substring(0, 2))) { + return ERROR_CODE_TO_ERROR_CATEGORY.get(sqlState.substring(0, 2)); + } + return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); + } +} diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 8fd91cc63..73430c1e2 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -116,6 +116,16 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { return new LineageRecorder(context, asset); } + @Override + protected String getErrorDetailsProviderClassName() { + return PostgresErrorDetailsProvider.class.getName(); + } + + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + /** * PostgreSQL action configuration. */ @@ -165,6 +175,11 @@ public Map getDBSpecificArguments() { return ImmutableMap.of(PostgresConstants.CONNECTION_TIMEOUT, String.valueOf(connectionTimeout)); } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override protected PostgresConnectorConfig getConnection() { return connection; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index d6677884f..b230f3d1e 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -67,11 +67,21 @@ protected SchemaReader getSchemaReader() { return new PostgresSchemaReader(); } + @Override + protected String getErrorDetailsProviderClassName() { + return PostgresErrorDetailsProvider.class.getName(); + } + @Override protected Class getDBRecordType() { return PostgresDBRecord.class; } + @Override + protected String getExternalDocumentationLink() { + return DBUtils.POSTGRES_SUPPORTED_DOC_URL; + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("postgres", @@ -133,6 +143,11 @@ protected PostgresConnectorConfig getConnection() { return connection; } + @Override + public String getTransactionIsolationLevel() { + return connection.getTransactionIsolationLevel(); + } + @Override public void validate(FailureCollector collector) { ConfigUtil.validateConnection(this, useConnection, connection, collector); diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 091afc972..9a7a02e14 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -31,6 +31,19 @@ "default": "5432" } }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "textbox", "label": "Database", diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 6aa2dad8a..14e6f8154 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -186,6 +199,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 0e4ba28c1..60de4725f 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -65,6 +65,19 @@ "label": "Password", "name": "password" }, + { + "widget-type": "select", + "label": "Transaction Isolation Level", + "name": "transactionIsolationLevel", + "widget-attributes": { + "values": [ + "TRANSACTION_READ_COMMITTED", + "TRANSACTION_REPEATABLE_READ", + "TRANSACTION_SERIALIZABLE" + ], + "default": "TRANSACTION_SERIALIZABLE" + } + }, { "widget-type": "keyvalue", "label": "Connection Arguments", @@ -206,6 +219,10 @@ "type": "property", "name": "port" }, + { + "type": "property", + "name": "transactionIsolationLevel" + }, { "type": "property", "name": "database" diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 86b40a38e..6e541ddf2 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -20,7 +20,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT SAP HANA plugin diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index fa770a19a..6fa152ad7 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -21,7 +21,7 @@ database-plugins-parent io.cdap.plugin - 1.11.0-SNAPSHOT + 1.12.4-SNAPSHOT teradata-plugin