Cdap e2e framework update

.github/workflows/build-report.yml

@@ -0,0 +1,51 @@
+# Copyright © 2024 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop
+name: Build Unit Tests Report
+
+on:
+  workflow_run:
+    workflows:
+    - Build with unit tests
+    types:
+    - completed
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+
+    steps:
+    # Pinned 1.0.0 version
+    - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a
+
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        run-id: ${{ github.event.workflow_run.id }}
+        path: artifacts/
+
+    - name: Surefire Report
+      # Pinned 3.5.2 version
+      uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
+      if: always()
+      with:
+        report_paths: '**/target/surefire-reports/TEST-*.xml'
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        detailed_summary: true
+        commit: ${{ github.event.workflow_run.head_sha }}
+        check_name: Build Test Report
+

.github/workflows/build.yml

@@ -15,28 +15,34 @@
 name: Build with unit tests
 
 on:
-  workflow_run:
-    workflows:
-      - Trigger build
-    types:
-      - completed
+  push:
+    branches: [ develop, release/** ]
+  pull_request:
+    branches: [ develop, release/** ]
+    types: [opened, synchronize, reopened, labeled]
 
 jobs:
   build:
     runs-on: k8s-runner-build
 
-    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
-
+    # We allow builds:
+    # 1) When it's a merge into a branch
+    # 2) For PRs that are labeled as build and
+    #  - It's a code change
+    #  - A build label was just added
+    # A bit complex, but prevents builds when other labels are manipulated
+    if: >
+      github.event_name == 'push'
+      || (contains(github.event.pull_request.labels.*.name, 'build')
+         && (github.event.action != 'labeled' || github.event.label.name == 'build')
+         )
     steps:
-      # Pinned 1.0.0 version
-      - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94
-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.workflow_run.head_sha }}
 
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
@@ -47,21 +53,12 @@ jobs:
         run: mvn clean test -fae -T 2 -B -V -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
 
       - name: Archive build artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Build debug files
+          name: reports-${{ github.run_id }}
           path: |
             **/target/rat.txt
             **/target/surefire-reports/*
 
-      - name: Surefire Report
-        # Pinned 3.5.2 version
-        uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
-        if: always()
-        with:
-          report_paths: '**/target/surefire-reports/TEST-*.xml'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          detailed_summary: true
-          commit: ${{ github.event.workflow_run.head_sha }}
-          check_name: Test Report
+

.github/workflows/e2e.yml

@@ -16,9 +16,9 @@ name: Build e2e tests
 
 on:
   push:
-    branches: [ develop ]
+    branches: [ develop, release/** ]
   pull_request:
-      branches: [ develop ]
+      branches: [ develop, release/** ]
       types: [ opened, synchronize, reopened, labeled ]
   workflow_dispatch:
 
@@ -45,7 +45,7 @@ jobs:
 
     steps:
       # Pinned 1.0.0 version
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           path: plugin
           submodules: 'recursive'
@@ -61,13 +61,14 @@ jobs:
               - '${{ matrix.module }}/**/e2e-test/**'
 
       - name: Checkout e2e test repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: cdapio/cdap-e2e-tests
           path: e2e
+          ref: release/6.11
 
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
@@ -156,24 +157,21 @@ jobs:
           CLOUDSQL_MYSQL_PASSWORD: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_PASSWORD }}
           CLOUDSQL_MYSQL_CONNECTION_NAME: ${{ steps.secrets.outputs.CLOUDSQL_MYSQL_CONNECTION_NAME }}
 
-      - name: Upload report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: Cucumber report - ${{ matrix.module }}
-          path: ./**/target/cucumber-reports
-
       - name: Upload debug files
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: Debug files - ${{ matrix.module }}
           path: ./**/target/e2e-debug
 
       - name: Upload files to GCS
-        uses: google-github-actions/upload-cloud-storage@v0
+        uses: google-github-actions/upload-cloud-storage@v2
         if: always()
         with:
           path: ./plugin
           destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}
           glob: '**/target/cucumber-reports/**'
+
+      - name: Cucumber Report URL
+        if: always()
+        run: echo "https://storage.googleapis.com/e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}/plugin/${{ matrix.module }}/target/cucumber-reports/advanced-reports/cucumber-html-reports/overview-features.html"

amazon-redshift-plugin/docs/Redshift-batchsource.md

@@ -0,0 +1,102 @@
+# Amazon Redshift Batch Source
+
+Description
+-----------
+Reads from an Amazon Redshift database using a configurable SQL query.
+Outputs one record for each row returned by the query.
+
+
+Use Case
+--------
+The source is used whenever you need to read from an Amazon Redshift database. For example, you may want
+to create daily snapshots of a database table by using this source and writing to
+a TimePartitionedFileSet.
+
+
+Properties
+----------
+**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc.
+
+**JDBC Driver name:** Name of the JDBC driver to use.
+
+**Host:** Host URL of the current master instance of Redshift cluster.
+
+**Port:** Port that Redshift master instance is listening to.
+
+**Database:** Redshift database name.
+
+**Import Query:** The SELECT query to use to import data from the specified table.
+You can specify an arbitrary number of columns to import, or import all columns using \*. The Query should
+contain the '$CONDITIONS' string. For example, 'SELECT * FROM table WHERE $CONDITIONS'.
+The '$CONDITIONS' string will be replaced by 'splitBy' field limits specified by the bounding query.
+The '$CONDITIONS' string is not required if numSplits is set to one.
+
+**Bounding Query:** Bounding Query should return the min and max of the values of the 'splitBy' field.
+For example, 'SELECT MIN(id),MAX(id) FROM table'. Not required if numSplits is set to one.
+
+**Split-By Field Name:** Field Name which will be used to generate splits. Not required if numSplits is set to one.
+
+**Number of Splits to Generate:** Number of splits to generate.
+
+**Username:** User identity for connecting to the specified database.
+
+**Password:** Password to use to connect to the specified database.
+
+**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments
+will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations.
+
+**Schema:** The schema of records output by the source. This will be used in place of whatever schema comes
+back from the query. However, it must match the schema that comes back from the query,
+except it can mark fields as nullable and can contain a subset of the fields.
+
+**Fetch Size:** The number of rows to fetch at a time per split. Larger fetch size can result in faster import,
+with the tradeoff of higher memory usage.
+
+Example
+------
+Suppose you want to read data from an Amazon Redshift database named "prod" that is running on
+"redshift.xyz.eu-central-1.redshift.amazonaws.com", port 5439, as "sa" user with "Test11" password.
+Ensure that the driver for Redshift is installed (you can also provide driver name for some specific driver,
+otherwise "redshift" will be used), then configure the plugin with:then configure plugin with:
+
+```
+Reference Name: "src1"
+Driver Name: "redshift"
+Host: "redshift.xyz.eu-central-1.redshift.amazonaws.com"
+Port: 5439
+Database: "prod"
+Import Query: "select id, name, email, phone from users;"
+Number of Splits to Generate: 1
+Username: "sa"
+Password: "Test11"
+```  
+
+Data Types Mapping
+------------------
+
+Mapping of Redshift types to CDAP schema:
+
+| Redshift Data Type                                  | CDAP Schema Data Type | Comment                          |
+|-----------------------------------------------------|-----------------------|----------------------------------|
+| bigint                                              | long                  |                                  |
+| boolean                                             | boolean               |                                  |
+| character                                           | string                |                                  |
+| character varying                                   | string                |                                  |
+| double precision                                    | double                |                                  |
+| integer                                             | int                   |                                  |
+| numeric(precision, scale)/decimal(precision, scale) | decimal               |                                  |
+| numeric(with 0 precision)                           | string                |                                  |
+| real                                                | float                 |                                  |
+| smallint                                            | int                   |                                  |
+| smallserial                                         | int                   |                                  |
+| text                                                | string                |                                  |
+| date                                                | date                  |                                  |
+| time [ (p) ] [ without time zone ]                  | time                  |                                  |
+| time [ (p) ] with time zone                         | string                |                                  |
+| timestamp [ (p) ] [ without time zone ]             | timestamp             |                                  |
+| timestamp [ (p) ] with time zone                    | timestamp             | stored in UTC format in database |
+| xml                                                 | string                |                                  |
+| json                                                | string                |                                  |
+| super                                               | string                |                                  |
+| geometry                                            | bytes                 |                                  |
+| hllsketch                                           | string                |                                  |

amazon-redshift-plugin/docs/Redshift-connector.md

@@ -0,0 +1,26 @@
+# Amazon Redshift Connection
+
+Description
+-----------
+Use this connection to access data in an Amazon Redshift database using JDBC.
+
+Properties
+----------
+**Name:** Name of the connection. Connection names must be unique in a namespace.
+
+**Description:** Description of the connection.
+
+**JDBC Driver name:** Name of the JDBC driver to use.
+
+**Host:** Host of the current master instance of Redshift cluster.
+
+**Port:** Port that Redshift master instance is listening to.
+
+**Database:** Redshift database name.
+
+**Username:** User identity for connecting to the specified database.
+
+**Password:** Password to use to connect to the specified database.
+
+**Connection Arguments:** A list of arbitrary string key/value pairs as connection arguments. These arguments
+will be passed to the JDBC driver as connection arguments for JDBC drivers that may need additional configurations.