[cherry-pick][6.10][CDAP-21269] Add support for Dataproc Flexible Machine Types in compute profiles

.github/workflows/build-report.yml

@@ -0,0 +1,55 @@
+# Copyright © 2024 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop
+name: Build Unit Tests Report
+
+on:
+  workflow_run:
+    workflows:
+    - Build with unit tests
+    types:
+    - completed
+
+permissions:
+  actions: read    # Allows reading workflow run information
+  statuses: write  # Required if the action updates commit statuses
+  checks: write    # Required if it updates GitHub Checks API
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+
+    steps:
+    # Pinned 1.0.0 version
+    - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a
+
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        run-id: ${{ github.event.workflow_run.id }}
+        path: artifacts/
+
+    - name: Surefire Report
+      # Pinned 3.5.2 version
+      uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
+      if: always()
+      with:
+        report_paths: '**/target/surefire-reports/TEST-*.xml'
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        detailed_summary: true
+        commit: ${{ github.event.workflow_run.head_sha }}
+        check_name: Build Test Report

.github/workflows/build-sonar.yml

@@ -12,41 +12,55 @@
 # This workflow will build a Java project with Maven
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 # Note: Any changes to this workflow would be used only after merging into develop
-name: Build with test coverage and Sonar
+name: Build with test coverage for Sonar
 
 on:
-  workflow_run:
-    workflows:
-      - Trigger build
-    types:
-      - completed
+  push:
+    branches: [ develop, release/** ]
+  pull_request:
+    branches: [ develop, release/** ]
+    types: [opened, synchronize, reopened, labeled]
+
+permissions:
+  checks: write
 
 jobs:
-  build:
+  build-sonar:
     runs-on: k8s-runner-build
 
-    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+    # We allow builds:
+    # 1) When it's a merge into a branch
+    # 2) For PRs that are labeled as build and
+    #  - It's a code change
+    #  - A build label was just added
+    # A bit complex, but prevents builds when other labels are manipulated
+    if: >
+      github.event_name == 'push'
+      || (contains(github.event.pull_request.labels.*.name, 'build')
+         && (github.event.action != 'labeled' || github.event.label.name == 'build')
+         )
 
     steps:
-      # Pinned 1.0.0 version
-      - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a
-      - uses: actions/checkout@v3
+      - name: Set up Spanner Emulator
+        run: |
+          gcloud config configurations create emulator
+          gcloud config set auth/disable_credentials true
+          gcloud config set api_endpoint_overrides/spanner http://localhost:9010/
+          gcloud config configurations activate emulator
+          gcloud emulators spanner start &
+          echo "SPANNER_EMULATOR_HOST=localhost:9010" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.workflow_run.head_sha }}
           submodules: recursive
+
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository11
           key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-${{ github.workflow }}
-      - name: Cache SonarCloud packages
-        uses: actions/cache@v3
-        with:
-          path: ~/.sonar/cache
-          key: ${{ runner.os }}-sonar
-          restore-keys: ${{ runner.os }}-sonar
+
       - name: Build with Maven
         run: >-
           mvn clean verify -fae -T 2 -B -V -Dmaven.test.failure.ignore
@@ -55,52 +69,54 @@ jobs:
           -Pcoverage,templates
           -Dmaven.wagon.http.retryHandler.count=3
           -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+
       - name: Archive build artifacts
-        uses: actions/upload-artifact@v2.2.2
+        uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Build debug files
+          name: reports-${{ github.run_id }}
           path: |
             **/target/rat.txt
             **/target/surefire-reports/*
-      - name: Validate PR
-        id: validate_pr
-        # For whatever reason we get PR 69 for develop branch with both head and base as develop.
-        if: ${{ github.event.workflow_run.pull_requests[0].head.ref != github.event.workflow_run.pull_requests[0].base.ref }}
-        run: |
-          echo ":set-output pr-key=${{ github.event.workflow_run.pull_requests[0].number }}"
-          echo ":set-output pr-branch=${{ github.event.workflow_run.pull_requests[0].head.ref }}"
-          echo ":set-output pr-base=${{ github.event.workflow_run.pull_requests[0].base.ref }}"
+            **/checkstyle-result.xml
+
+      - name: Sonar Report on PR
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
+          JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+        run: >-
+          mvn org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -fae -T 2 -B -V
+          -Dmaven.repo.local=$HOME/.m2/repository11
+          -DcloudBuild
+          -Pcoverage,templates
+          -Dmaven.wagon.http.retryHandler.count=3
+          -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+          -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
+          -Dsonar.pullrequest.branch="$PR_HEAD_REF"
+          -Dsonar.pullrequest.base=${{ github.event.pull_request.base.ref }}
+
       - name: Sonar report
+        if: ${{ github.event_name == 'push' }}
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+          JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
+          BRANCH: ${{ github.event.workflow_run.head_branch }}
         run: >-
           mvn org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -fae -T 2 -B -V
           -Dmaven.repo.local=$HOME/.m2/repository11
           -DcloudBuild
           -Pcoverage,templates
           -Dmaven.wagon.http.retryHandler.count=3
           -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-          -Dsonar.pullrequest.key=${{ steps.validate_pr.outputs.pr-key }}
-          -Dsonar.pullrequest.branch=${{ steps.validate_pr.outputs.pr-branch }}
-          -Dsonar.pullrequest.base=${{ steps.validate_pr.outputs.pr-base }}
-          -Dsonar.branch.name=${{ github.event.workflow_run.head_branch }}
-      - name: Archive build artifacts
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: Build debug files
-          path: |
-            **/target/rat.txt
-            **/target/surefire-reports/*
-      - name: Surefire Report
-        # Pinned 3.5.2 version
-        uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
-        if: always()
+          -Dsonar.branch.name=$BRANCH
+
+      - name: Checkstyle report
+        uses: cdapio/checkstyle-github-action@add_filter
+        if: ${{ github.event_name == 'pull_request' }}
         with:
-          report_paths: '**/target/surefire-reports/TEST-*.xml'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          detailed_summary: true
-          commit: ${{ github.event.workflow_run.head_sha }}
-          check_name: Sonar Build Test Report
+          path: '**/checkstyle-result.xml'
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit: ${{ github.event.pull_request.head.sha }}
+          changed-since: ${{ github.event.pull_request.base.ref }}

.github/workflows/build.yml

@@ -15,56 +15,54 @@
 name: Build with unit tests
 
 on:
-  workflow_run:
-    workflows:
-      - Trigger build
-    types:
-      - completed
+  push:
+    branches: [ develop, release/** ]
+  pull_request:
+    branches: [ develop, release/** ]
+    types: [opened, synchronize, reopened, labeled]
 
 jobs:
   build:
     runs-on: k8s-runner-build
 
-    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
-
+    # We allow builds:
+    # 1) When it's a merge into a branch
+    # 2) For PRs that are labeled as build and
+    #  - It's a code change
+    #  - A build label was just added
+    # A bit complex, but prevents builds when other labels are manipulated
+    if: >
+      github.event_name == 'push'
+      || (contains(github.event.pull_request.labels.*.name, 'build')
+         && (github.event.action != 'labeled' || github.event.label.name == 'build')
+         )
     steps:
-      # Pinned 1.0.0 version
-      - uses: marocchino/action-workflow_run-status@54b6e87d6cb552fc5f36dbe9a722a6048725917a
-      - uses: actions/checkout@v3
+      - name: Set up Spanner Emulator
+        run: |
+          gcloud config configurations create emulator
+          gcloud config set auth/disable_credentials true
+          gcloud config set api_endpoint_overrides/spanner http://localhost:9010/
+          gcloud config configurations activate emulator
+          gcloud emulators spanner start &
+          echo "SPANNER_EMULATOR_HOST=localhost:9010" >> $GITHUB_ENV
+      - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.workflow_run.head_sha }}
           submodules: recursive
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-${{ github.workflow }}
       - name: Build with Maven
-        run: mvn clean test -fae -T 2 -B -V -DcloudBuild -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+        # TODO(CDAP-21120): Enable PreviewDataPipelineTest after fixing the flaky test.
+        run: mvn clean test -fae -T 2 -B -V -DcloudBuild -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dtest='!io.cdap.cdap.datapipeline.preview.PreviewDataPipelineTest,!io.cdap.cdap.test.base.TestFrameworkTestBase,!io.cdap.cdap.datapipeline.ReducibleAggregatorTestBase' -Dsurefire.failIfNoSpecifiedTests=false
       - name: Archive build artifacts
-        uses: actions/upload-artifact@v2.2.2
+        uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Build debug files
+          name: reports-${{ github.run_id }}
           path: |
             **/target/rat.txt
             **/target/surefire-reports/*
-      - name: Surefire Report
-        # Pinned 3.5.2 version
-        uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
-        if: always()
-        with:
-          report_paths: '**/target/surefire-reports/TEST-*.xml'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          detailed_summary: true
-          commit: ${{ github.event.workflow_run.head_sha }}
-          check_name: Test Report
-      - name: Checkstyle report
-        uses: cdapio/checkstyle-github-action@add_filter
-        if: always()
-        with:
-          path: '**/checkstyle-result.xml'
-          commit: ${{ github.event.workflow_run.head_sha }}
-          changed-since: ${{ github.event.workflow_run.pull_requests[0].base.ref }}

.github/workflows/report-build-result.yml

@@ -27,6 +27,8 @@ jobs:
 
     steps:
       - name: Write result to BigQuery
+        env:
+          HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
         run: |
-          echo '{ "repository": "${{ github.repository }}", "build_name": "${{ github.event.workflow.name }}", "branch": "${{ github.event.workflow_run.head_branch }}", "conclusion": "${{ github.event.workflow_run.conclusion }}", "started_at": "${{ github.event.workflow_run.created_at }}", "ended_at": "${{ github.event.workflow_run.updated_at }}", "url": "${{ github.event.workflow_run.html_url }}", "run_id": "${{ github.event.workflow_run.id }}", "run_attempt": "${{ github.event.workflow_run.run_attempt }}" }' | bq insert ${{ vars.BUILD_HISTORY_TABLE }}
+          echo "{ \"repository\": \"${{ github.repository }}\", \"build_name\": \"${{ github.event.workflow.name }}\", \"branch\": \"$HEAD_BRANCH\", \"conclusion\": \"${{ github.event.workflow_run.conclusion }}\", \"started_at\": \"${{ github.event.workflow_run.created_at }}\", \"ended_at\": \"${{ github.event.workflow_run.updated_at }}\", \"url\": \"${{ github.event.workflow_run.html_url }}\", \"run_id\": \"${{ github.event.workflow_run.id }}\", \"run_attempt\": \"${{ github.event.workflow_run.run_attempt }}\" }" | bq insert ${{ vars.BUILD_HISTORY_TABLE }}
 

cdap-api-common/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>io.cdap.cdap</groupId>
     <artifactId>cdap</artifactId>
-    <version>6.10.0-SNAPSHOT</version>
+    <version>6.12.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>cdap-api-common</artifactId>