From 99f44169e00d740ec71c2bc2180655ca86023bf5 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Thu, 2 Jul 2026 09:16:22 -0400 Subject: [PATCH] Bump CI Java version to Java21 --- .../setup-environment-action/action.yml | 2 +- .../arc/images/Dockerfile | 18 +-- .github/workflows/README.md | 2 +- ..._PerformanceTests_xlang_KafkaIO_Python.yml | 6 +- .../beam_PostCommit_Java_Jpms_Dataflow.yml | 2 - .../beam_PostCommit_Java_Jpms_Direct.yml | 2 - ...m_PostCommit_Python_Xlang_Gcp_Dataflow.yml | 7 +- ...eam_PostCommit_Python_Xlang_Gcp_Direct.yml | 6 +- ...am_PostCommit_Python_Xlang_IO_Dataflow.yml | 7 +- ...beam_PostCommit_Python_Xlang_IO_Direct.yml | 7 +- ...eam_PostCommit_TransformService_Direct.yml | 9 +- .../workflows/beam_PostCommit_XVR_Direct.yml | 5 +- .../workflows/beam_PostCommit_XVR_Flink.yml | 5 +- ...am_PostCommit_XVR_GoUsingJava_Dataflow.yml | 5 +- ...ostCommit_XVR_JavaUsingPython_Dataflow.yml | 7 +- ...Commit_XVR_PythonUsingJavaSQL_Dataflow.yml | 7 +- ...ostCommit_XVR_PythonUsingJava_Dataflow.yml | 7 +- .../workflows/beam_PostCommit_XVR_Spark3.yml | 5 +- ...eam_PreCommit_Java_Cassandra_IO_Direct.yml | 2 + ...beam_PreCommit_Java_Debezium_IO_Direct.yml | 12 +- ...eCommit_Java_Examples_Dataflow_Java11.yml} | 18 +-- .../beam_PreCommit_Java_Flink_Versions.yml | 2 +- ...beam_PreCommit_Java_HCatalog_IO_Direct.yml | 2 + .../beam_PreCommit_Java_IOs_Direct.yml | 13 +++ .../beam_PreCommit_Java_Spark_Versions.yml | 4 +- .../beam_PreCommit_Kotlin_Examples.yml | 8 +- .../beam_Publish_Beam_SDK_Snapshots.yml | 5 +- build.gradle.kts | 7 +- examples/java/build.gradle | 7 ++ examples/kotlin/build.gradle | 7 ++ .../fnexecution/ServerFactoryTest.java | 19 +++- runners/spark/spark_runner.gradle | 5 +- .../org/apache/beam/sdk/util/ApiSurface.java | 7 +- .../options/PipelineOptionsFactoryTest.java | 56 +++++++--- .../JavaFilterTransformProviderTest.java | 8 +- .../sdk/util/common/ReflectHelpersTest.java | 15 ++- sdks/java/extensions/arrow/build.gradle | 7 +- sdks/java/extensions/avro/build.gradle | 7 ++ sdks/java/io/cassandra/build.gradle | 21 ++++ sdks/java/io/cdap/build.gradle | 5 + .../beam/sdk/io/csv/CsvIOParseTest.java | 6 +- sdks/java/io/influxdb/build.gradle | 10 +- .../beam/sdk/io/influxdb/InfluxDbIO.java | 22 ++++ .../beam/sdk/io/influxdb/InfluxDbIOTest.java | 104 +++++++++++------- sdks/java/io/kafka/build.gradle | 1 + .../SingleStoreDefaultRowMapper.java | 3 +- 46 files changed, 300 insertions(+), 192 deletions(-) rename .github/workflows/{beam_PreCommit_Java_Examples_Dataflow_Java21.yml => beam_PreCommit_Java_Examples_Dataflow_Java11.yml} (93%) diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index c82ad087fe46..c88b4a6fa8fc 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -74,7 +74,7 @@ runs: uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }} + java-version: ${{ inputs.java-version == 'default' && '21' || inputs.java-version }} - name: Setup Gradle uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 with: diff --git a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile index bc9494124f11..a2bae4dac74b 100644 --- a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile +++ b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile @@ -26,7 +26,7 @@ RUN docker buildx install && docker buildx version USER root # Native build toolchain for Python C extensions. RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends build-essential time rsync openssh-client && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends build-essential time rsync openssh-client wget && \ rm -rf /var/lib/apt/lists/* #Install Node RUN curl -OL https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz && \ @@ -41,16 +41,16 @@ RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \ rm go${go_version}.linux-amd64.tar.gz ENV PATH="${PATH}:/usr/local/go/bin" #Install Java -RUN curl -OL https://cdn.azul.com/zulu/bin/zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \ - tar -C /usr/local -xzf zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \ - rm zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \ - mv /usr/local/zulu11.74.15-ca-jdk11.0.24-linux_x64 /usr/local/java +RUN curl -OL https://cdn.azul.com/zulu/bin/zulu21.50.19-ca-jdk21.0.11-linux_x64.tar.gz && \ + tar -C /usr/local -xzf zulu21.50.19-ca-jdk21.0.11-linux_x64.tar.gz && \ + rm zulu21.50.19-ca-jdk21.0.11-linux_x64.tar.gz && \ + mv /usr/local/zulu21.50.19-ca-jdk21.0.11-linux_x64 /usr/local/java ENV PATH="${PATH}:/usr/local/java/bin" #Install Gradle -RUN curl -OL https://services.gradle.org/distributions/gradle-8.4-bin.zip && \ - unzip gradle-8.4-bin.zip && \ - rm gradle-8.4-bin.zip && \ - mv gradle-8.4 /usr/local/gradle +RUN curl -OL https://services.gradle.org/distributions/gradle-8.14.3-bin.zip && \ + unzip gradle-8.14.3-bin.zip && \ + rm gradle-8.14.3-bin.zip && \ + mv gradle-8.14.3 /usr/local/gradle ENV PATH="${PATH}:/usr/local/gradle/bin" #Install GitHub CLI RUN curl -OL https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz && \ diff --git a/.github/workflows/README.md b/.github/workflows/README.md index f6b1a440e175..667f5e3fef2b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -241,7 +241,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour | [ PreCommit Java Debezium IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml) | N/A |`Run Java_Debezium_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java ElasticSearch IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml) | N/A |`Run Java_ElasticSearch_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java_Examples_Dataflow PreCommit`| [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) | -| [ PreCommit Java Examples Dataflow Java21 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml) | N/A | `Run Java_Examples_Dataflow_Java21 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml?query=event%3Aschedule) | +| [ PreCommit Java Examples Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) | N/A | `Run Java_Examples_Dataflow_Java11 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml?query=event%3Aschedule) | | [ PreCommit Java File-schema-transform IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml) | N/A |`Run Java_File-schema-transform_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Flink Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml) | N/A |`Run Java_Flink_Versions PreCommit`| [![.github/workflows/beam_PreCommit_Java_Flink_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml?query=event%3Aschedule) | | [ PreCommit Java GCP IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml) | N/A |`Run Java_GCP_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml?query=event%3Aschedule) | diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml index 94c556e4c475..994fccbc46b8 100644 --- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -73,9 +73,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: default - name: Set k8s access uses: ./.github/actions/setup-k8s-access @@ -182,8 +180,6 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:io:expansion-service:shadowJar - arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 # The env variable is created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow.yml index b3b1e54ae9ea..17f7cd2d5881 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow.yml @@ -72,8 +72,6 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action - with: - java-version: 11 - name: run PostCommit Java Jpms Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct.yml index dd627b7bd235..cfe360f9d141 100644 --- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct.yml +++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct.yml @@ -72,8 +72,6 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action - with: - java-version: 11 - name: run PostCommit Java Jpms Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index 3202c29a89ab..b7f6898353fe 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 3.14 @@ -86,8 +84,7 @@ jobs: gradle-command: :sdks:python:test-suites:dataflow:gcpCrossLanguagePostCommit arguments: | -PpythonVersion=3.14 \ - -PuseWheelDistribution \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PuseWheelDistribution - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index 1862512a83db..60231abf0a0c 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 3.14 @@ -88,8 +86,6 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:gcpCrossLanguagePostCommit - arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index 036ba9e25f39..8b5c9c077869 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -73,9 +73,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 3.14 @@ -86,8 +84,7 @@ jobs: arguments: | -PpythonVersion=3.14 \ -PuseWheelDistribution \ - -PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml index d091a6bd6060..dca175505ca5 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml @@ -73,9 +73,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 3.14 @@ -84,8 +82,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit arguments: | - -PuseWheelDistribution \ - -Pjava17Home=$JAVA_HOME_17_X64 + -PuseWheelDistribution - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index 0da5ed93b202..a659b0b5682d 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 ${{ matrix.python_version }} @@ -85,10 +83,7 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava arguments: | - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - -PpythonVersion=${{ matrix.python_version }} \ + -PpythonVersion=${{ matrix.python_version }} - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index c8e03769bd04..ca370f67f236 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: ${{ matrix.python_version }} - name: run PostCommit XVR Direct script env: @@ -86,7 +84,6 @@ jobs: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ - -Pjava17Home=$JAVA_HOME_17_X64 \ -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml index 273cf5c994cc..e12699dfd1e5 100644 --- a/.github/workflows/beam_PostCommit_XVR_Flink.yml +++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml @@ -75,9 +75,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: ${{ matrix.python_version }} - name: run PostCommit XVR Flink script env: @@ -86,7 +84,6 @@ jobs: with: gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ -PpythonVersion=${{ matrix.python_version }} \ -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index d9f58f649e0a..0ca67325ea77 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -76,9 +76,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: default - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 @@ -89,7 +87,6 @@ jobs: with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ -PuseDockerBuildx - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml index 68fa1e646311..1de02c55a411 100644 --- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 ${{ matrix.python_version }} @@ -85,8 +83,7 @@ jobs: with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ - -PpythonVersion=${{ matrix.python_version }} \ + -PpythonVersion=${{ matrix.python_version }} - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml index c180e43cc8cc..7cab71f34892 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml @@ -73,17 +73,14 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: 3.14 - name: run PostCommit XVR PythonUsingJavaSQL Dataflow script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingSql arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ - -PpythonVersion=3.14 \ + -PpythonVersion=3.14 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml index 1c7bc806c035..4fd9ebc7a121 100644 --- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: | 3.10 ${{ matrix.python_version }} @@ -85,8 +83,7 @@ jobs: with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ - -PpythonVersion=${{ matrix.python_version }} \ + -PpythonVersion=${{ matrix.python_version }} - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml index e303e80a7ea3..1d9adbad868c 100644 --- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml +++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml @@ -74,9 +74,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default python-version: ${{ matrix.python_version }} - name: run PostCommit XVR Spark3 script env: @@ -85,7 +83,6 @@ jobs: with: gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ -PpythonVersion=${{ matrix.python_version }} \ -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results diff --git a/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml index c81e712e20af..0445d81e7706 100644 --- a/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml @@ -85,6 +85,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 11 - name: run Cassandra IO build script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml index 7b839616efac..f398d41a089e 100644 --- a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml @@ -86,18 +86,14 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default - name: run Debezium IO build task uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:io:debezium:build arguments: | -PdisableSpotlessCheck=true \ - -PdisableCheckStyle=true \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PdisableCheckStyle=true - name: run Debezium IO additional tasks uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -106,9 +102,7 @@ jobs: :sdks:java:io:debezium:integrationTest \ arguments: | -PdisableSpotlessCheck=true \ - -PdisableCheckStyle=true \ - -PtestJavaVersion=17 \ - -Pjava17Home=$JAVA_HOME_17_X64 \ + -PdisableCheckStyle=true - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml similarity index 93% rename from .github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml rename to .github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml index 2c4b607a1a5e..c5124c5c3e01 100644 --- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml +++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: PreCommit Java Examples Dataflow Java21 +name: PreCommit Java Examples Dataflow Java11 on: push: @@ -26,7 +26,7 @@ on: - 'examples/java/**' - 'examples/kotlin/**' - 'release/**' - - '.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml' + - '.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml' pull_request_target: branches: ['master', 'release-*'] paths: @@ -37,7 +37,7 @@ on: - 'examples/kotlin/**' - 'release/**' - 'release/trigger_all_tests.json' - - '.github/trigger_files/beam_PreCommit_Java_Examples_Dataflow_Java21.json' + - '.github/trigger_files/beam_PreCommit_Java_Examples_Dataflow_Java11.json' issue_comment: types: [created] schedule: @@ -71,19 +71,19 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: - beam_PreCommit_Java_Examples_Dataflow_Java21: + beam_PreCommit_Java_Examples_Dataflow_Java11: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: matrix: - job_name: ["beam_PreCommit_Java_Examples_Dataflow_Java21"] - job_phrase: ["Run Java_Examples_Dataflow_Java21 PreCommit"] + job_name: ["beam_PreCommit_Java_Examples_Dataflow_Java11"] + job_phrase: ["Run Java_Examples_Dataflow_Java11 PreCommit"] timeout-minutes: 75 if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || github.event_name == 'workflow_dispatch' || - github.event.comment.body == 'Run Java_Examples_Dataflow_Java21 PreCommit' + github.event.comment.body == 'Run Java_Examples_Dataflow_Java11 PreCommit' runs-on: [self-hosted, ubuntu-24.04, main] steps: - uses: actions/checkout@v7 @@ -116,9 +116,9 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=21 \ + -PtestJavaVersion=11 \ -PskipCheckerFramework \ - -Pjava21Home=$JAVA_HOME_21_X64 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ -Pdocker-pull-licenses \ max-workers: 12 - name: Archive JUnit Test Results diff --git a/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml b/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml index 7308963dc6d3..c816d01a9dcc 100644 --- a/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml +++ b/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml @@ -84,7 +84,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: default + java-version: 11 python-version: default - name: run Java Flink Versions PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml index f8561eb7c856..ead1ade833a2 100644 --- a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml @@ -87,6 +87,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 11 - name: run HCatalog IO build script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml index a3af909a9ad6..b5ea450dba1e 100644 --- a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml @@ -88,6 +88,10 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: | + 11 + 21 - name: run Java IOs PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -96,6 +100,15 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -Dfile.encoding=UTF-8 \ + - name: Run IOs need Java11 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:cassandra:build :sdks:java:io:hbase:build :sdks:java:io:hcatalog:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -PtestJavaVersion=11 \ + -Pjava11Home=$JAVA_HOME_11_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml b/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml index 9821419da139..ad2c398a04ff 100644 --- a/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml +++ b/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml @@ -88,9 +88,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: 11 - name: run sparkVersionsTest script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PreCommit_Kotlin_Examples.yml b/.github/workflows/beam_PreCommit_Kotlin_Examples.yml index 35c3786c00e3..3633db8d86d8 100644 --- a/.github/workflows/beam_PreCommit_Kotlin_Examples.yml +++ b/.github/workflows/beam_PreCommit_Kotlin_Examples.yml @@ -102,4 +102,10 @@ jobs: - name: run Kotlin Examples script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :examples:kotlin:preCommit \ No newline at end of file + gradle-command: :examples:kotlin:preCommit + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v7 + if: ${{ !success() }} + with: + name: JUnit Test Results + path: "**/build/reports/tests/" diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 8ffd35c06baa..c537feab45f8 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -109,9 +109,7 @@ jobs: if: ${{ startsWith(matrix.container_task, 'java') }} uses: ./.github/actions/setup-environment-action with: - java-version: | - 17 - 11 + java-version: default - name: Setup Python environment if: ${{ startsWith(matrix.container_task, 'python') }} uses: ./.github/actions/setup-environment-action @@ -122,7 +120,6 @@ jobs: with: gradle-command: :sdks:${{ matrix.container_task }} arguments: | - -Pjava17Home=$JAVA_HOME_17_X64 \ -Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \ -Pdocker-tag-list=${{ github.sha }},${BEAM_VERSION}${LATEST_TAG} \ -Pcontainer-architecture-list=arm64,amd64 \ diff --git a/build.gradle.kts b/build.gradle.kts index 4af8fa3f1ab4..ba2f7aaed67e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -339,7 +339,8 @@ tasks.register("javaPreCommit") { // a precommit task build multiple IOs (except those splitting into single jobs) tasks.register("javaioPreCommit") { dependsOn(":sdks:java:io:amqp:build") - dependsOn(":sdks:java:io:cassandra:build") + // CassandraIO, HBaseIO and HCatalogIO do not support Java17+, test ran separately + // dependsOn(":sdks:java:io:cassandra:build") dependsOn(":sdks:java:io:csv:build") dependsOn(":sdks:java:io:cdap:build") dependsOn(":sdks:java:io:clickhouse:build") @@ -349,8 +350,8 @@ tasks.register("javaioPreCommit") { dependsOn(":sdks:java:io:elasticsearch:build") dependsOn(":sdks:java:io:file-schema-transform:build") dependsOn(":sdks:java:io:google-ads:build") - dependsOn(":sdks:java:io:hbase:build") - dependsOn(":sdks:java:io:hcatalog:build") + // dependsOn(":sdks:java:io:hbase:build") + // dependsOn(":sdks:java:io:hcatalog:build") dependsOn(":sdks:java:io:influxdb:build") dependsOn(":sdks:java:io:jdbc:build") dependsOn(":sdks:java:io:jms:build") diff --git a/examples/java/build.gradle b/examples/java/build.gradle index cf168293968f..34d884b97780 100644 --- a/examples/java/build.gradle +++ b/examples/java/build.gradle @@ -140,6 +140,13 @@ for (String runner : preCommitRunners) { forkEvery 1 maxParallelForks 4 systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions) + def testJavaVer = project.findProperty('testJavaVersion') ? (project.property('testJavaVersion') as int) : JavaVersion.current().majorVersion.toInteger() + if ("sparkRunner".equals(runner) && testJavaVer >= 17) { + jvmArgs "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + } } } diff --git a/examples/kotlin/build.gradle b/examples/kotlin/build.gradle index 829aefd44745..60ddc1c52d7f 100644 --- a/examples/kotlin/build.gradle +++ b/examples/kotlin/build.gradle @@ -113,6 +113,13 @@ for (String runner : preCommitRunners) { forkEvery 1 maxParallelForks 4 systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions) + def testJavaVer = project.findProperty('testJavaVersion') ? (project.property('testJavaVersion') as int) : JavaVersion.current().majorVersion.toInteger() + if ("sparkRunner".equals(runner) && testJavaVer >= 17) { + jvmArgs "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + } } } diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java index f89d0e00bd3b..5d5bb6657ebe 100644 --- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java +++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java @@ -31,6 +31,7 @@ import java.net.Inet4Address; import java.net.InetAddress; +import java.net.ServerSocket; import java.util.ArrayList; import java.util.Collection; import java.util.concurrent.CountDownLatch; @@ -99,9 +100,14 @@ public void usesUrlFactory() throws Exception { @Test public void defaultServerWithPortSupplier() throws Exception { + int port; + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + int staticPort = port; Endpoints.ApiServiceDescriptor apiServiceDescriptor = runTestUsing( - ServerFactory.createWithPortSupplier(() -> 65535), + ServerFactory.createWithPortSupplier(() -> staticPort), ManagedChannelFactory.createDefault()); HostAndPort hostAndPort = HostAndPort.fromString(apiServiceDescriptor.getUrl()); assertThat( @@ -109,21 +115,26 @@ public void defaultServerWithPortSupplier() throws Exception { anyOf( equalTo(InetAddress.getLoopbackAddress().getHostName()), equalTo(InetAddress.getLoopbackAddress().getHostAddress()))); - assertThat(hostAndPort.getPort(), is(65535)); + assertThat(hostAndPort.getPort(), is(staticPort)); } @Test public void urlFactoryWithPortSupplier() throws Exception { + int port; + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + int staticPort = port; ServerFactory serverFactory = ServerFactory.createWithUrlFactoryAndPortSupplier( - (host, port) -> "foo" + ":" + port, () -> 65535); + (host, p) -> "foo" + ":" + p, () -> staticPort); CallStreamObserver observer = TestStreams.withOnNext((Elements unused) -> {}).withOnCompleted(() -> {}).build(); TestDataService service = new TestDataService(observer); ApiServiceDescriptor.Builder descriptorBuilder = ApiServiceDescriptor.newBuilder(); grpcCleanupRule.register( serverFactory.allocateAddressAndCreate(ImmutableList.of(service), descriptorBuilder)); - assertThat(descriptorBuilder.getUrl(), is("foo:65535")); + assertThat(descriptorBuilder.getUrl(), is("foo:" + staticPort)); } @Test diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index 1ae35b7449a4..1da044ab7b52 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -77,9 +77,8 @@ def sparkTestProperties(overrides = [:]) { def sparkTestJvmArgs() { - // run tests with Java 17 using -PtestJavaVersion=17 -Pjava17Home=??? - if (project.hasProperty('testJavaVersion') && - project.getProperty('testJavaVersion') in ['17', '21']) { + def testJavaVer = project.findProperty('testJavaVersion') ? (project.property('testJavaVersion') as int) : JavaVersion.current().majorVersion.toInteger() + if (testJavaVer >= 17) { return [ "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", // add-opens below required for Kryo FieldSerializer / SparkRunnerKryoRegistratorTest diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java index d72258cd9ac9..72a158f26132 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ApiSurface.java @@ -519,11 +519,16 @@ private boolean pruned(Type type) { return pruned(TypeToken.of(type).getRawType()); } + private static final ImmutableSet EXCLUDED_NAMES = + ImmutableSet.of( + "jdk.internal.HotSpotIntrinsicCandidate", + "jdk.internal.vm.annotation.IntrinsicCandidate"); + /** Whether a class and all that it references should be pruned from the graph. */ private boolean pruned(Class clazz) { return clazz.isPrimitive() || clazz.isArray() - || clazz.getCanonicalName().equals("jdk.internal.HotSpotIntrinsicCandidate") + || EXCLUDED_NAMES.contains(clazz.getCanonicalName()) || getPrunedPattern().matcher(clazz.getName()).matches(); } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java index 2755ade271dd..44de84996211 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/PipelineOptionsFactoryTest.java @@ -668,12 +668,16 @@ public void testGettersAnnotatedWithInconsistentDefault() throws Exception { // Make sure the error message gives actionable details about what // annotations were contradictory. - // Note that the quotes in the unparsed string are present in Java 11 but absent in Java 8 + // Note that the quotes in the unparsed string are present in Java 11 but absent in Java 8, + // and "value=" element name prefix is absent in Java 21+ expectedException.expectMessage( anyOf( containsString("Default.String(value=\"abc\")"), - containsString("Default.String(value=abc)"))); - expectedException.expectMessage("Default.Integer(value=1"); + containsString("Default.String(value=abc)"), + containsString("Default.String(\"abc\")"), + containsString("Default.String(abc)"))); + expectedException.expectMessage( + anyOf(containsString("Default.Integer(value=1"), containsString("Default.Integer(1"))); // When we attempt to convert, we should error at this moment. options.as(GetterWithInconsistentDefaultType.class); @@ -686,11 +690,19 @@ public void testGettersAnnotatedWithInconsistentDefaultValue() throws Exception expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "Property [object] is marked with contradictory annotations. Found [" - + "[Default.Integer(value=1) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" - + "$GetterWithDefault#getObject()], " - + "[Default.Integer(value=0) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" - + "$GetterWithInconsistentDefaultValue#getObject()]]."); + anyOf( + containsString( + "Property [object] is marked with contradictory annotations. Found [" + + "[Default.Integer(value=1) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithDefault#getObject()], " + + "[Default.Integer(value=0) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithInconsistentDefaultValue#getObject()]]."), + containsString( + "Property [object] is marked with contradictory annotations. Found [" + + "[Default.Integer(1) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithDefault#getObject()], " + + "[Default.Integer(0) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithInconsistentDefaultValue#getObject()]]."))); // When we attempt to convert, we should error at this moment. options.as(GetterWithInconsistentDefaultValue.class); @@ -703,11 +715,19 @@ public void testGettersAnnotatedWithInconsistentJsonIgnoreValue() throws Excepti expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "Property [object] is marked with contradictory annotations. Found [" - + "[JsonIgnore(value=false) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" - + "$GetterWithInconsistentJsonIgnoreValue#getObject()], " - + "[JsonIgnore(value=true) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" - + "$GetterWithJsonIgnore#getObject()]]."); + anyOf( + containsString( + "Property [object] is marked with contradictory annotations. Found [" + + "[JsonIgnore(value=false) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithInconsistentJsonIgnoreValue#getObject()], " + + "[JsonIgnore(value=true) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithJsonIgnore#getObject()]]."), + containsString( + "Property [object] is marked with contradictory annotations. Found [" + + "[JsonIgnore(false) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithInconsistentJsonIgnoreValue#getObject()], " + + "[JsonIgnore(true) on org.apache.beam.sdk.options.PipelineOptionsFactoryTest" + + "$GetterWithJsonIgnore#getObject()]]."))); // When we attempt to convert, we should error at this moment. options.as(GetterWithInconsistentJsonIgnoreValue.class); @@ -731,12 +751,16 @@ public void testGettersWithMultipleDefaults() throws Exception { // Make sure the error message gives actionable details about what annotations were // contradictory. - // Note that the quotes in the unparsed string are present in Java 11 but absent in Java 8 + // Note that the quotes in the unparsed string are present in Java 11 but absent in Java 8, + // and "value=" element name prefix is absent in Java 21+ expectedException.expectMessage( anyOf( containsString("Default.String(value=\"abc\")"), - containsString("Default.String(value=abc)"))); - expectedException.expectMessage("Default.Integer(value=0)"); + containsString("Default.String(value=abc)"), + containsString("Default.String(\"abc\")"), + containsString("Default.String(abc)"))); + expectedException.expectMessage( + anyOf(containsString("Default.Integer(value=0)"), containsString("Default.Integer(0)"))); // When we attempt to create, we should error at this moment. PipelineOptionsFactory.as(GettersWithMultipleDefault.class); diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/providers/JavaFilterTransformProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/providers/JavaFilterTransformProviderTest.java index b269b12f1554..733a84e75258 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/providers/JavaFilterTransformProviderTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/providers/JavaFilterTransformProviderTest.java @@ -107,12 +107,18 @@ public void testErrorHandling() { PCollection errors = result.get("errors"); Schema errorSchema = errors.getSchema(); + // StringIndexOutOfBoundsException message format changed in Java 17+ + int javaVersion = + Integer.parseInt(System.getProperty("java.specification.version").replace("1.", "")); + String expectedErrorMessage = + javaVersion >= 17 ? "Index 7 out of bounds for length 5" : "String index out of range: 7"; + PAssert.that(errors) .containsInAnyOrder( Row.withSchema(errorSchema) .withFieldValue( "failed_row", Row.withSchema(inputSchema).addValues("short").build()) - .withFieldValue("error_message", "String index out of range: 7") + .withFieldValue("error_message", expectedErrorMessage) .build()); pipeline.run(); } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/common/ReflectHelpersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/common/ReflectHelpersTest.java index 5f20c4576fef..e999169abb7d 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/common/ReflectHelpersTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/common/ReflectHelpersTest.java @@ -137,15 +137,20 @@ public void testFormatAnnotationDefault() throws Exception { assertThat( ReflectHelpers.formatAnnotation(Options.class.getMethod("getString").getAnnotations()[0]), anyOf( - equalTo("Default.String(value=package.OuterClass$InnerClass#method())"), - equalTo("Default.String(value=\"package.OuterClass$InnerClass#method()\")"))); + equalTo("Default.String(value=package.OuterClass$InnerClass#method())"), // Java8 + equalTo("Default.String(value=\"package.OuterClass$InnerClass#method()\")"), // Java11 + equalTo("Default.String(\"package.OuterClass$InnerClass#method()\")") // Javaa21 + )); } @Test public void testFormatAnnotationJsonIgnore() throws Exception { - assertEquals( - "JsonIgnore(value=true)", - ReflectHelpers.formatAnnotation(Options.class.getMethod("getObject").getAnnotations()[0])); + assertThat( + ReflectHelpers.formatAnnotation(Options.class.getMethod("getObject").getAnnotations()[0]), + anyOf( + equalTo("JsonIgnore(value=true)"), // Java11 + equalTo("JsonIgnore(true)") // Java21 + )); } @Test diff --git a/sdks/java/extensions/arrow/build.gradle b/sdks/java/extensions/arrow/build.gradle index 4fc663ddbc28..4e9f238adf45 100644 --- a/sdks/java/extensions/arrow/build.gradle +++ b/sdks/java/extensions/arrow/build.gradle @@ -31,4 +31,9 @@ dependencies { testImplementation library.java.junit testImplementation library.java.hamcrest testRuntimeOnly library.java.slf4j_simple -} \ No newline at end of file +} + +test { + // Open java.nio for Apache Arrow direct memory access under Java 17+ + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' +} diff --git a/sdks/java/extensions/avro/build.gradle b/sdks/java/extensions/avro/build.gradle index 718f07c03660..6c21dbb7c08b 100644 --- a/sdks/java/extensions/avro/build.gradle +++ b/sdks/java/extensions/avro/build.gradle @@ -58,6 +58,13 @@ test { } } +tasks.withType(Test).configureEach { + // Open modules for Kryo reflection and memory access under Java 17+ + jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' +} + dependencies { implementation library.java.byte_buddy implementation library.java.vendored_guava_32_1_2_jre diff --git a/sdks/java/io/cassandra/build.gradle b/sdks/java/io/cassandra/build.gradle index 5256ac1f2148..43ac8bc8bf85 100644 --- a/sdks/java/io/cassandra/build.gradle +++ b/sdks/java/io/cassandra/build.gradle @@ -69,3 +69,24 @@ configurations.all (Configuration it) -> { force 'com.google.guava:guava:25.1-jre' } } + +def testJavaVer = project.findProperty('testJavaVersion') ?: JavaVersion.current().getMajorVersion() +def isJava17OrHigher = testJavaVer.toInteger() >= 17 + +test { + if (isJava17OrHigher) { + systemProperty 'java.security.manager', 'allow' + } + // Open and export modules for embedded Cassandra under Java 17+ + jvmArgs '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED', + '--add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED', + '--add-opens=java.base/java.io=ALL-UNNAMED', + '--add-opens=java.base/java.net=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' +} diff --git a/sdks/java/io/cdap/build.gradle b/sdks/java/io/cdap/build.gradle index 6ccb10a5c891..e0af1717e162 100644 --- a/sdks/java/io/cdap/build.gradle +++ b/sdks/java/io/cdap/build.gradle @@ -85,3 +85,8 @@ dependencies { testImplementation project(path: ":runners:direct-java", configuration: "shadow") testImplementation project(path: ":sdks:java:io:common") } + +test { + // Open java.lang for Gson reflection on StackTraceElement under Java 17+ + jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED' +} diff --git a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseTest.java b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseTest.java index a517cef3d51f..6d04ba4fafdf 100644 --- a/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseTest.java +++ b/sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseTest.java @@ -238,6 +238,7 @@ public void givenMultipleCustomParsingLambdas_parsesPOJOs() { "instant", input -> DateTimeFormat.forPattern("yyyy-MM-dd@HH:mm:ss") + .withZoneUTC() .parseDateTime(input) .toInstant()) .withCustomRecordParsing("instantList", instantListParsingLambda()); @@ -308,7 +309,10 @@ private static SerializableFunction> instantListParsingLam List output = new ArrayList<>(); for (String cell : cells) { output.add( - DateTimeFormat.forPattern("HH-mm-ss-yyyy-MM-dd").parseDateTime(cell).toInstant()); + DateTimeFormat.forPattern("HH-mm-ss-yyyy-MM-dd") + .withZoneUTC() + .parseDateTime(cell) + .toInstant()); } return output; }; diff --git a/sdks/java/io/influxdb/build.gradle b/sdks/java/io/influxdb/build.gradle index dc6c85059cdb..888c54e71ee1 100644 --- a/sdks/java/io/influxdb/build.gradle +++ b/sdks/java/io/influxdb/build.gradle @@ -10,7 +10,7 @@ * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an AS IS BASIS, +* distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. @@ -30,16 +30,8 @@ dependencies { implementation "com.squareup.okhttp3:okhttp:4.6.0" implementation library.java.vendored_guava_32_1_2_jre testImplementation library.java.junit - // TODO(https://github.com/apache/beam/issues/34056) remove powermock once removed static mocks. - testImplementation "org.powermock:powermock-module-junit4:2.0.9" - testImplementation "org.powermock:powermock-api-mockito2:2.0.9" testImplementation library.java.mockito_core testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") testImplementation project(path: ":sdks:java:io:common") } - -configurations.testRuntimeClasspath { - // Pin mockito-core 3 for now due to powermock not support mockito 4 - resolutionStrategy.force "org.mockito:mockito-core:3.7.7" -} diff --git a/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java b/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java index a2aef8766921..71d9a3e721d0 100644 --- a/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java +++ b/sdks/java/io/influxdb/src/main/java/org/apache/beam/sdk/io/influxdb/InfluxDbIO.java @@ -45,6 +45,7 @@ import org.apache.beam.sdk.values.PBegin; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; import org.checkerframework.checker.nullness.qual.Nullable; import org.influxdb.InfluxDB; import org.influxdb.InfluxDB.ConsistencyLevel; @@ -565,12 +566,20 @@ private void flush() { @AutoValue public abstract static class DataSourceConfiguration implements Serializable { + @VisibleForTesting + @FunctionalInterface + public interface ConnectionFactory extends Serializable { + InfluxDB create(DataSourceConfiguration configuration, boolean disableCertificateValidation); + } + abstract ValueProvider url(); abstract ValueProvider userName(); abstract ValueProvider password(); + abstract @Nullable ConnectionFactory connectionFactory(); + abstract Builder builder(); @AutoValue.Builder @@ -582,6 +591,8 @@ abstract static class Builder { abstract Builder setPassword(ValueProvider password); + abstract Builder setConnectionFactory(@Nullable ConnectionFactory connectionFactory); + abstract DataSourceConfiguration build(); } @@ -594,6 +605,14 @@ public static DataSourceConfiguration create( .build(); } + /** + * For testing only. Note that userName and password are ignored when connectionFactory is set. + */ + @VisibleForTesting + DataSourceConfiguration withConnectionFactory(ConnectionFactory connectionFactory) { + return builder().setConnectionFactory(connectionFactory).build(); + } + void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("url", url())); builder.add(DisplayData.item("userName", userName())); @@ -642,6 +661,9 @@ private static boolean checkDatabase( public static InfluxDB getConnection( DataSourceConfiguration configuration, boolean disableCertificateValidation) { + if (configuration.connectionFactory() != null) { + return configuration.connectionFactory().create(configuration, disableCertificateValidation); + } if (!disableCertificateValidation) { return InfluxDBFactory.connect( configuration.url().get(), diff --git a/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOTest.java b/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOTest.java index 78579b82c93d..a54eec72e41c 100644 --- a/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOTest.java +++ b/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOTest.java @@ -18,8 +18,6 @@ package org.apache.beam.sdk.io.influxdb; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.powermock.api.mockito.PowerMockito.doReturn; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -28,10 +26,11 @@ import java.util.List; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; -import okhttp3.OkHttpClient; +import java.util.concurrent.atomic.AtomicReference; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.PipelineResult.State; import org.apache.beam.sdk.io.influxdb.InfluxDbIO.DataSourceConfiguration; +import org.apache.beam.sdk.io.influxdb.InfluxDbIO.DataSourceConfiguration.ConnectionFactory; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; @@ -39,43 +38,71 @@ import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.influxdb.InfluxDB; -import org.influxdb.InfluxDBFactory; import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; import org.influxdb.dto.QueryResult.Series; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.junit.runner.RunWith; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; - -@RunWith(PowerMockRunner.class) -@PrepareForTest({ - org.influxdb.InfluxDBFactory.class, - org.apache.beam.sdk.io.influxdb.InfluxDbIO.InfluxDBSource.class -}) + public class InfluxDbIOTest { @Rule public final TestPipeline pipeline = TestPipeline.create(); + // ACTIVE_MOCK provides access to the mock InfluxDB instance on worker threads after + // TestConnectionFactory is deserialized, avoiding non-serializable Mockito state. + private static final AtomicReference ACTIVE_MOCK = new AtomicReference<>(); + + private InfluxDB influxDb; + @Before public void setupTest() { - PowerMockito.mockStatic(InfluxDBFactory.class); + influxDb = Mockito.mock(InfluxDB.class); + ACTIVE_MOCK.set(influxDb); + } + + @After + public void tearDown() { + ACTIVE_MOCK.set(null); + } + + private DataSourceConfiguration getDataSourceConfiguration( + String influxHost, String userName, String password) { + return DataSourceConfiguration.create( + StaticValueProvider.of(influxHost), + StaticValueProvider.of(userName), + StaticValueProvider.of(password)) + .withConnectionFactory(new TestConnectionFactory(influxDb, influxHost, userName, password)); + } + + private static class TestConnectionFactory implements ConnectionFactory { + private final String expectedUrl; + private final String expectedUser; + private final String expectedPassword; + private final transient InfluxDB mockInstance; + + TestConnectionFactory( + InfluxDB mockInstance, String expectedUrl, String expectedUser, String expectedPassword) { + this.mockInstance = mockInstance; + this.expectedUrl = expectedUrl; + this.expectedUser = expectedUser; + this.expectedPassword = expectedPassword; + } + + @Override + public InfluxDB create(DataSourceConfiguration config, boolean disableCert) { + Assert.assertEquals(expectedUrl, config.url().get()); + Assert.assertEquals(expectedUser, config.userName().get()); + Assert.assertEquals(expectedPassword, config.password().get()); + return mockInstance != null ? mockInstance : ACTIVE_MOCK.get(); + } } @Test public void validateWriteTest() { - InfluxDB influxDb = Mockito.mock(InfluxDB.class); - PowerMockito.when( - InfluxDBFactory.connect( - anyString(), anyString(), anyString(), any(OkHttpClient.Builder.class))) - .thenReturn(influxDb); - PowerMockito.when(InfluxDBFactory.connect(anyString(), anyString(), anyString())) - .thenReturn(influxDb); String influxHost = "http://localhost"; String userName = "admin"; String password = "admin"; @@ -84,7 +111,9 @@ public void validateWriteTest() { Mockito.doAnswer(invocation -> countInvocation.getAndIncrement()) .when(influxDb) .write(any(List.class)); - doReturn(getDatabase(influxDatabaseName)).when(influxDb).query(new Query("SHOW DATABASES")); + Mockito.doReturn(getDatabase(influxDatabaseName)) + .when(influxDb) + .query(new Query("SHOW DATABASES")); final int numOfElementsToWrite = 1000; pipeline .apply("Generate data", Create.of(GenerateData.getMetric("test_m", numOfElementsToWrite))) @@ -92,10 +121,7 @@ public void validateWriteTest() { "Write data to InfluxDB", InfluxDbIO.write() .withDataSourceConfiguration( - DataSourceConfiguration.create( - StaticValueProvider.of(influxHost), - StaticValueProvider.of(userName), - StaticValueProvider.of(password))) + getDataSourceConfiguration(influxHost, userName, password)) .withDatabase(influxDatabaseName)); PipelineResult result = pipeline.run(); Assert.assertEquals(State.DONE, result.waitUntilFinish()); @@ -108,20 +134,17 @@ public void validateReadTest() { String userName = "admin"; String password = "admin"; String influxDatabaseName = "testDataBase"; - InfluxDB influxDb = Mockito.mock(InfluxDB.class); - PowerMockito.when( - InfluxDBFactory.connect( - anyString(), anyString(), anyString(), any(OkHttpClient.Builder.class))) - .thenReturn(influxDb); - PowerMockito.when(InfluxDBFactory.connect(anyString(), anyString(), anyString())) - .thenReturn(influxDb); - doReturn(getDatabase(influxDatabaseName)).when(influxDb).query(new Query("SHOW DATABASES")); - doReturn(getDatabase(influxDatabaseName)).when(influxDb).query(new Query("SHOW SHARDS")); - doReturn(mockResultForNumberAndSizeOfBlocks()) + Mockito.doReturn(getDatabase(influxDatabaseName)) + .when(influxDb) + .query(new Query("SHOW DATABASES")); + Mockito.doReturn(getDatabase(influxDatabaseName)) + .when(influxDb) + .query(new Query("SHOW SHARDS")); + Mockito.doReturn(mockResultForNumberAndSizeOfBlocks()) .when(influxDb) .query(new Query("EXPLAIN SELECT * FROM cpu", influxDatabaseName)); - doReturn(mockResult("cpu", 20)) + Mockito.doReturn(mockResult("cpu", 20)) .when(influxDb) .query(new Query("SELECT * FROM cpu", influxDatabaseName)); @@ -131,10 +154,7 @@ public void validateReadTest() { "Read data to InfluxDB", InfluxDbIO.read() .withDataSourceConfiguration( - DataSourceConfiguration.create( - StaticValueProvider.of(influxHost), - StaticValueProvider.of(userName), - StaticValueProvider.of(password))) + getDataSourceConfiguration(influxHost, userName, password)) .withDatabase(influxDatabaseName) .withQuery("SELECT * FROM cpu")) .apply(Count.globally()); diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle index ba25078b64e3..13969f7a9aed 100644 --- a/sdks/java/io/kafka/build.gradle +++ b/sdks/java/io/kafka/build.gradle @@ -64,6 +64,7 @@ dependencies { // this dependency is a provided dependency for kafka-avro-serializer. It is not needed to compile with Java<=17 // but needed for compile only under Java21, specifically, required for extending from AbstractKafkaAvroDeserializer compileOnly library.java.kafka + permitUnusedDeclared library.java.kafka } testImplementation library.java.kafka_clients testImplementation project(path: ":runners:core-java") diff --git a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultRowMapper.java b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultRowMapper.java index 6837f1893d06..dcfb1813c494 100644 --- a/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultRowMapper.java +++ b/sdks/java/io/singlestore/src/main/java/org/apache/beam/sdk/io/singlestore/SingleStoreDefaultRowMapper.java @@ -64,6 +64,7 @@ import org.apache.beam.sdk.values.Row; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.LocalDate; import org.joda.time.chrono.ISOChronology; @@ -256,7 +257,7 @@ Object getValue(ResultSet rs, Integer index) throws SQLException { return null; } return new DateTime(time.getTime(), ISOChronology.getInstanceUTC()) - .withDate(new LocalDate(0L)); + .withDate(new LocalDate(0L, DateTimeZone.UTC)); } @Override