Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/setup-environment-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ runs:
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }}
java-version: ${{ inputs.java-version == 'default' && '21' || inputs.java-version }}
- name: Setup Gradle
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
with:
Expand Down
18 changes: 9 additions & 9 deletions .github/gh-actions-self-hosted-runners/arc/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ RUN docker buildx install && docker buildx version
USER root
# Native build toolchain for Python C extensions.
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends build-essential time rsync openssh-client && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends build-essential time rsync openssh-client wget && \
rm -rf /var/lib/apt/lists/*
#Install Node
RUN curl -OL https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz && \
Expand All @@ -41,16 +41,16 @@ RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \
rm go${go_version}.linux-amd64.tar.gz
ENV PATH="${PATH}:/usr/local/go/bin"
#Install Java
RUN curl -OL https://cdn.azul.com/zulu/bin/zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \
tar -C /usr/local -xzf zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \
rm zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \
mv /usr/local/zulu11.74.15-ca-jdk11.0.24-linux_x64 /usr/local/java
RUN curl -OL https://cdn.azul.com/zulu/bin/zulu21.50.19-ca-jdk21.0.11-linux_x64.tar.gz && \
tar -C /usr/local -xzf zulu21.50.19-ca-jdk21.0.11-linux_x64.tar.gz && \
rm zulu21.50.19-ca-jdk21.0.11-linux_x64.tar.gz && \
mv /usr/local/zulu21.50.19-ca-jdk21.0.11-linux_x64 /usr/local/java
ENV PATH="${PATH}:/usr/local/java/bin"
#Install Gradle
RUN curl -OL https://services.gradle.org/distributions/gradle-8.4-bin.zip && \
unzip gradle-8.4-bin.zip && \
rm gradle-8.4-bin.zip && \
mv gradle-8.4 /usr/local/gradle
RUN curl -OL https://services.gradle.org/distributions/gradle-8.14.3-bin.zip && \
unzip gradle-8.14.3-bin.zip && \
rm gradle-8.14.3-bin.zip && \
mv gradle-8.14.3 /usr/local/gradle
ENV PATH="${PATH}:/usr/local/gradle/bin"
#Install GitHub CLI
RUN curl -OL https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz && \
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour
| [ PreCommit Java Debezium IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml) | N/A |`Run Java_Debezium_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml?query=event%3Aschedule) |
| [ PreCommit Java ElasticSearch IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml) | N/A |`Run Java_ElasticSearch_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml?query=event%3Aschedule) |
| [ PreCommit Java Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java_Examples_Dataflow PreCommit`| [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) |
| [ PreCommit Java Examples Dataflow Java21 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml) | N/A | `Run Java_Examples_Dataflow_Java21 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml?query=event%3Aschedule) |
| [ PreCommit Java Examples Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml) | N/A | `Run Java_Examples_Dataflow_Java11 PreCommit` | [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow_Java11.yml?query=event%3Aschedule) |
| [ PreCommit Java File-schema-transform IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml) | N/A |`Run Java_File-schema-transform_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml?query=event%3Aschedule) |
| [ PreCommit Java Flink Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml) | N/A |`Run Java_Flink_Versions PreCommit`| [![.github/workflows/beam_PreCommit_Java_Flink_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Flink_Versions.yml?query=event%3Aschedule) |
| [ PreCommit Java GCP IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml) | N/A |`Run Java_GCP_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml?query=event%3Aschedule) |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: default
- name: Set k8s access
uses: ./.github/actions/setup-k8s-access
Expand Down Expand Up @@ -182,8 +180,6 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:io:expansion-service:shadowJar
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64
# The env variable is created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
- name: run integrationTest
uses: ./.github/actions/gradle-command-self-hosted-action
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/beam_PostCommit_Java_Jpms_Dataflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,6 @@ jobs:
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: 11
- name: run PostCommit Java Jpms Dataflow script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/beam_PostCommit_Java_Jpms_Direct.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,6 @@ jobs:
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: 11
- name: run PostCommit Java Jpms Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
3.14
Expand All @@ -86,8 +84,7 @@ jobs:
gradle-command: :sdks:python:test-suites:dataflow:gcpCrossLanguagePostCommit
arguments: |
-PpythonVersion=3.14 \
-PuseWheelDistribution \
-Pjava17Home=$JAVA_HOME_17_X64 \
-PuseWheelDistribution
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
3.14
Expand All @@ -88,8 +86,6 @@ jobs:
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:python:test-suites:direct:gcpCrossLanguagePostCommit
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
3.14
Expand All @@ -86,8 +84,7 @@ jobs:
arguments: |
-PpythonVersion=3.14 \
-PuseWheelDistribution \
-PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \
-Pjava17Home=$JAVA_HOME_17_X64 \
-PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
7 changes: 2 additions & 5 deletions .github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
3.14
Expand All @@ -84,8 +82,7 @@ jobs:
with:
gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit
arguments: |
-PuseWheelDistribution \
-Pjava17Home=$JAVA_HOME_17_X64
-PuseWheelDistribution
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
9 changes: 2 additions & 7 deletions .github/workflows/beam_PostCommit_TransformService_Direct.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
${{ matrix.python_version }}
Expand All @@ -85,10 +83,7 @@ jobs:
with:
gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava
arguments: |
-PtestJavaVersion=17 \
-Pjava17Home=$JAVA_HOME_17_X64 \
-Pjava11Home=$JAVA_HOME_11_X64 \
-PpythonVersion=${{ matrix.python_version }} \
-PpythonVersion=${{ matrix.python_version }}
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/beam_PostCommit_XVR_Direct.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: ${{ matrix.python_version }}
- name: run PostCommit XVR Direct script
env:
Expand All @@ -86,7 +84,6 @@ jobs:
gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner
arguments: |
-PpythonVersion=${{ matrix.python_version }} \
-Pjava17Home=$JAVA_HOME_17_X64 \
-PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v7
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/beam_PostCommit_XVR_Flink.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: ${{ matrix.python_version }}
- name: run PostCommit XVR Flink script
env:
Expand All @@ -86,7 +84,6 @@ jobs:
with:
gradle-command: :runners:flink:${{ env.FlinkVersion }}:job-server:validatesCrossLanguageRunner
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64 \
-PpythonVersion=${{ matrix.python_version }} \
-PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \
- name: Archive JUnit Test Results
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: default
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
Expand All @@ -89,7 +87,6 @@ jobs:
with:
gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64 \
-PuseDockerBuildx
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
${{ matrix.python_version }}
Expand All @@ -85,8 +83,7 @@ jobs:
with:
gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64 \
-PpythonVersion=${{ matrix.python_version }} \
-PpythonVersion=${{ matrix.python_version }}
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v7
if: ${{ !success() }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,17 +73,14 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: 3.14
- name: run PostCommit XVR PythonUsingJavaSQL Dataflow script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingSql
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64 \
-PpythonVersion=3.14 \
-PpythonVersion=3.14
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: |
3.10
${{ matrix.python_version }}
Expand All @@ -85,8 +83,7 @@ jobs:
with:
gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingJava
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64 \
-PpythonVersion=${{ matrix.python_version }} \
-PpythonVersion=${{ matrix.python_version }}
- name: Archive Python Test Results
uses: actions/upload-artifact@v7
if: failure()
Expand Down
5 changes: 1 addition & 4 deletions .github/workflows/beam_PostCommit_XVR_Spark3.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,7 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
python-version: ${{ matrix.python_version }}
- name: run PostCommit XVR Spark3 script
env:
Expand All @@ -85,7 +83,6 @@ jobs:
with:
gradle-command: :runners:spark:3:job-server:validatesCrossLanguageRunner
arguments: |
-Pjava17Home=$JAVA_HOME_17_X64 \
-PpythonVersion=${{ matrix.python_version }} \
-PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \
- name: Archive JUnit Test Results
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ jobs:
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: 11
- name: run Cassandra IO build script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
Expand Down
12 changes: 3 additions & 9 deletions .github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,18 +86,14 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
java-version: |
17
11
java-version: default
- name: run Debezium IO build task
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:io:debezium:build
arguments: |
-PdisableSpotlessCheck=true \
-PdisableCheckStyle=true \
-PtestJavaVersion=17 \
-Pjava17Home=$JAVA_HOME_17_X64 \
-PdisableCheckStyle=true
- name: run Debezium IO additional tasks
uses: ./.github/actions/gradle-command-self-hosted-action
with:
Expand All @@ -106,9 +102,7 @@ jobs:
:sdks:java:io:debezium:integrationTest \
arguments: |
-PdisableSpotlessCheck=true \
-PdisableCheckStyle=true \
-PtestJavaVersion=17 \
-Pjava17Home=$JAVA_HOME_17_X64 \
-PdisableCheckStyle=true
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v7
if: ${{ !success() }}
Expand Down
Loading
Loading