diff --git a/.github/workflows/cpp-serde-compat.yml b/.github/workflows/cpp-serde-compat.yml deleted file mode 100644 index 57f7226b2..000000000 --- a/.github/workflows/cpp-serde-compat.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: CPP SerDe Compatibility Test - -on: - push: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/LICENSE', '**/NOTICE' ] - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] - pull_request: - paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/LICENSE', '**/NOTICE' ] - # The branches below must be a subset of the branches above - branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] - workflow_dispatch: - -jobs: - build: - name: SerDe Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v5 - - - name: Checkout C++ - uses: actions/checkout@v5 - with: - repository: apache/datasketches-cpp - path: cpp - - - name: Setup Java - uses: actions/setup-java@v5 - with: - java-version: '25' - distribution: 'temurin' - - - name: Configure C++ build - run: cd cpp/build && cmake .. -DGENERATE=true - - - name: Build C++ unit tests - run: cd cpp && cmake --build build --config Release - - - name: Run C++ tests - run: cd cpp && cmake --build build --config Release --target test - - - name: Make dir - run: mkdir -p serialization_test_data/cpp_generated_files - - - name: Copy files - run: cp cpp/build/*/test/*_cpp.sk serialization_test_data/cpp_generated_files - - - name: Run Java tests - run: mvn test -P check-cpp-files - - - name: Upload C++ Generated Sketch Files - uses: actions/upload-artifact@v7 - with: - name: cpp_generated_files - path: serialization_test_data/cpp_generated_files/ - retention-days: 30 diff --git a/.github/workflows/serde-compat.yml b/.github/workflows/serde-compat.yml new file mode 100644 index 000000000..27aa4803b --- /dev/null +++ b/.github/workflows/serde-compat.yml @@ -0,0 +1,41 @@ +name: SerDe Compatibility Test + +on: + push: + paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/LICENSE', '**/NOTICE' ] + branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] + pull_request: + paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/LICENSE', '**/NOTICE' ] + # The branches below must be a subset of the branches above + branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ] + workflow_dispatch: + +jobs: + build: + name: ${{ matrix.name }} SerDe Test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - language: cpp + name: C++ + profile: check-cpp-files + - language: go + name: Go + profile: check-go-files + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup Java + uses: actions/setup-java@v5 + with: + java-version: '25' + distribution: 'temurin' + + - name: Download ${{ matrix.name }} snapshots + run: ./tools/download_serialization_test_data.sh ${{ matrix.language }} + + - name: Run Java tests against ${{ matrix.name }} snapshots + run: mvn test -P ${{ matrix.profile }} diff --git a/tools/download_serialization_test_data.sh b/tools/download_serialization_test_data.sh new file mode 100755 index 000000000..800e6206e --- /dev/null +++ b/tools/download_serialization_test_data.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +# Pin the archive so compatibility tests always use an immutable snapshot set. +readonly TCK_REVISION="d363b12d293b395d90abb42677f9ea63178dbc0d" +readonly TCK_ARCHIVE_URL="https://api.github.com/repos/apache/datasketches-tck/tarball/${TCK_REVISION}" +readonly SCRIPT_DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPOSITORY_ROOT="$(cd "${SCRIPT_DIRECTORY}/.." && pwd)" +readonly SERIALIZATION_DATA="${REPOSITORY_ROOT}/serialization_test_data" + +usage() { + echo "Usage: $0 [cpp] [go]" + echo "Download C++ and/or Go serialization snapshots (both by default)." +} + +if [[ $# -eq 0 ]]; then + set -- cpp go +fi + +languages=() +for language in "$@"; do + case "${language}" in + cpp | go) + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unsupported language: ${language}" >&2 + usage >&2 + exit 2 + ;; + esac + + case " ${languages[*]-} " in + *" ${language} "*) + ;; + *) + languages+=("${language}") + ;; + esac +done + +for command in curl tar mktemp; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +mkdir -p "${SERIALIZATION_DATA}" +temporary_directory="$(mktemp -d "${TMPDIR:-/tmp}/datasketches-tck.XXXXXX")" +staging_directory="" + +cleanup() { + rm -rf "${temporary_directory}" + if [[ -n "${staging_directory}" && -d "${staging_directory}" ]]; then + rm -rf "${staging_directory}" + fi +} +trap cleanup EXIT + +archive_path="${temporary_directory}/datasketches-tck.tar.gz" +echo "Downloading serialization snapshots from ${TCK_ARCHIVE_URL}" +curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --connect-timeout 60 \ + --max-time 120 \ + --header "Accept: application/vnd.github+json" \ + --header "User-Agent: apache-datasketches-java" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --output "${archive_path}" \ + "${TCK_ARCHIVE_URL}" + +for language in "${languages[@]}"; do + staging_directory="$( + mktemp -d "${SERIALIZATION_DATA}/.${language}_generated_files.XXXXXX" + )" + members=() + names=() + + while IFS= read -r member; do + case "${member}" in + */serialization/"${language}"/snapshots/*.sk) + name="${member##*/}" + for existing_name in "${names[@]-}"; do + if [[ "${name}" == "${existing_name}" ]]; then + echo "Duplicate snapshot in archive: ${name}" >&2 + exit 1 + fi + done + members+=("${member}") + names+=("${name}") + ;; + esac + done < <(tar -tzf "${archive_path}") + + count=${#members[@]} + if [[ ${count} -eq 0 ]]; then + echo "No ${language} snapshots found in the TCK archive" >&2 + exit 1 + fi + + tar \ + -xzf "${archive_path}" \ + -C "${staging_directory}" \ + --strip-components=4 \ + "${members[@]}" + for name in "${names[@]}"; do + if [[ ! -f "${staging_directory}/${name}" ]]; then + echo "Failed to extract snapshot: ${name}" >&2 + exit 1 + fi + done + + destination="${SERIALIZATION_DATA}/${language}_generated_files" + if [[ -L "${destination}" ]]; then + echo "Snapshot output path cannot be a symbolic link: ${destination}" >&2 + exit 1 + fi + if [[ -e "${destination}" && ! -d "${destination}" ]]; then + echo "Snapshot output path is not a directory: ${destination}" >&2 + exit 1 + fi + if [[ -d "${destination}" ]]; then + rm -rf "${destination}" + fi + mv "${staging_directory}" "${destination}" + staging_directory="" + echo "Extracted ${count} ${language} snapshots into ${destination}" +done