From c778535215ffb6bcbee14a26c4da75710f787b48 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 13 Aug 2026 22:27:36 +0800 Subject: [PATCH 1/3] ci: download snapshots from datasketches-tck --- .github/workflows/cpp-serde-compat.yml | 47 +++---- tools/download_serialization_test_data.py | 149 ++++++++++++++++++++++ 2 files changed, 165 insertions(+), 31 deletions(-) create mode 100755 tools/download_serialization_test_data.py diff --git a/.github/workflows/cpp-serde-compat.yml b/.github/workflows/cpp-serde-compat.yml index 57f7226b2..8bbf080fe 100644 --- a/.github/workflows/cpp-serde-compat.yml +++ b/.github/workflows/cpp-serde-compat.yml @@ -1,4 +1,4 @@ -name: CPP SerDe Compatibility Test +name: SerDe Compatibility Test on: push: @@ -12,45 +12,30 @@ on: jobs: build: - name: SerDe Test + name: ${{ matrix.name }} SerDe Test runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - language: cpp + name: C++ + profile: check-cpp-files + - language: go + name: Go + profile: check-go-files steps: - name: Checkout uses: actions/checkout@v5 - - name: Checkout C++ - uses: actions/checkout@v5 - with: - repository: apache/datasketches-cpp - path: cpp - - name: Setup Java uses: actions/setup-java@v5 with: java-version: '25' distribution: 'temurin' - - name: Configure C++ build - run: cd cpp/build && cmake .. -DGENERATE=true - - - name: Build C++ unit tests - run: cd cpp && cmake --build build --config Release - - - name: Run C++ tests - run: cd cpp && cmake --build build --config Release --target test + - name: Download ${{ matrix.name }} snapshots + run: python3 tools/download_serialization_test_data.py ${{ matrix.language }} - - name: Make dir - run: mkdir -p serialization_test_data/cpp_generated_files - - - name: Copy files - run: cp cpp/build/*/test/*_cpp.sk serialization_test_data/cpp_generated_files - - - name: Run Java tests - run: mvn test -P check-cpp-files - - - name: Upload C++ Generated Sketch Files - uses: actions/upload-artifact@v7 - with: - name: cpp_generated_files - path: serialization_test_data/cpp_generated_files/ - retention-days: 30 + - name: Run Java tests against ${{ matrix.name }} snapshots + run: mvn test -P ${{ matrix.profile }} diff --git a/tools/download_serialization_test_data.py b/tools/download_serialization_test_data.py new file mode 100755 index 000000000..ec2f7ef93 --- /dev/null +++ b/tools/download_serialization_test_data.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import shutil +import tarfile +import tempfile +import urllib.request +from pathlib import Path, PurePosixPath + + +# Pin the archive so compatibility tests always use an immutable snapshot set. +TCK_REVISION = "d363b12d293b395d90abb42677f9ea63178dbc0d" +TCK_ARCHIVE_URL = ( + f"https://api.github.com/repos/apache/datasketches-tck/tarball/{TCK_REVISION}" +) +SUPPORTED_LANGUAGES = ("cpp", "go") + + +def download_archive(destination: Path) -> None: + print(f"Downloading serialization snapshots from {TCK_ARCHIVE_URL}", flush=True) + request = urllib.request.Request( + TCK_ARCHIVE_URL, + headers={ + "Accept": "application/vnd.github+json", + "User-Agent": "apache-datasketches-java", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(request, timeout=60) as response: + with destination.open("wb") as output: + shutil.copyfileobj(response, output) + + +def extract_snapshots(archive_path: Path, languages: tuple[str, ...]) -> None: + repository_root = Path(__file__).resolve().parents[1] + serialization_data = repository_root / "serialization_test_data" + serialization_data.mkdir(parents=True, exist_ok=True) + + staging_directories = { + language: Path( + tempfile.mkdtemp( + prefix=f".{language}_generated_files-", + dir=serialization_data, + ) + ) + for language in languages + } + counts = dict.fromkeys(languages, 0) + + try: + with tarfile.open(archive_path, mode="r:gz") as archive: + for member in archive: + if not member.isfile(): + continue + + path = PurePosixPath(member.name) + if path.suffix != ".sk": + continue + + language = next( + ( + candidate + for candidate in languages + if path.parent.parts[-3:] + == ("serialization", candidate, "snapshots") + ), + None, + ) + if language is None: + continue + + source = archive.extractfile(member) + if source is None: + raise RuntimeError(f"could not read snapshot from archive: {path}") + + destination = staging_directories[language] / path.name + if destination.exists(): + raise RuntimeError(f"duplicate snapshot in archive: {path.name}") + with source, destination.open("wb") as output: + shutil.copyfileobj(source, output) + counts[language] += 1 + + for language, count in counts.items(): + if count == 0: + raise RuntimeError( + f"no {language} snapshots found in the TCK archive" + ) + + for language, staging_directory in staging_directories.items(): + destination = serialization_data / f"{language}_generated_files" + if destination.is_symlink(): + raise RuntimeError( + f"snapshot output path cannot be a symbolic link: {destination}" + ) + if destination.exists(): + if not destination.is_dir(): + raise RuntimeError( + f"snapshot output path is not a directory: {destination}" + ) + shutil.rmtree(destination) + staging_directory.replace(destination) + print( + f"Extracted {counts[language]} {language} snapshots into {destination}" + ) + finally: + for staging_directory in staging_directories.values(): + if staging_directory.exists(): + shutil.rmtree(staging_directory) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Download serialization snapshots from apache/datasketches-tck." + ) + parser.add_argument( + "languages", + choices=SUPPORTED_LANGUAGES, + metavar="LANG", + nargs="*", + help="languages to download (cpp and go by default)", + ) + args = parser.parse_args() + languages = tuple(dict.fromkeys(args.languages or SUPPORTED_LANGUAGES)) + + with tempfile.TemporaryDirectory(prefix="datasketches-tck-") as temp_directory: + archive_path = Path(temp_directory) / "datasketches-tck.tar.gz" + download_archive(archive_path) + extract_snapshots(archive_path, languages) + + +if __name__ == "__main__": + main() From 9944956217506e4115dbcb3f3be72d7bf3e43dff Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 13 Aug 2026 22:51:30 +0800 Subject: [PATCH 2/3] ci: use shell for SerDe snapshot setup --- ...{cpp-serde-compat.yml => serde-compat.yml} | 2 +- tools/download_serialization_test_data.py | 149 ------------------ tools/download_serialization_test_data.sh | 138 ++++++++++++++++ 3 files changed, 139 insertions(+), 150 deletions(-) rename .github/workflows/{cpp-serde-compat.yml => serde-compat.yml} (92%) delete mode 100755 tools/download_serialization_test_data.py create mode 100755 tools/download_serialization_test_data.sh diff --git a/.github/workflows/cpp-serde-compat.yml b/.github/workflows/serde-compat.yml similarity index 92% rename from .github/workflows/cpp-serde-compat.yml rename to .github/workflows/serde-compat.yml index 8bbf080fe..27aa4803b 100644 --- a/.github/workflows/cpp-serde-compat.yml +++ b/.github/workflows/serde-compat.yml @@ -35,7 +35,7 @@ jobs: distribution: 'temurin' - name: Download ${{ matrix.name }} snapshots - run: python3 tools/download_serialization_test_data.py ${{ matrix.language }} + run: ./tools/download_serialization_test_data.sh ${{ matrix.language }} - name: Run Java tests against ${{ matrix.name }} snapshots run: mvn test -P ${{ matrix.profile }} diff --git a/tools/download_serialization_test_data.py b/tools/download_serialization_test_data.py deleted file mode 100755 index ec2f7ef93..000000000 --- a/tools/download_serialization_test_data.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import argparse -import shutil -import tarfile -import tempfile -import urllib.request -from pathlib import Path, PurePosixPath - - -# Pin the archive so compatibility tests always use an immutable snapshot set. -TCK_REVISION = "d363b12d293b395d90abb42677f9ea63178dbc0d" -TCK_ARCHIVE_URL = ( - f"https://api.github.com/repos/apache/datasketches-tck/tarball/{TCK_REVISION}" -) -SUPPORTED_LANGUAGES = ("cpp", "go") - - -def download_archive(destination: Path) -> None: - print(f"Downloading serialization snapshots from {TCK_ARCHIVE_URL}", flush=True) - request = urllib.request.Request( - TCK_ARCHIVE_URL, - headers={ - "Accept": "application/vnd.github+json", - "User-Agent": "apache-datasketches-java", - "X-GitHub-Api-Version": "2022-11-28", - }, - ) - with urllib.request.urlopen(request, timeout=60) as response: - with destination.open("wb") as output: - shutil.copyfileobj(response, output) - - -def extract_snapshots(archive_path: Path, languages: tuple[str, ...]) -> None: - repository_root = Path(__file__).resolve().parents[1] - serialization_data = repository_root / "serialization_test_data" - serialization_data.mkdir(parents=True, exist_ok=True) - - staging_directories = { - language: Path( - tempfile.mkdtemp( - prefix=f".{language}_generated_files-", - dir=serialization_data, - ) - ) - for language in languages - } - counts = dict.fromkeys(languages, 0) - - try: - with tarfile.open(archive_path, mode="r:gz") as archive: - for member in archive: - if not member.isfile(): - continue - - path = PurePosixPath(member.name) - if path.suffix != ".sk": - continue - - language = next( - ( - candidate - for candidate in languages - if path.parent.parts[-3:] - == ("serialization", candidate, "snapshots") - ), - None, - ) - if language is None: - continue - - source = archive.extractfile(member) - if source is None: - raise RuntimeError(f"could not read snapshot from archive: {path}") - - destination = staging_directories[language] / path.name - if destination.exists(): - raise RuntimeError(f"duplicate snapshot in archive: {path.name}") - with source, destination.open("wb") as output: - shutil.copyfileobj(source, output) - counts[language] += 1 - - for language, count in counts.items(): - if count == 0: - raise RuntimeError( - f"no {language} snapshots found in the TCK archive" - ) - - for language, staging_directory in staging_directories.items(): - destination = serialization_data / f"{language}_generated_files" - if destination.is_symlink(): - raise RuntimeError( - f"snapshot output path cannot be a symbolic link: {destination}" - ) - if destination.exists(): - if not destination.is_dir(): - raise RuntimeError( - f"snapshot output path is not a directory: {destination}" - ) - shutil.rmtree(destination) - staging_directory.replace(destination) - print( - f"Extracted {counts[language]} {language} snapshots into {destination}" - ) - finally: - for staging_directory in staging_directories.values(): - if staging_directory.exists(): - shutil.rmtree(staging_directory) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Download serialization snapshots from apache/datasketches-tck." - ) - parser.add_argument( - "languages", - choices=SUPPORTED_LANGUAGES, - metavar="LANG", - nargs="*", - help="languages to download (cpp and go by default)", - ) - args = parser.parse_args() - languages = tuple(dict.fromkeys(args.languages or SUPPORTED_LANGUAGES)) - - with tempfile.TemporaryDirectory(prefix="datasketches-tck-") as temp_directory: - archive_path = Path(temp_directory) / "datasketches-tck.tar.gz" - download_archive(archive_path) - extract_snapshots(archive_path, languages) - - -if __name__ == "__main__": - main() diff --git a/tools/download_serialization_test_data.sh b/tools/download_serialization_test_data.sh new file mode 100755 index 000000000..c87eaeb73 --- /dev/null +++ b/tools/download_serialization_test_data.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +# Pin the archive so compatibility tests always use an immutable snapshot set. +readonly TCK_REVISION="d363b12d293b395d90abb42677f9ea63178dbc0d" +readonly TCK_ARCHIVE_URL="https://api.github.com/repos/apache/datasketches-tck/tarball/${TCK_REVISION}" +readonly SCRIPT_DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly REPOSITORY_ROOT="$(cd "${SCRIPT_DIRECTORY}/.." && pwd)" +readonly SERIALIZATION_DATA="${REPOSITORY_ROOT}/serialization_test_data" + +usage() { + echo "Usage: $0 [cpp] [go]" + echo "Download C++ and/or Go serialization snapshots (both by default)." +} + +if [[ $# -eq 0 ]]; then + set -- cpp go +fi + +languages=() +for language in "$@"; do + case "${language}" in + cpp | go) + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unsupported language: ${language}" >&2 + usage >&2 + exit 2 + ;; + esac + + case " ${languages[*]-} " in + *" ${language} "*) + ;; + *) + languages+=("${language}") + ;; + esac +done + +for command in curl tar mktemp; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +mkdir -p "${SERIALIZATION_DATA}" +temporary_directory="$(mktemp -d "${TMPDIR:-/tmp}/datasketches-tck.XXXXXX")" +staging_directory="" + +cleanup() { + rm -rf "${temporary_directory}" + if [[ -n "${staging_directory}" && -d "${staging_directory}" ]]; then + rm -rf "${staging_directory}" + fi +} +trap cleanup EXIT + +archive_path="${temporary_directory}/datasketches-tck.tar.gz" +echo "Downloading serialization snapshots from ${TCK_ARCHIVE_URL}" +curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --connect-timeout 60 \ + --max-time 120 \ + --header "Accept: application/vnd.github+json" \ + --header "User-Agent: apache-datasketches-java" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + --output "${archive_path}" \ + "${TCK_ARCHIVE_URL}" + +for language in "${languages[@]}"; do + staging_directory="$( + mktemp -d "${SERIALIZATION_DATA}/.${language}_generated_files.XXXXXX" + )" + count=0 + + while IFS= read -r member; do + case "${member}" in + */serialization/"${language}"/snapshots/*.sk) + name="${member##*/}" + output="${staging_directory}/${name}" + if [[ -e "${output}" || -L "${output}" ]]; then + echo "Duplicate snapshot in archive: ${name}" >&2 + exit 1 + fi + tar -xOzf "${archive_path}" "${member}" > "${output}" + count=$((count + 1)) + ;; + esac + done < <(tar -tzf "${archive_path}") + + if [[ ${count} -eq 0 ]]; then + echo "No ${language} snapshots found in the TCK archive" >&2 + exit 1 + fi + + destination="${SERIALIZATION_DATA}/${language}_generated_files" + if [[ -L "${destination}" ]]; then + echo "Snapshot output path cannot be a symbolic link: ${destination}" >&2 + exit 1 + fi + if [[ -e "${destination}" && ! -d "${destination}" ]]; then + echo "Snapshot output path is not a directory: ${destination}" >&2 + exit 1 + fi + if [[ -d "${destination}" ]]; then + rm -rf "${destination}" + fi + mv "${staging_directory}" "${destination}" + staging_directory="" + echo "Extracted ${count} ${language} snapshots into ${destination}" +done From a035f41e98e39ce74d4742d664901ec02b543a1d Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 13 Aug 2026 22:55:03 +0800 Subject: [PATCH 3/3] ci: extract TCK snapshots in one pass --- tools/download_serialization_test_data.sh | 31 +++++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tools/download_serialization_test_data.sh b/tools/download_serialization_test_data.sh index c87eaeb73..800e6206e 100755 --- a/tools/download_serialization_test_data.sh +++ b/tools/download_serialization_test_data.sh @@ -98,28 +98,43 @@ for language in "${languages[@]}"; do staging_directory="$( mktemp -d "${SERIALIZATION_DATA}/.${language}_generated_files.XXXXXX" )" - count=0 + members=() + names=() while IFS= read -r member; do case "${member}" in */serialization/"${language}"/snapshots/*.sk) name="${member##*/}" - output="${staging_directory}/${name}" - if [[ -e "${output}" || -L "${output}" ]]; then - echo "Duplicate snapshot in archive: ${name}" >&2 - exit 1 - fi - tar -xOzf "${archive_path}" "${member}" > "${output}" - count=$((count + 1)) + for existing_name in "${names[@]-}"; do + if [[ "${name}" == "${existing_name}" ]]; then + echo "Duplicate snapshot in archive: ${name}" >&2 + exit 1 + fi + done + members+=("${member}") + names+=("${name}") ;; esac done < <(tar -tzf "${archive_path}") + count=${#members[@]} if [[ ${count} -eq 0 ]]; then echo "No ${language} snapshots found in the TCK archive" >&2 exit 1 fi + tar \ + -xzf "${archive_path}" \ + -C "${staging_directory}" \ + --strip-components=4 \ + "${members[@]}" + for name in "${names[@]}"; do + if [[ ! -f "${staging_directory}/${name}" ]]; then + echo "Failed to extract snapshot: ${name}" >&2 + exit 1 + fi + done + destination="${SERIALIZATION_DATA}/${language}_generated_files" if [[ -L "${destination}" ]]; then echo "Snapshot output path cannot be a symbolic link: ${destination}" >&2