Download SerDe snapshots from datasketches-tck

.github/workflows/serde-compat.yml

@@ -0,0 +1,41 @@
+name: SerDe Compatibility Test
+
+on:
+  push:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/LICENSE', '**/NOTICE' ]
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  pull_request:
+    paths-ignore: [ '**/*.html', '**/*.md', '**/*.txt', '**/LICENSE', '**/NOTICE' ]
+    # The branches below must be a subset of the branches above
+    branches: [ 'main', '[0-9]+.[0-9]+.[Xx]' ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: ${{ matrix.name }} SerDe Test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: cpp
+            name: C++
+            profile: check-cpp-files
+          - language: go
+            name: Go
+            profile: check-go-files
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Java
+        uses: actions/setup-java@v5
+        with:
+          java-version: '25'
+          distribution: 'temurin'
+
+      - name: Download ${{ matrix.name }} snapshots
+        run: ./tools/download_serialization_test_data.sh ${{ matrix.language }}
+
+      - name: Run Java tests against ${{ matrix.name }} snapshots
+        run: mvn test -P ${{ matrix.profile }}

tools/download_serialization_test_data.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Pin the archive so compatibility tests always use an immutable snapshot set.
+readonly TCK_REVISION="d363b12d293b395d90abb42677f9ea63178dbc0d"
+readonly TCK_ARCHIVE_URL="https://api.github.com/repos/apache/datasketches-tck/tarball/${TCK_REVISION}"
+readonly SCRIPT_DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly REPOSITORY_ROOT="$(cd "${SCRIPT_DIRECTORY}/.." && pwd)"
+readonly SERIALIZATION_DATA="${REPOSITORY_ROOT}/serialization_test_data"
+
+usage() {
+  echo "Usage: $0 [cpp] [go]"
+  echo "Download C++ and/or Go serialization snapshots (both by default)."
+}
+
+if [[ $# -eq 0 ]]; then
+  set -- cpp go
+fi
+
+languages=()
+for language in "$@"; do
+  case "${language}" in
+    cpp | go)
+      ;;
+    -h | --help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Unsupported language: ${language}" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+
+  case " ${languages[*]-} " in
+    *" ${language} "*)
+      ;;
+    *)
+      languages+=("${language}")
+      ;;
+  esac
+done
+
+for command in curl tar mktemp; do
+  if ! command -v "${command}" >/dev/null 2>&1; then
+    echo "Required command not found: ${command}" >&2
+    exit 1
+  fi
+done
+
+mkdir -p "${SERIALIZATION_DATA}"
+temporary_directory="$(mktemp -d "${TMPDIR:-/tmp}/datasketches-tck.XXXXXX")"
+staging_directory=""
+
+cleanup() {
+  rm -rf "${temporary_directory}"
+  if [[ -n "${staging_directory}" && -d "${staging_directory}" ]]; then
+    rm -rf "${staging_directory}"
+  fi
+}
+trap cleanup EXIT
+
+archive_path="${temporary_directory}/datasketches-tck.tar.gz"
+echo "Downloading serialization snapshots from ${TCK_ARCHIVE_URL}"
+curl \
+  --fail \
+  --location \
+  --silent \
+  --show-error \
+  --connect-timeout 60 \
+  --max-time 120 \
+  --header "Accept: application/vnd.github+json" \
+  --header "User-Agent: apache-datasketches-java" \
+  --header "X-GitHub-Api-Version: 2022-11-28" \
+  --output "${archive_path}" \
+  "${TCK_ARCHIVE_URL}"
+
+for language in "${languages[@]}"; do
+  staging_directory="$(
+    mktemp -d "${SERIALIZATION_DATA}/.${language}_generated_files.XXXXXX"
+  )"
+  members=()
+  names=()
+
+  while IFS= read -r member; do
+    case "${member}" in
+      */serialization/"${language}"/snapshots/*.sk)
+        name="${member##*/}"
+        for existing_name in "${names[@]-}"; do
+          if [[ "${name}" == "${existing_name}" ]]; then
+            echo "Duplicate snapshot in archive: ${name}" >&2
+            exit 1
+          fi
+        done
+        members+=("${member}")
+        names+=("${name}")
+        ;;
+    esac
+  done < <(tar -tzf "${archive_path}")
+
+  count=${#members[@]}
+  if [[ ${count} -eq 0 ]]; then
+    echo "No ${language} snapshots found in the TCK archive" >&2
+    exit 1
+  fi
+
+  tar \
+    -xzf "${archive_path}" \
+    -C "${staging_directory}" \
+    --strip-components=4 \
+    "${members[@]}"
+  for name in "${names[@]}"; do
+    if [[ ! -f "${staging_directory}/${name}" ]]; then
+      echo "Failed to extract snapshot: ${name}" >&2
+      exit 1
+    fi
+  done
+
+  destination="${SERIALIZATION_DATA}/${language}_generated_files"
+  if [[ -L "${destination}" ]]; then
+    echo "Snapshot output path cannot be a symbolic link: ${destination}" >&2
+    exit 1
+  fi
+  if [[ -e "${destination}" && ! -d "${destination}" ]]; then
+    echo "Snapshot output path is not a directory: ${destination}" >&2
+    exit 1
+  fi
+  if [[ -d "${destination}" ]]; then
+    rm -rf "${destination}"
+  fi
+  mv "${staging_directory}" "${destination}"
+  staging_directory=""
+  echo "Extracted ${count} ${language} snapshots into ${destination}"
+done