diff --git a/.ci/scripts/test-cadence-xtensa.sh b/.ci/scripts/test-cadence-xtensa.sh new file mode 100755 index 00000000000..dc24eb61f19 --- /dev/null +++ b/.ci/scripts/test-cadence-xtensa.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# Build the Cadence Xtensa op-level gtest tests for the configured backend and +# run them on the Instruction Set Simulator (xt-run). +# +# Requires the Xtensa toolchain env to already be set (run +# .ci/scripts/setup-xtensa-tools.sh first): XTENSA_TOOLCHAIN, +# TOOLCHAIN_VER, XTENSA_CORE, CADENCE_OPT_FLAG, and xt-clang/xt-run on PATH. +# +# Unlike build-cadence-xtensa.sh (the runner, built -fno-exceptions -fno-rtti), +# the gtest tests need exceptions + RTTI, so those flags are NOT set here. + +set -euo pipefail + +: "${XTENSA_TOOLCHAIN:?run setup-xtensa-tools.sh first}" +: "${TOOLCHAIN_VER:?run setup-xtensa-tools.sh first}" +: "${XTENSA_CORE:?run setup-xtensa-tools.sh first}" +: "${CADENCE_OPT_FLAG:?run setup-xtensa-tools.sh first}" + +# Map the optimized-kernel flag to the backend dir + gtest target name. +case "${CADENCE_OPT_FLAG}" in + EXECUTORCH_NNLIB_OPT) TARGET_DIR=hifi ;; + EXECUTORCH_VISION_OPT) TARGET_DIR=vision ;; + EXECUTORCH_FUSION_G3_OPT) TARGET_DIR=fusion_g3 ;; + *) + echo "ERROR: unknown CADENCE_OPT_FLAG='${CADENCE_OPT_FLAG}'" >&2 + exit 1 + ;; +esac +TEST_TARGET="cadence_${TARGET_DIR}_op_tests" +TEST_ELF="cmake-out/backends/cadence/${TARGET_DIR}/operators/tests/${TEST_TARGET}" + +NPROC=$(nproc) +echo "=== building ${TEST_TARGET} for ${XTENSA_CORE} (${CADENCE_OPT_FLAG}) ===" +xt-clang --version | head -1 + +rm -rf cmake-out +cmake \ + -DCMAKE_TOOLCHAIN_FILE=./backends/cadence/cadence.cmake \ + -DCMAKE_INSTALL_PREFIX=cmake-out \ + -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_CADENCE=ON \ + "-D${CADENCE_OPT_FLAG}=ON" \ + -DEXECUTORCH_BUILD_PORTABLE_OPS=ON \ + -DEXECUTORCH_BUILD_CADENCE_OP_TESTS=ON \ + -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \ + -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \ + -DEXECUTORCH_ENABLE_LOGGING=ON \ + -DEXECUTORCH_BUILD_PTHREADPOOL=OFF \ + -DEXECUTORCH_BUILD_CPUINFO=OFF \ + -DEXECUTORCH_USE_DL=OFF \ + -DEXECUTORCH_BUILD_KERNELS_LLM=OFF \ + -DEXECUTORCH_BUILD_DEVTOOLS=OFF \ + -DHAVE_FNMATCH_H=OFF \ + -DFLATCC_ALLOW_WERROR=OFF \ + -DPYTHON_EXECUTABLE="$(which python3)" \ + -Bcmake-out . + +cmake --build cmake-out --target "${TEST_TARGET}" -j"${NPROC}" + +if [[ ! -f "${TEST_ELF}" ]]; then + echo "ERROR: ${TEST_ELF} was not produced" >&2 + exit 1 +fi + +echo "=== running ${TEST_TARGET} on xt-run ===" +LOG=$(mktemp) +# --exit_with_target_code propagates gtest_main's exit code, so a failing test +# fails this step; also assert on the gtest summary lines as a backstop. +xt-run --turbo --exit_with_target_code "${TEST_ELF}" 2>&1 | tee "${LOG}" +if grep -q "\[ FAILED \]" "${LOG}"; then + echo "ERROR: gtest reported failures for ${TEST_TARGET}" >&2 + exit 1 +fi +if ! grep -q "\[ PASSED \]" "${LOG}"; then + echo "ERROR: ${TEST_TARGET} did not report a gtest PASSED summary" >&2 + exit 1 +fi +echo "Cadence ${TARGET_DIR} op tests passed on ${XTENSA_CORE}." diff --git a/.github/workflows/_xtensa_test.yml b/.github/workflows/_xtensa_test.yml new file mode 100644 index 00000000000..ae65b19f24e --- /dev/null +++ b/.github/workflows/_xtensa_test.yml @@ -0,0 +1,84 @@ +# Reusable: build + run the Cadence Xtensa op-level gtest tests for one core on +# the Instruction Set Simulator (xt-run). Mirrors _xtensa_build.yml's native +# OIDC + docker-run skeleton (running xt-run needs the same licensed toolchain), +# then builds the gtest op-test ELF and runs it on the simulator. The runner +# cross-compile and these op tests are separate build configs (the tests need +# exceptions/RTTI that the runner build disables), so this is a self-contained +# job rather than consuming the runner artifact. +name: xtensa-test + +on: + workflow_call: + inputs: + backend: + description: "Cadence backend to test (hifi4 | vision | fusion_g3)" + required: true + type: string + ref: + description: "Git ref to check out" + required: false + type: string + default: "" + +jobs: + test: + name: ${{ inputs.backend }} + runs-on: linux.2xlarge + environment: cadence + permissions: + id-token: write + contents: read + steps: + - name: Checkout executorch + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ inputs.ref }} + + - name: Calculate docker image + id: calculate-docker-image + uses: pytorch/test-infra/.github/actions/calculate-docker-image@main + with: + docker-image-name: ci-image:executorch-ubuntu-22.04-clang12 + + - name: Pull docker image + run: docker pull "${{ steps.calculate-docker-image.outputs.docker-image }}" + + - name: Assume Cadence artifacts role (host OIDC) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.CADENCE_CI_AWS_ROLE }} + aws-region: ${{ vars.CADENCE_CI_AWS_REGION }} + + - name: Build and run op tests on xt-run + env: + DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }} + BACKEND: ${{ inputs.backend }} + XTENSA_S3_BUCKET: ${{ vars.CADENCE_CI_S3_BUCKET }} + shell: bash + run: | + set -eux + # OIDC/role assumption already happened on the host above; pass the + # resulting AWS creds and the store/backend into the CI image, where + # the toolchain download + op-test build + xt-run happen. + docker run --rm \ + -e BACKEND -e XTENSA_S3_BUCKET \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \ + -e AWS_DEFAULT_REGION -e AWS_REGION \ + -v "${GITHUB_WORKSPACE}:/work/executorch" -w /work/executorch \ + "${DOCKER_IMAGE}" \ + bash -c ' + set -exo pipefail + eval "$(/opt/conda/bin/conda shell.bash hook)" + conda activate "$(conda env list --json | jq -r ".envs | .[-1]")" + ./install_requirements.sh > /dev/null + pip install --quiet awscli + # hifi4/fusion_g3 optimized kernels need the foss-xtensa nnlib + # sources, which are not vendored in executorch; the cadence + # installer clones them. vision has no nnlib dependency. + if [ "${BACKEND}" != "vision" ]; then + backends/cadence/install_requirements.sh + fi + source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}" + .ci/scripts/test-cadence-xtensa.sh + ' diff --git a/.github/workflows/build-cadence-runner.yml b/.github/workflows/build-cadence-runner.yml index c447e4f9a20..c455ad83205 100644 --- a/.github/workflows/build-cadence-runner.yml +++ b/.github/workflows/build-cadence-runner.yml @@ -87,6 +87,25 @@ jobs: backend: hifi4 ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} + # Op-level gtest tests on the Xtensa ISS, mirroring cpu-build -> cpu-test. The + # op tests are a self-contained cross-compile (the gtests need exceptions and + # RTTI that the runner build disables), so this does not consume hifi-build's + # artifact; needs: hifi-build only to fail fast when the backend cannot build. + hifi-op-test: + needs: hifi-build + if: >- + github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && + contains(github.event.pull_request.labels.*.name, 'CLA Signed') && contains(github.event.pull_request.labels.*.name, 'meta-exported')) + permissions: + id-token: write + contents: read + uses: ./.github/workflows/_xtensa_test.yml + with: + backend: hifi4 + ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} + vision-build: if: >- github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || diff --git a/backends/cadence/CMakeLists.txt b/backends/cadence/CMakeLists.txt index f04bda30a69..c4bcd47f343 100644 --- a/backends/cadence/CMakeLists.txt +++ b/backends/cadence/CMakeLists.txt @@ -132,3 +132,13 @@ if(EXECUTORCH_BUILD_CADENCE_RUNNER) endif() target_link_options(cadence_executor_runner PRIVATE -static -lm) endif() + +# Cadence op-level gtest tests, cross-compiled for the Xtensa ISS (run via +# xt-run). Built only when EXECUTORCH_BUILD_CADENCE_OP_TESTS is ON and the +# selected backend ships a tests dir. See .ci/scripts/test-cadence-xtensa.sh. +if(EXECUTORCH_BUILD_CADENCE_OP_TESTS + AND EXISTS + "${CMAKE_CURRENT_SOURCE_DIR}/${TARGET_DIR}/operators/tests/CMakeLists.txt" +) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${TARGET_DIR}/operators/tests) +endif() diff --git a/backends/cadence/hifi/operators/tests/CMakeLists.txt b/backends/cadence/hifi/operators/tests/CMakeLists.txt new file mode 100644 index 00000000000..a12bf22e283 --- /dev/null +++ b/backends/cadence/hifi/operators/tests/CMakeLists.txt @@ -0,0 +1,83 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# gtest op-level tests for the Cadence HiFi backend, cross-compiled for Xtensa +# and run on the Instruction Set Simulator (xt-run). Each test calls the kernel +# directly (impl::HiFi::native::) and checks with EXPECT_TENSOR_EQ, mirroring +# the BUCK tests. Built only when EXECUTORCH_BUILD_CADENCE_OP_TESTS is ON; see +# .ci/scripts/test-cadence-xtensa.sh. + +cmake_minimum_required(VERSION 3.19) + +# Tame the vendored googletest build on the Xtensa clang toolchain. It hardcodes +# -Werror -Wconversion on its own sources, which clang-10 trips (e.g. an int -> +# signed char narrowing in gtest-port.h); -Wno-error keeps those non-fatal. A +# ccache launcher also preprocesses then recompiles the .i, where the -I dirs +# are unused and clang warns; -Qunused-arguments silences that. These land after +# googletest's own flags, so they win, and apply to this whole subtree. +add_compile_options(-Qunused-arguments -Wno-error) + +# Let files say "include ". +set(_common_include_directories + ${EXECUTORCH_ROOT}/.. ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10 +) + +# Build googletest FOR the Xtensa target. find_package(GTest) would resolve the +# HOST gtest, which cannot link Xtensa objects, so add the source tree instead. +# The cadence toolchain uses -stdlib=libc++ (exceptions/RTTI/iostream +# available); the ISS is single-threaded, so disable pthreads. +if(NOT TARGET gtest) + set(gtest_disable_pthreads + ON + CACHE BOOL "" FORCE + ) + set(INSTALL_GTEST + OFF + CACHE BOOL "" FORCE + ) + add_subdirectory( + ${EXECUTORCH_ROOT}/third-party/googletest + ${CMAKE_CURRENT_BINARY_DIR}/googletest + ) +endif() + +# Op-level gtests, one source per operator. Each calls its kernel directly and +# checks with EXPECT_TENSOR_EQ, mirroring the BUCK tests. +add_executable( + cadence_hifi_op_tests + test_op_cat.cpp + test_op_dequantize_per_tensor_out.cpp + test_op_div.cpp + test_op_im2row_out.cpp + test_op_permute_copy.cpp + test_op_quantize_per_tensor.cpp + test_op_quantized_conv2d_out.cpp + test_op_quantized_matmul_out.cpp + test_op_quantized_relu_out.cpp + test_op_transpose_copy.cpp + ${EXECUTORCH_ROOT}/runtime/core/exec_aten/testing_util/tensor_util.cpp +) +target_compile_definitions(cadence_hifi_op_tests PRIVATE GTEST_HAS_PTHREAD=0) +target_include_directories( + cadence_hifi_op_tests PRIVATE ${_common_include_directories} + ${CMAKE_BINARY_DIR} +) + +# Direct-call: link the libs that define the kernels under test. custom_ops has +# the quantized ops; aten_ops_cadence has the aten-compliant ops (cat, div, +# im2row, permute, transpose); cadence_kernels is the shared nnlib layer; +# executorch provides the platform layer the tests' runtime_init() needs. No +# cadence_ops_lib registry is needed for direct-call tests. +target_link_libraries( + cadence_hifi_op_tests + PRIVATE gtest + gtest_main + gmock + executorch + custom_ops + aten_ops_cadence + cadence_kernels +)