diff --git a/.github/scripts/build-openusd.py b/.github/scripts/build-openusd.py index bcdb7fd574..1bfa60cac8 100644 --- a/.github/scripts/build-openusd.py +++ b/.github/scripts/build-openusd.py @@ -26,6 +26,13 @@ from swiftusd_ci_common import * def install_cmake(): + if which("cmake"): + print("CMake is already on PATH, will not reinstall") + return + else: + print(os.environ) + print("Couldn't find CMake, will install") + print("Downloading CMake...") run(["curl", "-L", "https://github.com/Kitware/CMake/releases/download/v3.28.6/cmake-3.28.6-macos-universal.dmg", "--output", "CMake.dmg"], diff --git a/.github/scripts/compute-openusd-build-cache-key.py b/.github/scripts/compute-openusd-build-cache-key.py index 9cd4c29af1..a66d7fc59d 100644 --- a/.github/scripts/compute-openusd-build-cache-key.py +++ b/.github/scripts/compute-openusd-build-cache-key.py @@ -32,7 +32,10 @@ # Turn branch names like `dev` into a hash commit to avoid incorrect cache key matches clone_openusd() rev_parsed_ref = run(["git", "rev-parse", Environment.GitRef.openusd], cwd=Environment.Path.openusd, logOutput=False).output[0] - build_flags = "".join(get_openusd_build_flags(Environment.TestCombination.target_platform)) + # Don't include the build directory when forming the cache key + build_flags = get_openusd_build_flags(Environment.TestCombination.target_platform) + build_flags = [x for x in build_flags if not x.startswith("/")] + build_flags = "".join(build_flags) result = " ".join([ Environment.TestCombination.target_platform, @@ -43,4 +46,7 @@ openusd_patch_hash ]) - printAndWrite(output=f"cache-key='{result}'") \ No newline at end of file + # Cache keys cannot contain commas + result = result.replace(",", "") + + printAndWrite(output=f"cache-key={result}") diff --git a/.github/scripts/define-test-matrix.py b/.github/scripts/define-test-matrix.py index 7c4aa3701c..19a0f9560e 100644 --- a/.github/scripts/define-test-matrix.py +++ b/.github/scripts/define-test-matrix.py @@ -23,8 +23,18 @@ import json import math +def get_xcodebuild_destination(target_platform): + if target_platform == "macOS": return "platform=macOS,name=My Mac" + elif target_platform == "iOS": return Environment.TestCombination.at_desk_iOS_xcodebuild_destination + elif target_platform == "iOSSimulator": return "platform=iOS Simulator,name=iPhone 17 Pro" + elif target_platform == "visionOS": return Environment.TestCombination.at_desk_visionOS_xcodebuild_destination + elif target_platform == "visionOSSimulator": return "platform=visionOS Simulator,name=Apple Vision Pro (at 2732x2048)" + else: + print(f"Error: Unknown target '{target_platform}'") + exit(1) + if __name__ == "__main__": - target_platforms = ["macOS", "iOSSimulator", "visionOSSimulator"] + target_platforms = ["macOS", "iOS", "iOSSimulator", "visionOS", "visionOSSimulator"] configs = ["Debug", "Release"] build_systems = ["xcodebuild-xcodeproj", "swiftbuild-SPM-Tests", "xcodebuild-SPM-Tests"] @@ -32,16 +42,25 @@ for target_platform in target_platforms: for config in configs: for build_system in build_systems: - - # if target_platform == "macOS" and build_system == "xcodebuild-xcodeproj": - # # Disabled for now - # continue - + exclusivity_keys = [] + if build_system == "swiftbuild-SPM-Tests" and target_platform != "macOS": # swiftbuild only supports macOS continue - all_combinations.append({"target_platform" : target_platform, "config" : config, "build_system" : build_system}) + if build_system == "xcodebuild-SPM-Tests" and target_platform in ["iOS", "visionOS"]: + # xcodebuild on a Swift Package doesn't support physical iOS/visionOS devices + continue + + xcodebuild_destination = get_xcodebuild_destination(target_platform) + if xcodebuild_destination is None: continue + + if target_platform in ["iOS", "visionOS"]: + exclusivity_keys.append(target_platform) + + all_combinations.append({"target_platform" : target_platform, "config" : config, + "build_system" : build_system, "xcodebuild_destination" : xcodebuild_destination, + "exclusivity_keys" : exclusivity_keys}) random.shuffle(all_combinations) diff --git a/.github/scripts/release-new-version.py b/.github/scripts/release-new-version.py new file mode 100644 index 0000000000..4bedd8a98b --- /dev/null +++ b/.github/scripts/release-new-version.py @@ -0,0 +1,90 @@ +#===----------------------------------------------------------------------===# +# This source file is part of github.com/apple/SwiftUsd +# +# Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +#===----------------------------------------------------------------------===# + +from swiftusd_ci_common import * +import argparse +import re + +def quiet_run(args): + return run(args, cwd=Environment.Path.swiftusd, logOutput=False) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("swiftusd_tag") + args = parser.parse_args() + + unstaged = quiet_run(["git", "status", "--porcelain=v1"]).output + modified_files = [] + for l in unstaged: + # porcelain format: or -> , + # where xy is a two-character status code + l = l[3:] + if m := re.match("(.*)->(.*)", l): + modified_files.append(m.group(1).strip()) + modified_files.append(m.group(2).strip()) + else: + modified_files.append(l.strip()) + + modified_roots = set() + for f in modified_files: + if m := re.match("([^/]*)/.*", f): + modified_roots.add(m.group(1)) + else: + modified_roots.add(f) + + required_roots = ["swift-package", "docs", "SwiftUsd.doccarchive"] + expected_roots = ["Package.swift"] + had_error = False + + for r in required_roots: + if r not in modified_roots: + print(f"Error: {r} was not modified") + had_error = True + modified_roots.discard(r) + + for r in expected_roots: + if r not in modified_roots: + print(f"Warning: {r} was not modified") + modified_roots.discard(r) + + if len(modified_roots) != 0: + print(f"Error: unexpected modifications: {modified_roots}") + had_error = True + + if had_error: + exit(1) + + quiet_run(["git", "add", "-A"]) + quiet_run(["git", "commit", "-m", f"Publish documentation and update swift-package for SwiftUsd {args.swiftusd_tag}"]) + quiet_run(["git", "tag", args.swiftusd_tag]) + + + printAndWrite(summary=f"""To release SwiftUsd {args.swiftusd_tag}, run: +``` +cd {Environment.Path.swiftusd} +git push +git push origin {args.swiftusd_tag} +``` +Remember to run these commands in SwiftUsd-Tests and SwiftUsd-ast-answerer: +``` +git tag {args.swiftusd_tag} +git push +git push origin {args.swiftusd_tag} +```""") diff --git a/.github/scripts/run-tests-helper.py b/.github/scripts/run-tests-helper.py index ad800bebe2..3f96fdd7c5 100644 --- a/.github/scripts/run-tests-helper.py +++ b/.github/scripts/run-tests-helper.py @@ -45,7 +45,7 @@ def should_extract(l): if action == "build": return any([s in l for s in ["error:"]]) elif action == "test": - return any([s in l for s in ["failed:", "launchd"]]) + return any([s in l for s in ["failed:", "launchd", "crash"]]) def processRunResult(runResult): end = time.time() @@ -78,22 +78,52 @@ def processRunResult(runResult): def prepare_to_build_tests(): print("Preparing to build tests...") + run(["swift", "package", "--package-path", "ReconfigurePbxprojPackageDependency", "clean"], + cwd=Environment.Path.swiftusd_tests) run(["swift", "run", "--package-path", "ReconfigurePbxprojPackageDependency", "ReconfigurePbxprojPackageDependency", "SwiftUsdTests.xcodeproj/project.pbxproj", "--replace", "https://github.com/apple/SwiftUsd", "--with", "SwiftUsd"], cwd=Environment.Path.swiftusd_tests) - if not (Environment.Path.swiftusd_tests / "SwiftUsd").exists(): - (Environment.Path.swiftusd_tests / "SwiftUsd").symlink_to(Environment.Path.swiftusd) + + if (Environment.Path.swiftusd_tests / "SwiftUsd").exists(): + os.unlink(Environment.Path.swiftusd_tests / "SwiftUsd") + (Environment.Path.swiftusd_tests / "SwiftUsd").symlink_to(Environment.Path.swiftusd) run(["python3", "make-spm-tests.py", "--local", "--force"], cwd=Environment.Path.swiftusd_tests) -def get_xcodebuild_destination(): - if Environment.TestCombination.target_platform == "macOS": return "platform=macOS,name=My Mac" - elif Environment.TestCombination.target_platform == "iOSSimulator": return "platform=iOS Simulator,name=iPhone 17 Pro" - elif Environment.TestCombination.target_platform == "visionOSSimulator": return "platform=visionOS Simulator,name=Apple Vision Pro (at 2732x2048)" +def conditionalCommonArgs(): + result = [] + + def handleJobs(x, singleMinus): + nonlocal result + try: + if int(x) > 0: + result += ["-jobs" if singleMinus else "--jobs", x] + except: + pass + + def handleDevelopmentTeam(): + nonlocal result + if Environment.TestCombination.at_desk_development_team: + result += [f"DEVELOPMENT_TEAM={Environment.TestCombination.at_desk_development_team}"] + + if Environment.TestCombination.build_system == "xcodebuild-xcodeproj": + handleJobs(Environment.TestCombination.at_desk_xcodebuild_jobs, True) + handleDevelopmentTeam() + + elif Environment.TestCombination.build_system == "swiftbuild-SPM-Tests": + handleJobs(Environment.TestCombination.at_desk_swiftbuild_jobs, False) + + elif Environment.TestCombination.build_system == "xcodebuild-SPM-Tests": + handleJobs(Environment.TestCombination.at_desk_xcodebuild_jobs, True) + handleDevelopmentTeam() + else: - print(f"Error: Unknown target '{Environment.TestCombination.target_platform}'") + print(f"Error: Unknown build system {Environment.TestCombination.build_system}") exit(1) + + + return result def do_xcodebuild_xcodeproj_tests(action): build_or_run_test_suite( @@ -106,11 +136,11 @@ def do_xcodebuild_xcodeproj_tests(action): ], buildTestCommonArgs=[ "-verbose", "-skipMacroValidation", - "-scheme", "UnitTests", + "-scheme", "UnitTests", "-configuration", Environment.TestCombination.config, - "-destination", get_xcodebuild_destination(), + "-destination", Environment.TestCombination.xcodebuild_destination, "OTHER_SWIFT_FLAGS=$(inherited) -DSWIFTUSD_TESTS_SUPPRESS_PERFORMANCE_FAILURES", - ], + ] + conditionalCommonArgs(), cwd=Environment.Path.swiftusd_tests, action=action, ) @@ -119,10 +149,14 @@ def do_swiftbuild_spm_tests(action): env = os.environ.copy() env["SWIFT_BACKTRACE"] = "interactive=no" - # Set DEVELOPER_DIR to work around - # error: cannot load module 'SwiftCompilerPlugin' built with SDK 'macosx26.0' when using SDK 'macosx26.2' - # https://github.com/apple/SwiftUsd/actions/runs/21256906902/job/61175337955 - env["DEVELOPER_DIR"] = "/Applications/Xcode-latest.app" + # todo: revisit this, setting it here means that + # xcodebuild -version in the logs is misleading because + # it's computed without this environment variable + if os.path.exists("/Applications/Xcode-latest.app"): + # Set DEVELOPER_DIR to work around + # error: cannot load module 'SwiftCompilerPlugin' built with SDK 'macosx26.0' when using SDK 'macosx26.2' + # https://github.com/apple/SwiftUsd/actions/runs/21256906902/job/61175337955 + env["DEVELOPER_DIR"] = "/Applications/Xcode-latest.app" build_or_run_test_suite( name="swiftbuild-SPM-Tests", @@ -134,7 +168,7 @@ def do_swiftbuild_spm_tests(action): "-Xswiftc", "-DOPENUSD_SWIFT_BUILD_FROM_CLI", "-Xcxx", "-DOPENUSD_SWIFT_BUILD_FROM_CLI", "--configuration", Environment.TestCombination.config.lower(), "-Xswiftc", "-DSWIFTUSD_TESTS_SUPPRESS_PERFORMANCE_FAILURES", - ], + ] + conditionalCommonArgs(), cwd=Environment.Path.swiftusd_tests / "SPM-Tests", action=action, env=env @@ -153,9 +187,9 @@ def do_xcodebuild_spm_tests(action): "-verbose", "-skipMacroValidation", "-scheme", "SPM-Tests-Package", "-config", Environment.TestCombination.config, - "-destination", get_xcodebuild_destination(), + "-destination", Environment.TestCombination.xcodebuild_destination, "OTHER_SWIFT_FLAGS=$(inherited) -DSWIFTUSD_TESTS_SUPPRESS_PERFORMANCE_FAILURES", - ], + ] + conditionalCommonArgs(), cwd=Environment.Path.swiftusd_tests / "SPM-Tests", action=action, ) @@ -198,4 +232,4 @@ def run_tests(args, subparsers): args = parser.parse_args() - args.func(args) \ No newline at end of file + args.func(args) diff --git a/.github/scripts/swiftusd_ci_common/environment.py b/.github/scripts/swiftusd_ci_common/environment.py index cb67933938..bc15f723ba 100644 --- a/.github/scripts/swiftusd_ci_common/environment.py +++ b/.github/scripts/swiftusd_ci_common/environment.py @@ -37,6 +37,12 @@ class TestCombination: config = os.getenv("CONFIG") build_system = os.getenv("BUILD_SYSTEM") github_run_id = os.getenv("GITHUB_RUN_ID") + xcodebuild_destination = os.getenv("XCODEBUILD_DESTINATION") + at_desk_iOS_xcodebuild_destination = os.getenv("ATDESK_IOS_XCODEBUILD_DESTINATION") + at_desk_visionOS_xcodebuild_destination = os.getenv("ATDESK_VISIONOS_XCODEBUILD_DESTINATION") + at_desk_swiftbuild_jobs = os.getenv("ATDESK_SWIFTBUILD_JOBS") + at_desk_xcodebuild_jobs = os.getenv("ATDESK_XCODEBUILD_JOBS") + at_desk_development_team = os.getenv("ATDESK_DEVELOPMENT_TEAM") class Path: swiftusd = _getenvpath("SWIFTUSD_PATH") @@ -48,4 +54,4 @@ class Path: tmp_dir = _getenvpath("RUNNER_TEMP") github_workspace = _getenvpath("GITHUB_WORKSPACE") matrix_result = _getenvpath("MATRIX_RESULT_PATH") - matrix_results = _getenvpath("MATRIX_RESULTS_PATH") \ No newline at end of file + matrix_results = _getenvpath("MATRIX_RESULTS_PATH") diff --git a/.github/scripts/swiftusd_ci_common/openusd_building.py b/.github/scripts/swiftusd_ci_common/openusd_building.py index 71e8a5ddb5..5a005a8bae 100644 --- a/.github/scripts/swiftusd_ci_common/openusd_building.py +++ b/.github/scripts/swiftusd_ci_common/openusd_building.py @@ -32,24 +32,26 @@ def clone_openusd(checkout=None): run(["git", "checkout", checkout], cwd=Environment.Path.openusd, logOutput=False) def get_openusd_build_flags(target): + file_prefix_map = f"--build-args=USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map={Environment.Path.openusd}=OpenUSD\"" + if target == "macOS": return ["--embree", "--imageio", "--alembic", "--openvdb", "--no-python", - "--ignore-homebrew", "--build-target", "native", openusd_build_dir("macOS")] + "--ignore-homebrew", "--build-target", "native", openusd_build_dir("macOS"), file_prefix_map] if target == "iOS": return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew", - "--build-target", "iOS", openusd_build_dir("iOS")] + "--build-target", "iOS", openusd_build_dir("iOS"), file_prefix_map] if target == "iOSSimulator": return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew", - "--build-target", "iOSSimulator", openusd_build_dir("iOSSimulator")] + "--build-target", "iOSSimulator", openusd_build_dir("iOSSimulator"), file_prefix_map] if target == "visionOS": return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew", - "--build-target", "visionOS", openusd_build_dir("visionOS")] + "--build-target", "visionOS", openusd_build_dir("visionOS"), file_prefix_map] if target == "visionOSSimulator": return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew", - "--build-target", "visionOSSimulator", openusd_build_dir("visionOSSimulator")] + "--build-target", "visionOSSimulator", openusd_build_dir("visionOSSimulator"), file_prefix_map] - raise ValueError(f"Unknown target {target}") \ No newline at end of file + raise ValueError(f"Unknown target {target}") diff --git a/.github/scripts/swiftusd_ci_common/subprocesses.py b/.github/scripts/swiftusd_ci_common/subprocesses.py index 0fccbfcabf..401dbdd755 100644 --- a/.github/scripts/swiftusd_ci_common/subprocesses.py +++ b/.github/scripts/swiftusd_ci_common/subprocesses.py @@ -67,4 +67,7 @@ def run(args, cwd=None, env=None, input=None, logCmd=True, logOutput=True, check exit(result.returncode) - return result \ No newline at end of file + return result + +def which(cmd): + return run(["which", cmd], logCmd=False, logOutput=False, check=False).returncode == 0 \ No newline at end of file diff --git a/.github/scripts/swiftusd_ci_common/test_matrix_results.py b/.github/scripts/swiftusd_ci_common/test_matrix_results.py index a163319b1f..f2a4448d84 100644 --- a/.github/scripts/swiftusd_ci_common/test_matrix_results.py +++ b/.github/scripts/swiftusd_ci_common/test_matrix_results.py @@ -40,12 +40,12 @@ def __init__(self): raw_xcodebuild_version = "\n".join(run(["xcodebuild", "-version"]).output) short_xcodebuild_version = re.search(r"Xcode (.*)", raw_xcodebuild_version).group(1) + " (" + re.search(r"Build version (.*)", raw_xcodebuild_version).group(1) + ")" if Environment.GitRef.swiftusd and Environment.Path.swiftusd: - swiftusd_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd], cwd=Environment.Path.swiftusd).output[0] + swiftusd_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd], cwd=Environment.Path.swiftusd, check=False).output[0] else: swiftusd_ref = "null" if Environment.GitRef.swiftusd_tests and Environment.Path.swiftusd_tests: - swiftusd_tests_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd_tests], cwd=Environment.Path.swiftusd_tests).output[0] + swiftusd_tests_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd_tests], cwd=Environment.Path.swiftusd_tests, check=False).output[0] else: swiftusd_tests_ref = "null" diff --git a/.github/workflows/release-new-version.yml b/.github/workflows/release-new-version.yml deleted file mode 100644 index 1132323f43..0000000000 --- a/.github/workflows/release-new-version.yml +++ /dev/null @@ -1,86 +0,0 @@ -#===----------------------------------------------------------------------===# -# This source file is part of github.com/apple/SwiftUsd -# -# Copyright © 2025 Apple Inc. and the SwiftUsd project authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -#===----------------------------------------------------------------------===# - -name: Release new version -run-name: Releasing new version -permissions: - contents: read - -on: - workflow_dispatch: - inputs: - swiftusd-ref: - description: 'SwiftUsd ref' - required: true - type: string - - openusd-ref: - description: 'OpenUSD ref' - required: true - type: string - - swiftusd-version: - description: 'SwiftUsd version' - required: true - type: string - -jobs: - Release-New-Version: - name: Release new version - runs-on: self-hosted - permissions: - contents: write - steps: - - - name: Check out repository code - uses: actions/checkout@v5 - with: - ref: ${{ inputs.swiftusd-ref }} - - - name: Build SwiftUsd - uses: ./.github/actions/build-swiftusd - with: - swiftusd-ref: ${{ inputs.swiftusd-ref }} - openusd-ref: ${{ inputs.openusd-ref }} - - - name: Build symbol graphs - uses: ./.github/actions/build-symbol-graphs - - - name: Update SwiftUsd.doccarchive - run: | - xcrun docc convert --additional-symbol-graph-dir ./.symbol-graphs ./SwiftUsd.docc \ - --output-path ./SwiftUsd.doccarchive --emit-lmbdb-index - - name: Update SwiftUsd/docs - run: | - xcrun docc convert --additional-symbol-graph-dir ./.symbol-graphs ./SwiftUsd.docc \ - --output-path ./docs --emit-lmdb-index \ - --transform-for-static-hosting --hosting-base-path SwiftUsd - - - name: Commit updated swift-package and documentation - run: | - git config --global user.name 'github-actions[bot]' - git config --global user.email 'github-actions[bot]@users.noreply.github.com' - git commit -am "Publish documentation and update swift-package for SwiftUsd $SWIFTUSD_TAG" - git push - git tag $SWIFTUSD_TAG - git push origin $SWIFTUSD_TAG - env: - SWIFTUSD_TAG: ${{ inputs.swiftusd-tag }} - diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a2e14e6bd5..a51aae2c8a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -82,6 +82,7 @@ jobs: env: TARGET_PLATFORM: ${{ matrix.target_platform }} + XCODEBUILD_DESTINATION: ${{ matrix.xcodebuild_destination }} BUILD_SYSTEM: ${{ matrix.build_system }} CONFIG: ${{ matrix.config }} SWIFTUSD_REF: ${{ inputs.swiftusd-ref }} diff --git a/SwiftUsd.docc/AboutThisRepo/ChangeLog.md b/SwiftUsd.docc/AboutThisRepo/ChangeLog.md index 44431accbc..92b745d716 100644 --- a/SwiftUsd.docc/AboutThisRepo/ChangeLog.md +++ b/SwiftUsd.docc/AboutThisRepo/ChangeLog.md @@ -11,6 +11,11 @@ Changes to SwiftUsd ``` } +### TBD +Released TBD, based on OpenUSD TBD +- Added ci-at-desk, an internal tool for running CI workflows locally + + ### 6.1.0 Released 2026-03-18, based on OpenUSD v26.03 - Add support for Swift 6.3 diff --git a/SwiftUsd.docc/AboutThisRepo/Miscellaneous.md b/SwiftUsd.docc/AboutThisRepo/Miscellaneous.md index a06b553e87..6c93d92039 100644 --- a/SwiftUsd.docc/AboutThisRepo/Miscellaneous.md +++ b/SwiftUsd.docc/AboutThisRepo/Miscellaneous.md @@ -69,10 +69,13 @@ Landing page for GitHub. - `SwiftUsd/scripts` Parent directory for utility scripts used while maintaining `SwiftUsd`. Only needed by advanced users. + - `SwiftUsd/scripts/ci-at-desk` + Script for running CI workflows locally. See its README file for more information. + - `SwiftUsd/scripts/docc` Parent directory for documentation scripts. See [the cheatsheet]() for more information. - - `SwiftUsd/scripts/make-swift-package.zsh` + - `SwiftUsd/scripts/make-swift-package.zsh` Helper script that invokes make-swift-package. - `SwiftUsd/scripts/make-swift-package` diff --git a/SwiftUsd.docc/AboutThisRepo/ReleaseChecklist.md b/SwiftUsd.docc/AboutThisRepo/ReleaseChecklist.md index 6ac32aaff5..7f9e327dd9 100644 --- a/SwiftUsd.docc/AboutThisRepo/ReleaseChecklist.md +++ b/SwiftUsd.docc/AboutThisRepo/ReleaseChecklist.md @@ -42,6 +42,7 @@ Checklist for releasing new versions of SwiftUsd 1. [Building Locally]() 1. Links to vanilla OpenUSD source files on GitHub 1. Default openusd-ref in .github/workflows/build-swiftusd.yml + 1. ci-at-desk sample YAML config file 1. Pixar namespace 1. [Getting Started, "Using SwiftUsd"]() 1. [Getting Started, "Common Issues"]() @@ -55,6 +56,7 @@ Checklist for releasing new versions of SwiftUsd 1. project.pbxproj in SwiftUsdTests 1. project.pbxproj for each Xcode project in Examples 1. Package.swift for each Swift Package in Examples + 1. ci-at-desk sample YAML config file 1. If files have been added or removed, update [Miscellaneous, "Repo structure"]() 1. Update [Ongoing Work]() 1. Update the [Change Log]() diff --git a/SwiftUsd.docc/TechnicalDetails/BuildingLocally.md b/SwiftUsd.docc/TechnicalDetails/BuildingLocally.md index 201b8a6f2a..5d78b4a930 100644 --- a/SwiftUsd.docc/TechnicalDetails/BuildingLocally.md +++ b/SwiftUsd.docc/TechnicalDetails/BuildingLocally.md @@ -35,7 +35,8 @@ python3 build_scripts/build_usd.py \ --no-python \ --ignore-homebrew \ --build-target native \ - ~/SwiftUsd/openusd-builds/macOS + ~/SwiftUsd/openusd-builds/macOS \ + --build-args="USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map=$(realpath .)=OpenUSD\"" cd ~/SwiftUsd/openusd-source python3 build_scripts/build_usd.py \ @@ -44,7 +45,8 @@ python3 build_scripts/build_usd.py \ --no-python \ --ignore-homebrew \ --build-target iOS \ - ~/SwiftUsd/openusd-builds/iOS + ~/SwiftUsd/openusd-builds/iOS \ + --build-args="USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map=$(realpath .)=OpenUSD\"" cd ~/SwiftUsd/openusd-source python3 build_scripts/build_usd.py \ @@ -53,7 +55,8 @@ python3 build_scripts/build_usd.py \ --no-python \ --ignore-homebrew \ --build-target iOSSimulator \ - ~/SwiftUsd/openusd-builds/iOSSimulator + ~/SwiftUsd/openusd-builds/iOSSimulator \ + --build-args="USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map=$(realpath .)=OpenUSD\"" cd ~/SwiftUsd/openusd-source python3 build_scripts/build_usd.py \ @@ -62,7 +65,8 @@ python3 build_scripts/build_usd.py \ --no-python \ --ignore-homebrew \ --build-target visionOS \ - ~/SwiftUsd/openusd-builds/visionOS + ~/SwiftUsd/openusd-builds/visionOS \ + --build-args="USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map=$(realpath .)=OpenUSD\"" cd ~/SwiftUsd/openusd-source python3 build_scripts/build_usd.py \ @@ -71,7 +75,8 @@ python3 build_scripts/build_usd.py \ --no-python \ --ignore-homebrew \ --build-target visionOSSimulator \ - ~/SwiftUsd/openusd-builds/visionOSSimulator + ~/SwiftUsd/openusd-builds/visionOSSimulator \ + --build-args="USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map=$(realpath .)=OpenUSD\"" ``` > Note: Custom feature flags are experimental, and may have some restrictions. For example, building with Python will not work on Apple platforms, building with additional plugins may not work for app-bundlable packages, and packaging multiple Usd builds with incompatible feature flags may not work. diff --git a/scripts/ci-at-desk/.gitignore b/scripts/ci-at-desk/.gitignore new file mode 100644 index 0000000000..0023a53406 --- /dev/null +++ b/scripts/ci-at-desk/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/scripts/ci-at-desk/Package.swift b/scripts/ci-at-desk/Package.swift new file mode 100644 index 0000000000..9aa50b45f2 --- /dev/null +++ b/scripts/ci-at-desk/Package.swift @@ -0,0 +1,66 @@ +// swift-tools-version: 6.1 +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import PackageDescription + +#if os(macOS) +let excludeDir = [String]() +#else +let excludeDir = ["UI"] +#endif + +let package = Package( + name: "ci-at-desk", + platforms: [.macOS(.v15)], + products: [ + .library(name: "WorkflowDescription", targets: ["WorkflowDescription"]), + .library(name: "WorkflowRunning", targets: ["WorkflowRunning"]), + .executable(name: "ci-at-desk", targets: ["ci-at-desk"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.2.0"), + .package(url: "https://github.com/swiftlang/swift-subprocess.git", branch: "main"), + .package(url: "https://github.com/apple/swift-log", from: "1.6.0"), + .package(url: "https://github.com/jpsim/Yams.git", from: "6.0.1"), + ], + targets: [ + .target(name: "WorkflowDescription", + dependencies: [ + .product(name: "Logging", package: "swift-log"), + ]), + .target(name: "WorkflowRunning", + dependencies: [ + "WorkflowDescription", + .product(name: "Subprocess", package: "swift-subprocess"), + .product(name: "Yams", package: "Yams"), + ]), + + .executableTarget( + name: "ci-at-desk", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + "WorkflowRunning" + ], + exclude: excludeDir, + ), + ] +) diff --git a/scripts/ci-at-desk/README.md b/scripts/ci-at-desk/README.md new file mode 100644 index 0000000000..45c15f0ef3 --- /dev/null +++ b/scripts/ci-at-desk/README.md @@ -0,0 +1,118 @@ +# ci-at-desk + +A script for running SwiftUsd CI workflows locally + + + +## Using ci-at-desk + +ci-at-desk requires a YAML configuration file. Here is a sample YAML config file. +``` + workflow: Run-Tests-Once + + precheckouts: + - remote: git@github.com:apple/SwiftUsd + ref: 6.1.0 + path: precheckouts/SwiftUsd + - remote: git@github.com:apple/SwiftUsd-Tests + ref: 6.1.0 + path: precheckouts/SwiftUsd-Tests + + requiredPaths: + cache: cache + artifacts: artifacts + runnerRoot: runnerRoot + logging: logging + SwiftUsd: precheckouts/SwiftUsd + SwiftUsd-Tests: precheckouts/SwiftUsd-Tests + + ci-inputs: + build-targets: ALL + openusd-ref: v26.03 + swiftusd-ref: local + swiftusd-tests-ref: local + + optional: + PATH-prepend: /usr/local/bin +``` + +To use it: +1. Create an empty directory at `~/Desktop/ci-at-desk-runs` +2. Paste the contents of the YAML config file into `~/Desktop/ci-at-desk-runs/config.yaml` +3. Run `cd ~/SwiftUsd/scripts/ci-at-desk; swift run --configuration release ci-at-desk ~/Desktop/ci-at-desk-runs/config.yaml` + +Given this config file, ci-at-desk will: +1. Create these directories: + - `~/Desktop/ci-at-desk-runs/precheckouts` + - `~/Desktop/ci-at-desk-runs/cache` + - `~/Desktop/ci-at-desk-runs/artifacts` + - `~/Desktop/ci-at-desk-runs/runnerRoot` + - `~/Desktop/ci-at-desk-runs/logging` +2. Clone SwiftUsd and SwiftUsd-Tests into `~/Desktop/ci-at-desk-runs/precheckouts` +3. Run the CI test suite in parallel using `~/Desktop/ci-at-desk-runs/precheckouts/SwiftUsd` and `~/Desktop/ci-at-desk-runs/precheckouts/SwiftUsd-Tests` + + + +## YAML config file syntax + +### Paths + +Paths in the YAML config are interpreted differently based on their starting characters: +- Paths beginning with `/` are treated as absolute paths +- Paths beginning with `~/` have tilde expansion performed (i.e. `~/foo` becomes `/Users//foo`) +- All other paths are treated as paths relative to the directory containing the config file (i.e. `foo` in a config file at `/fizz/buzz` becomes `/fizz/foo`) + +### YAML fields + +`workflow`: string, the id of the workflow to run. Required. + +`precheckouts`: array of precheckout objects. Optional, defaults to empty array. +- `precheckouts[*].remote`: string of github remote to clone from, e.g. `git@github.com:apple/SwiftUsd`. Required. +- `precheckouts[*].ref`: string of ref to checkout after cloning, e.g. `6.1.0`. Required. +- `precheckouts[*].path`: string path to clone to, e.g. `precheckouts/SwiftUsd`. Required. + +`requiredPaths`: +- `requiredPaths.runnerRoot`: string path under which to create the directories for individual matrix instance runs. **If the directory already exists, it will be removed.** Required. +- `requiredPaths.cache`: string path under which to store cache objects between different CI runs. If the directory doesn't exist, it will be created. Required. +- `requiredPaths.artifacts`: string path under which to store artifacts created during a CI run. **If the directory already exists, it will be removed.** Required. +- `requiredPaths.logging`: string path under which to write log files during CI runs. **If the directory already exists, it will be removed.** Required. +- `requiredPaths.SwiftUsd`: string path containing a local SwiftUsd repo to use for CI runs. Required. +- `requiredPaths.SwiftUsd-Tests`: string path containing a local SwiftUsd-Tests repo to use for CI runs. Required. + +`ci-inputs[*]`: a map of variables that will get inserted into orchestrator/runner contexts under `inputs`. Optional, defaults to empty map. + +`max-parallelism`: +- `max-parallelism.jobs`: int limiting the maximum number of jobs that may run in parallel under an individual workflow, with <=0 meaning no limit. Optional, defaults to 0. +- `max-parallelism.matrices`: int limiting the maximum number of matrix instances that may run in parallel under an individual job, with <=0 meaning no limit. Optional, defaults to 0. +- `max-parallelism.ATDESK_SWIFTBUILD_JOBS`: int limiting the maximum number of jobs swift-build can spawn (passed as `--jobs N`), with <= 0 meaning no limit. Optional, defaults to 0. +- `max-parallelism.ATDESK_XCODEBUILD_JOBS`: int limiting the maximum number of jobs xcodebuild can spawn (passed as `-jobs N`), with <= 0 meaning no limit. Optional, defaults to 0. + +`skips[*]`: a map of strings to ints that will get inserted into orchestrator/runner contexts under `skips`. Optional, defaults to empty map. + +`optional:` +- `optional.PATH-prepend`: string that will be prepended to $PATH before running shell commands during step execution. Optional, defaults to empty string. +- `optional.ATDESK_IOS_XCODEBUILD_DESTINATION`: string for a `-destination` xcodebuild argument for a physical iOS device to use for CI test suite runs. Optional, defaults to empty string. e.g. "platform=iOS,name=My iPad Pro". +- `optional.ATDESK_VISIONOS_XCODEBUILD_DESTINATION`: string for a `-destination` xcodebuild argument for a physical visionOS device to use for CI test suite runs. Optional, defaults to empty string. e.g. "platform=visionOS,name=My Apple Vision Pro". +- `optional.ATDESK_DEVELOPMENT_TEAM`: string for a `DEVELOPMENT_TEAM=` xcodebuild build setting override to use for CI test suite runs on a physical iOS or visionOS device. Optional. + +### YAML Development Team: + +You can find a valid development team argument by running `security find-certificate -c $(security find-identity -vp codesigning | grep ')' | head -n 1 | sed -E 's/.*\((.*)\).*/\1/') -p | openssl x509 -subject | grep 'OU=' | sed -E 's/.*\/OU=([A-Za-z0-9_]+)\/.*/\1/'` +Or as a step by step process: +1. `security find-identity -vp codesigning` will print out 0 or more valid code signing identities in your keychain. +2. ` | grep ')' | head -n 1 |` will choose the first line that contains `)` +3. ` | sed -E 's/.*\((.*)\).*/\1/'` will extract the parenthesized section at the end of a line from the output in step 1. +4. `security find-certificate -c -p` will print out the certificate for the code sign identity from step 3. +5. ` | openssl x509 -subject` will print out the certificate as well fields like the User ID (UID), Common Name (CN), Organizational Unit (OU), Organization (O), and Country (C). +6. ` | sed -E 's/.*\/OU=([A-Za-z0-9_]+)\/.*/\1/` will extract the value of the OU field, i.e. the `DEVELOPMENT_TEAM` argument + + + +## Architecture + +ci-at-desk is structured into three targets: +- WorkflowDescription contains the types used for defining a workflow consisting of jobs and steps. +- WorkflowRunning contains the logic for orchestrating and running a workflow invocation. + "Orchestrating" refers to scheduling parallel subtasks (WorkflowOrchestrator kicks off JobOrchestrator instances, JobOrchestrator kicks off MatrixInstanceRunner instances), while "Running" refers to serially executing subtasks (MatrixInstanceRunner runs StepRunner instances in order, StepRunner performs the underlying work of an individual step, including dealing with caching). + `YamlConfig.swift` also lives here. +- ci-at-desk is the executable entry point. It can be used as a command line program, and on macOS it can also be used with the --ui flag to see a SwiftUI GUI version that automatically monitors the logs of the running workflow. The actual workflow definitions live in `ci_at_desk.swift`. \ No newline at end of file diff --git a/scripts/ci-at-desk/Sources/WorkflowDescription/WorkflowDescription.swift b/scripts/ci-at-desk/Sources/WorkflowDescription/WorkflowDescription.swift new file mode 100644 index 0000000000..0d3fe95f37 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowDescription/WorkflowDescription.swift @@ -0,0 +1,447 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Logging + +/// A workflow containing jobs +public struct Workflow: Sendable, Codable { + public let id: String + public let jobs: [Job] + + public init(id: String, jobs: [Job]) { + self.id = id + self.jobs = jobs + } + + public var loggerRepresentation: Logger.MetadataValue { + id.loggerRepresentation + } +} + +/// A job containing steps +public struct Job: Sendable, Codable { + public let id: String + public let if_: Expression + public let name: Expression? + public let needs: [String] + public let env: [String : Expression] + public let strategy: Strategy + public let continueOnError: Bool + public let outputs: [String : Expression] + + public enum Kind: Sendable, Codable { + case steps([Step]) + case workflow(Workflow) + } + public let kind: Kind + + public init(id: String, if_: Expression = .success, name: Expression? = nil, + needs: [String] = [], env: [String : Expression] = [:], + strategy: Strategy? = nil, continueOnError: Bool = false, + outputs: [String : Expression] = [:], kind: Kind) { + self.id = id + self.if_ = if_ + self.name = name + self.needs = needs + self.env = env + self.strategy = strategy ?? .empty + self.continueOnError = continueOnError + self.outputs = outputs + self.kind = kind + } + + public init(id: String, if_: Expression = .success, name: Expression? = nil, + needs: [String] = [], env: [String : Expression] = [:], + strategy: Strategy? = nil, continueOnError: Bool = false, + outputs: [String : Expression] = [:], steps: [Step]) { + self.init(id: id, if_: if_, name: name, + needs: needs, env: env, + strategy: strategy, continueOnError: continueOnError, + outputs: outputs, kind: .steps(steps)) + } + + public init(id: String, if_: Expression = .success, name: Expression? = nil, + needs: [String] = [], env: [String : Expression] = [:], + strategy: Strategy? = nil, continueOnError: Bool = false, + outputs: [String : Expression] = [:], workflow: Workflow) { + self.init(id: id, if_: if_, name: name, + needs: needs, env: env, + strategy: strategy, continueOnError: continueOnError, + outputs: outputs, kind: .workflow(workflow)) + } + + public struct Strategy: Sendable, Codable, CustomStringConvertible { + public let failFast: Bool + public let maxParallel: Expression + public let matrix: Expression + + public init(failFast: Bool = true, + maxParallel: Expression = 0, + matrix: Expression) { + self.failFast = failFast + self.maxParallel = maxParallel + self.matrix = matrix + } + + public var description: String { + "Strategy(failFast: \(failFast), maxParallel: \(maxParallel), matrix: \(matrix))" + } + + public static let empty = Strategy(matrix: ["include": [[:]]]) + + internal enum StrategyError: Error { + case matrixMustBeDictionary(Expression) + case matrixDictionaryMustNotBeEmpty(Expression) + case matrixIncludeCantBeMixedWithOtherKeys(Expression) + case matrixExcludeIsntSupportedYet(Expression) + case matrixIncludeMustBeArray(Expression) + case matrixIncludeElementMustBeDictionary(Expression) + case matrixValuesMustBeArrays(Expression) + } + } + + public var loggerRepresentation: Logger.MetadataValue { + id.loggerRepresentation + } +} + +/// The individual actions performed sequentially in a job +public struct Step: Sendable, Codable { + public let name: String + public let if_: Expression + public let id: String? + public let cache: Cache? + public let timeout: Duration + public let kind: Kind + + /// The underlying kind of step + public enum Kind: Sendable, Codable, CustomStringConvertible { + case runShellCommand([Expression]) + case saveArtifact(SaveArtifact) + case restoreArtifact(RestoreArtifact) + case sparseCheckout([String]) + case checkout(CheckoutKind) + + public enum CheckoutKind: Sendable, Codable, CustomStringConvertible { + case swiftUsd + case swiftUsd_tests + + public var description: String { + switch self { + case .swiftUsd: "SwiftUsd" + case .swiftUsd_tests: "SwiftUsd-Tests" + } + } + + public var loggerRepresentation: Logger.MetadataValue { + description.loggerRepresentation + } + } + + public var description: String { + switch self { + case .runShellCommand(let array): "runShellCommand: \(array)" + case .saveArtifact(let saveArtifact): "\(saveArtifact)" + case .restoreArtifact(let restoreArtifact): "\(restoreArtifact)" + case .sparseCheckout(let array): "SparseCheckout: \(array)" + case .checkout(let checkoutKind): "Checkout: \(checkoutKind)" + } + } + + public var loggerRepresentation: Logger.MetadataValue { + switch self { + case .runShellCommand(let e): ["kind" : "runShellCommand", "expressions" : .array(e.map(\.loggerRepresentation))] + case .saveArtifact(let s): ["kind" : "saveArtifact", "data" : s.loggerRepresentation] + case .restoreArtifact(let r): ["kind" : "restoreArtifact", "data" : r.loggerRepresentation] + case .sparseCheckout(let s): ["kind" : "sparseCheckout", "paths" : .array(s.map(\.loggerRepresentation))] + case .checkout(let c): ["kind" : "checkout", "repo" : c.loggerRepresentation] + } + } + } + + public struct SaveArtifact: Sendable, Codable, CustomStringConvertible { + public let name: Expression + public let path: Expression + public let allowNoFilesFound: Bool + + public var description: String { + "SaveArtifact(name: \(name), path: \(path), allowNoFilesFound: \(allowNoFilesFound))" + } + + public init(name: Expression, path: Expression, allowNoFilesFound: Bool = false) { + self.name = name + self.path = path + self.allowNoFilesFound = allowNoFilesFound + } + + public var loggerRepresentation: Logger.MetadataValue { + ["name" : name.loggerRepresentation, + "path" : path.loggerRepresentation, + "allowNoFilesFound" : allowNoFilesFound.loggerRepresentation] + } + } + + public struct RestoreArtifact: Sendable, Codable, CustomStringConvertible { + public let path: Expression + public let pattern: Expression + public let requiresIndependentCopy: Bool + + public var description: String { + "RestoreArtifact(path: \(path), pattern: \(pattern), requiresIndependentCopy: \(requiresIndependentCopy))" + } + + public init(path: Expression, pattern: Expression, requiresIndependentCopy: Bool = false) { + self.path = path + self.pattern = pattern + self.requiresIndependentCopy = requiresIndependentCopy + } + + public var loggerRepresentation: Logger.MetadataValue { + ["path" : path.loggerRepresentation, + "pattern" : pattern.loggerRepresentation, + "requiresIndependentCopy" : requiresIndependentCopy.loggerRepresentation] + } + } + + public init(name: String, if_: Expression = .success, id: String?, cache: Cache?, timeout: Duration, kind: Kind) { + self.name = name + self.if_ = if_ + self.id = id + self.cache = cache + self.timeout = timeout + self.kind = kind + } + + public init(name: String, if_: Expression = .success, id: String? = nil, cache: Cache? = nil, timeout: Duration = .hours(1), run: Expression...) { + self.init(name: name, if_: if_, id: id, cache: cache, timeout: timeout, kind: .runShellCommand(run)) + } + + public init(name: String, if_: Expression = .success, timeout: Duration = .hours(1), saveArtifact: SaveArtifact) { + self.init(name: name, if_: if_, id: nil, cache: nil, timeout: timeout, kind: .saveArtifact(saveArtifact)) + } + + public init(name: String, if_: Expression = .success, timeout: Duration = .hours(1), restoreArtifact: RestoreArtifact) { + self.init(name: name, if_: if_, id: nil, cache: nil, timeout: timeout, kind: .restoreArtifact(restoreArtifact)) + } + + public init(name: String, if_: Expression = .success, timeout: Duration = .hours(1), sparseCheckout: [String]) { + self.init(name: name, if_: if_, id: nil, cache: nil, timeout: timeout, kind: .sparseCheckout(sparseCheckout)) + } + + public init(name: String, if_: Expression = .success, timeout: Duration = .hours(1), checkout: Kind.CheckoutKind) { + self.init(name: name, if_: if_, id: nil, cache: nil, timeout: timeout, kind: .checkout(checkout)) + } + + + public struct Cache: Sendable, Codable, CustomStringConvertible { + public let key: Expression + public let path: Expression + public let requiresIndependentCopy: Bool + + public var description: String { + "Cache(key: \(key), path: \(path), requiresIndependentCopy: \(requiresIndependentCopy))" + } + + public init(key: Expression, path: Expression, requiresIndependentCopy: Bool = false) { + self.key = key + self.path = path + self.requiresIndependentCopy = requiresIndependentCopy + } + + public var loggerRepresentation: Logger.MetadataValue { + ["key" : key.loggerRepresentation, + "path" : path.loggerRepresentation, + "requiresIndependentCopy" : requiresIndependentCopy.loggerRepresentation] + } + } + + public var loggerRepresentation: Logger.MetadataValue { + ["name" : name.loggerRepresentation, + "kind" : kind.loggerRepresentation] + } +} + +/// A dynamically typed expression (int, string, array, or dictionary) +public enum Expression: Sendable, Codable, + ExpressibleByIntegerLiteral, ExpressibleByStringLiteral, + ExpressibleByArrayLiteral, ExpressibleByDictionaryLiteral, CustomStringConvertible { + case int(Int) + case string(String) + case array([Expression]) + case dictionary([String : Expression]) + + public static let success: Expression = "${{ success() }}" + + public var description: String { + switch self { + case .int(let int): "\(int)" + case .string(let string): "'\(string)'" + case .array(let array): "\(array)" + case .dictionary(let dictionary): "\(dictionary)" + } + } + + public init(integerLiteral value: Int) { + self = .int(value) + } + + public init(stringLiteral value: String) { + self = .string(value) + } + + public init(arrayLiteral elements: Expression...) { + self = .array(elements) + } + + public init(dictionaryLiteral elements: (String, Expression)...) { + self = .dictionary(.init(uniqueKeysWithValues: elements)) + } + + public var asInt: Int? { + switch self { + case let .int(x): x + default: nil + } + } + + public var asString: String? { + switch self { + case let .string(x): x + default: nil + } + } + + public var asArray: [Expression]? { + switch self { + case let .array(x): x + default: nil + } + } + + public var asDictionary: [String : Expression]? { + switch self { + case let .dictionary(x): x + default: nil + } + } + + public var coerceToString: String { + switch self { + case let .string(s): s + case let .int(i): String(i) + case let .array(a): "[\(a.map(\.coerceToString).joined(separator: ", "))]" + case let .dictionary(d): "[\(d.map { "\($0) : \($1.coerceToString)" }.joined(separator: ", "))]" + } + } + + public func encode(to encoder: any Encoder) throws { + var container = encoder.singleValueContainer() + + switch self { + case let .string(s): try container.encode(s) + case let .int(i): try container.encode(i) + case let .array(a): try container.encode(a) + case let .dictionary(d): try container.encode(d) + } + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.singleValueContainer() + if let s = try? container.decode(String.self) { + self = .string(s) + } else if let i = try? container.decode(Int.self) { + self = .int(i) + } else if let a = try? container.decode([Expression].self) { + self = .array(a) + } else if let d = try? container.decode([String : Expression].self) { + self = .dictionary(d) + } else { + throw DecodingError.dataCorrupted(.init(codingPath: decoder.codingPath, debugDescription: "Not a valid Expression")) + } + } + + public static func fromJson(_ s: String) throws -> Expression { + try JSONDecoder().decode(Expression.self, from: s.data(using: .utf8)!) + } + + public var loggerRepresentation: Logger.MetadataValue { + switch self { + case .array(let x): .array(x.map(\.loggerRepresentation)) + case .dictionary(let d): d.loggerRepresentation + case .int(let i): i.loggerRepresentation + case .string(let s): s.loggerRepresentation + } + } +} + +extension [String : Expression] { + public var loggerRepresentation: Logger.MetadataValue { + var result = [String : Logger.MetadataValue]() + + var stack = [String]() + func handle(_ k: String, _ v: Expression) { + stack.append(k) + defer { stack.removeLast() } + + switch v { + case let .array(a): result[stack.joined(separator: ".")] = .array(a.map(\.loggerRepresentation)) + case let .int(i): result[stack.joined(separator: ".")] = i.loggerRepresentation + case let .string(s): result[stack.joined(separator: ".")] = s.loggerRepresentation + case let .dictionary(d): + for (k, v) in d { + handle(k, v) + } + } + } + + for (k, v) in self { + handle(k, v) + } + + return .dictionary(result) + } +} + +extension Int { + public var loggerRepresentation: Logger.MetadataValue { .string("\(self)") } +} + +extension String { + public var loggerRepresentation: Logger.MetadataValue { .string(self) } +} + +extension Bool { + public var loggerRepresentation: Logger.MetadataValue { .string(self ? "1" : "0") } +} + +extension URL { + public var loggerRepresentation: Logger.MetadataValue { .string(absoluteURL.path(percentEncoded: false)) } +} + +extension UUID { + public var loggerRepresentation: Logger.MetadataValue { .string(uuidString) } +} + +extension Duration { + public static func minutes(_ x: some BinaryInteger) -> Duration { .seconds(x * 60) } + public static func hours(_ x: some BinaryInteger) -> Duration { .minutes(x * 60) } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/Context.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/Context.swift new file mode 100644 index 0000000000..eee938cbd3 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/Context.swift @@ -0,0 +1,162 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import WorkflowDescription +import Logging +import Synchronization + +typealias Expression = WorkflowDescription.Expression + +/// A context object, used by orchestrators and runners for evaluating expressions +struct Context: ~Copyable, Sendable { + let yamlConfig: YamlConfig + private var impl: [String : Expression] = [:] + var everyJobSucceeded = true + var everyMatrixSucceded = true + var everyStepSucceeded = true + + init(yamlConfig: YamlConfig, impl: [String : Expression], everyJobSucceeded: Bool = true, everyMatrixSucceded: Bool = true, everyStepSucceeded: Bool = true) { + self.yamlConfig = yamlConfig + self.impl = impl + self.everyJobSucceeded = everyJobSucceeded + self.everyMatrixSucceded = everyMatrixSucceded + self.everyStepSucceeded = everyStepSucceeded + } + + init(yamlConfig: YamlConfig) { + self.yamlConfig = yamlConfig + } + + /// Creates an independent copy of `self` + func detachedCopy() -> Context { + .init(yamlConfig: yamlConfig, + impl: impl, + everyJobSucceeded: everyJobSucceeded, + everyMatrixSucceded: everyMatrixSucceded, + everyStepSucceeded: everyStepSucceeded) + } + + subscript(key: String) -> Expression? { + get { impl.nested(get: key) } + set { impl.nested(set: key, to: newValue) } + } + + mutating func augment(stepId: String, stepOutputs: [String : String], logger: Logger) { + logger.trace("Context.augment(stepId:stepOutputs:)") + for (k, v) in stepOutputs { + self["steps.\(stepId).outputs.\(k)"] = .string(v) + } + } + + mutating func augment(matrixInclude: [String : Expression], logger: Logger) { + logger.trace("Context.augment(matrixInclude:)") + for (k, v) in matrixInclude { + self["matrix.\(k)"] = v + } + } + + mutating func augment(jobId: String, jobOutputs: [String : Expression], logger: Logger) { + logger.trace("Context.augment(jobId:jobOutputs:)") + for (k, v) in jobOutputs { + self["needs.\(jobId).outputs.\(k)"] = v + } + } + + func hasFailures(for purpose: ExpressionEvaluator.EvaluationPurpose, logger: Logger) -> Bool { + logger.trace("Context.hasFailures(for:)") + let toConsider = switch purpose { + case .default_: [everyJobSucceeded, everyMatrixSucceded, everyStepSucceeded] + case .jobIf: [everyJobSucceeded] + case .stepIf: [everyStepSucceeded] + } + let result = !toConsider.allSatisfy { $0 } + logger.trace("Context.hasFailures(for:) returning", metadata: ["result" : result.loggerRepresentation]) + return result + } + + mutating func merge(other: borrowing Context, logger: Logger) { + logger.trace("Context.merge(other:)") + + self.impl.nestedMerge(other: other.impl) + self.everyJobSucceeded = self.everyJobSucceeded && other.everyJobSucceeded + self.everyMatrixSucceded = self.everyMatrixSucceded && other.everyMatrixSucceded + self.everyStepSucceeded = self.everyStepSucceeded && other.everyStepSucceeded + } + + var loggerRepresentation: Logger.MetadataValue { + ["everyJobSucceeded" : everyJobSucceeded.loggerRepresentation, + "everyMatrixSucceeded" : everyMatrixSucceded.loggerRepresentation, + "everyStepSucceeded" : everyStepSucceeded.loggerRepresentation, + "values" : impl.loggerRepresentation] + } +} + +extension [String : Expression] { + fileprivate func nested(get key: String) -> Expression? { + let parts = key.split(separator: ".", maxSplits: 1) + guard let value = self[String(parts[0])] else { + return nil + } + if parts.count == 1 { + return value + } else { + return value.asDictionary?.nested(get: String(parts[1])) + } + } + + fileprivate mutating func nested(set key: String, to newValue: Expression?) { + let parts = key.split(separator: ".", maxSplits: 1) + if parts.count == 1 { + self[key] = newValue + } else { + switch self[String(parts[0])] { + case .array, .int, .string: break + case nil: + var newDict = [String : Expression]() + newDict.nested(set: String(parts[1]), to: newValue) + self[String(parts[0])] = .dictionary(newDict) + case .dictionary(var d): + d.nested(set: String(parts[1]), to: newValue) + self[String(parts[0])] = .dictionary(d) + } + } + } + + fileprivate mutating func nestedMerge(other: [String : Expression]) { + for (k, v) in other { + if self[k] == nil { self[k] = v } + else { self[k]!.nestedMerge(other: v) } + } + } +} + +extension Expression { + fileprivate mutating func nestedMerge(other: Expression) { + switch (self, other) { + case (.int, _), (.string, _), (.array, _): self = other + case (var .dictionary(l), let .dictionary(r)): + l.nestedMerge(other: r) + self = .dictionary(l) + case (.dictionary, _): self = other + } + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/ExpressionEvaluator.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/ExpressionEvaluator.swift new file mode 100644 index 0000000000..e6b2cb69f5 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/ExpressionEvaluator.swift @@ -0,0 +1,293 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import WorkflowDescription +import Synchronization +import Logging + +// MARK: Interface + +/// A type for evaluation expressions, used by runners and orchestrators +internal struct ExpressionEvaluator { + let logger: Logger + init(logger: Logger) { + self.logger = logger + } + + internal enum EvaluationPurpose { + case default_ + case stepIf + case jobIf + + var loggerRepresentation: Logger.MetadataValue { + switch self { + case .default_: "default_" + case .stepIf: "stepIf" + case .jobIf: "jobIf" + } + } + } + + internal func evaluate(expression e: Expression, in context: borrowing Context, purpose: EvaluationPurpose) throws -> Expression { + logger.trace("ExpressionEvaluator.evaluate", metadata: [ + "expression": e.loggerRepresentation, + "context": context.loggerRepresentation, + "purpose": purpose.loggerRepresentation, + ]) + + let result: Expression + switch e { + case let .int(i): result = .int(i) + case let .array(x): result = try .array(x.map { try evaluate(expression: $0, in: context, purpose: purpose) }) + case let .dictionary(x): result = try .dictionary(x.mapValues { try evaluate(expression: $0, in: context, purpose: purpose) }) + case let .string(s): + let parts = try getEvaluationGroups(s) + let substitutedParts = try parts.map { try substitute(evaluationGroup: $0, in: context, purpose: purpose, logger: logger) } + result = join(substitutedParts: substitutedParts) + } + logger.trace("ExpressionEvaluator.evaluate returning", metadata: ["result" : result.loggerRepresentation]) + return result + } + + internal func evaluateAsString(_ expression: Expression, in context: borrowing Context) throws -> String { + logger.trace("ExpressionEvaluator.evaluateAsString", metadata: [ + "expression": expression.loggerRepresentation, + ]) + let evaluated = try evaluate(expression: expression, in: context, purpose: .default_) + if let result = evaluated.asString { + return result + } + throw ExpressionEvaluatorError.expressionEvaluatedToNonStringValue(evaluated) + } + + internal func evaluateAsBool(_ e: Expression, in context: borrowing Context, purpose: EvaluationPurpose) throws -> Bool { + logger.trace("ExpressionEvaluator.evaluateAsBool", metadata: [ + "expression": e.loggerRepresentation, + ]) + let evaluated = try evaluate(expression: e, in: context, purpose: purpose) + + if let result = evaluated.asInt { + return result != 0 + } + throw ExpressionEvaluatorError.expressionEvaluatedToNonIntValue(evaluated) + } + + internal func evaluateAsInt(_ e: Expression, in context: borrowing Context) throws -> Int { + logger.trace("ExpressionEvaluator.evaluateAsInt", metadata: [ + "expression": e.loggerRepresentation, + ]) + let evaluated = try evaluate(expression: e, in: context, purpose: .default_) + + if let result = evaluated.asInt { + return result + } + throw ExpressionEvaluatorError.expressionEvaluatedToNonIntValue(evaluated) + } + + private static let _cacheMutex = Mutex(()) + internal func evaluateCacheEntry(cache: Step.Cache, in context: borrowing Context, fileSystemHelper: borrowing FileSystemHelper) throws -> (key: URL, path: URL) { + logger.trace("ExpressionEvaluator.evaluateCacheEntry", metadata: [ + "cache": cache.loggerRepresentation, + ]) + let cacheDirectory = fileSystemHelper.cacheDirectory + + func map(key: String) throws -> String { + try Self._cacheMutex.withLock { _ in + let mappingUrl = cacheDirectory.appending(path: ".mapping.json") + var decodedJson = [String : String]() + if let data = try? Data(contentsOf: mappingUrl) { + try decodedJson = JSONDecoder().decode([String : String].self, from: data) + } + let mappedPath = decodedJson[key] ?? String(decodedJson.count) + decodedJson[key] = mappedPath + try JSONEncoder().encode(decodedJson).write(to: mappingUrl) + return mappedPath + + } + } + + let key = try evaluateAsString(cache.key, in: context) + let mappedKey = try map(key: key) + + let path = try evaluateAsString(cache.path, in: context) + + let resultKey = cacheDirectory.appending(path: mappedKey) + let resultPath = URL(fileURLWithPath: path) + + logger.trace("ExpressionEvaluator.evaluateCacheEntry returning", metadata: [ + "key" : resultKey.loggerRepresentation, + "path" : resultPath.loggerRepresentation, + ]) + + return (resultKey, resultPath) + } + + internal func evaluateAsMatrixList(from job: Job, in context: borrowing Context) throws -> (matrixList: [[String : Expression]], maxParallel: Int) { + logger.trace("ExpressionEvaluator.evaluateAsMatrixList", metadata: [ + "job": job.loggerRepresentation, + ]) + + let maxParallel = try evaluateAsInt(job.strategy.maxParallel, in: context) + + let matrix = job.strategy.matrix + let evaluated = try evaluate(expression: matrix, in: context, purpose: .default_) + + guard let d = evaluated.asDictionary else { + logger.error("Matrix must be dictionary: \(matrix)") + throw ExpressionEvaluatorError.matrixMustBeDictionary(matrix) + } + if d.isEmpty { throw ExpressionEvaluatorError.matrixDictionaryMustNotBeEmpty(matrix) } + if d["include"] != nil && d.count != 1 { + throw ExpressionEvaluatorError.matrixIncludeCantBeMixedWithOtherKeys(matrix) + } + if d["exclude"] != nil { + throw ExpressionEvaluatorError.matrixExcludeIsntSupportedYet(matrix) + } + + if let include = d["include"] { + guard let includeAsArray = include.asArray else { throw ExpressionEvaluatorError.matrixIncludeMustBeArray(matrix) } + var result = [[String : Expression]]() + for includeElement in includeAsArray { + guard let matrixInstance = includeElement.asDictionary else { + throw ExpressionEvaluatorError.matrixIncludeElementMustBeDictionary(matrix) + } + result.append(matrixInstance) + } + logger.trace("ExpressionEvaluator.evaluateAsMatrixList returning", metadata: [ + "result" : .array(result.map { $0.loggerRepresentation }) + ]) + return (result, maxParallel) + } else { + // Compute the cartesian product of d, and return it + + var result: [[String : Expression]] = [[:]] + for (k, v) in d { + guard let values = v.asArray else { + throw ExpressionEvaluatorError.matrixValuesMustBeArrays(matrix) + } + + // For each key, + // add each of its values to all of the existing + // cartesian products + result = values.flatMap { value in + result.map { existingItem in + var existingItem = existingItem + existingItem[k] = value + return existingItem + } + } + } + + logger.trace("ExpressionEvaluator.evaluateAsMatrixList returning", metadata: [ + "result" : .array(result.map { $0.loggerRepresentation }) + ]) + return (result, maxParallel) + } + } +} + +// MARK: Implementation +extension ExpressionEvaluator { + internal enum EvaluationGroup { + case rawString(String) + case substitutableString(String) + } + + private func getEvaluationGroups(_ s: String) throws -> [EvaluationGroup] { + var result = [EvaluationGroup]() + + var i = s.startIndex + var buffer = [Character]() + var isInSubstitutionMode = false + + func pushCharacter() { + buffer.append(s[i]) + s.formIndex(after: &i) + } + func pushBufferIfNonEmpty() { + if !buffer.isEmpty { + result.append(isInSubstitutionMode ? .substitutableString(String(buffer).trimmingCharacters(in: .whitespaces)) : .rawString(String(buffer))) + } + buffer = [] + isInSubstitutionMode.toggle() + } + func advanceIfPossible(withPrefix: String) -> Bool { + if s[i...].hasPrefix(withPrefix) { + i = s.index(i, offsetBy: withPrefix.count) + return true + } + return false + } + + while i < s.endIndex { + if isInSubstitutionMode { + if advanceIfPossible(withPrefix: "}}") { + pushBufferIfNonEmpty() + } else { + pushCharacter() + } + } else { + if advanceIfPossible(withPrefix: "${{") { + pushBufferIfNonEmpty() + } else { + pushCharacter() + } + } + } + pushBufferIfNonEmpty() + // pushBufferIfNonEmpty toggled the substitution mode, + // so if we're not in substitution mode _now_, it means that we just were + if !isInSubstitutionMode { + throw ExpressionEvaluatorError.unclosedSubstitution(s) + } + + return result + } + + private func substitute(evaluationGroup g: EvaluationGroup, in context: borrowing Context, purpose: EvaluationPurpose, logger: Logger) throws -> Expression { + switch g { + case let .rawString(s): return .string(s) + case let .substitutableString(s): + return try ExpressionSublanguageInterpreter.substitute(sublanguageString: s, in: context, purpose: purpose, logger: logger) + } + } + + private enum ExpressionEvaluatorError: Error, Sendable, Codable { + case unclosedSubstitution(String) + case expressionEvaluatedToNonStringValue(Expression) + case expressionEvaluatedToNonIntValue(Expression) + case matrixMustBeDictionary(Expression) + case matrixDictionaryMustNotBeEmpty(Expression) + case matrixIncludeCantBeMixedWithOtherKeys(Expression) + case matrixExcludeIsntSupportedYet(Expression) + case matrixIncludeMustBeArray(Expression) + case matrixIncludeElementMustBeDictionary(Expression) + case matrixValuesMustBeArrays(Expression) + } + + private func join(substitutedParts: [Expression]) -> Expression { + if substitutedParts.isEmpty { return .string("") } + if substitutedParts.count == 1 { return substitutedParts.first! } + + return .string(substitutedParts.reduce(into: "") { $0 += $1.coerceToString }) + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/ExpressionSublanguageInterpreter.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/ExpressionSublanguageInterpreter.swift new file mode 100644 index 0000000000..20a7248c06 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/ExpressionSublanguageInterpreter.swift @@ -0,0 +1,458 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Logging + +/// Interprets the expression sublanguage, i.e. the text between `${{` and `}}` in expressions +internal enum ExpressionSublanguageInterpreter { + static func substitute(sublanguageString s: String, in context: borrowing Context, purpose: ExpressionEvaluator.EvaluationPurpose, logger: Logger) throws -> Expression { + let tokens = try Token.tokenize(s) + let rawNode = try Parser.parse(tokens) + let augmentedNode = rawNode.augmented(for: purpose) + let result = try Interpreter.interpret(augmentedNode, in: context, for: purpose, logger: logger) + return result + } +} + +fileprivate enum Token { + case lparen + case rparen + case exclamationMark + case doublePipe + case doubleAmpersand + case period + case comma + case identifier(String) + + enum TokenError: Error { + case illegalCharacter(Character, String) + } + + static func tokenize(_ s: String) throws -> [Token] { + var i = s.startIndex + var buffer = [Character]() + var result = [Token]() + + func pushBufferIfNonEmpty() { + if !buffer.isEmpty { + result.append(.identifier(String(buffer))) + } + buffer = [] + } + func advanceIfPossible(withPrefix: String) -> Bool { + if s[i...].hasPrefix(withPrefix) { + i = s.index(i, offsetBy: withPrefix.count) + return true + } + return false + } + + while i < s.endIndex { + switch s[i] { + case "(": pushBufferIfNonEmpty(); result.append(.lparen) + case ")": pushBufferIfNonEmpty(); result.append(.rparen) + case "!": pushBufferIfNonEmpty(); result.append(.exclamationMark) + case ".": pushBufferIfNonEmpty(); result.append(.period) + case ",": pushBufferIfNonEmpty(); result.append(.comma) + case let c where c.isWhitespace: pushBufferIfNonEmpty() + case let c where "abcdefghijklmnopqrstuvwxyz_-".contains(c.lowercased()): buffer.append(c) + case _ where advanceIfPossible(withPrefix: "&&"): pushBufferIfNonEmpty(); result.append(.doubleAmpersand); continue + case _ where advanceIfPossible(withPrefix: "||"): pushBufferIfNonEmpty(); result.append(.doublePipe); continue + default: + throw TokenError.illegalCharacter(s[i], s) + } + s.formIndex(after: &i) + } + pushBufferIfNonEmpty() + return result + } +} + +fileprivate enum Parser { + indirect enum Pass1Node: Sendable { + case functionCall(String, [Pass1Node]) + case parenthesizedExpression([Pass1Node]) + case operator_(String) + case identifierChain(String) + case comma + + enum Pass1Error: Error { + case unexpectedRParen([Token], [IntermediateNode]) + case unexpectedPeriod([Token], [IntermediateNode]) + case unexpectedIdentifier([Token], [IntermediateNode]) + case unclosedPass1Node(IntermediateNode) + case identifierChainsCannotEndInPeriod(String) + } + + enum IntermediateNode: Sendable { + case openedFunctionCall(String) + case closedFunctionCall(String, [IntermediateNode]) + case openedParenthesizedExpression + case closedParenthesizedExpression([IntermediateNode]) + case operator_(String) + case identifierChain(String) + case comma + + var finalized: Pass1Node { + get throws { + switch self { + case .openedFunctionCall, .openedParenthesizedExpression: throw Pass1Error.unclosedPass1Node(self) + case let .closedFunctionCall(f, args): .functionCall(f, try args.map { try $0.finalized }) + case let .closedParenthesizedExpression(nodes): .parenthesizedExpression(try nodes.map { try $0.finalized }) + case let .operator_(x): .operator_(x) + case let .identifierChain(x): if x.last == "." { throw Pass1Error.identifierChainsCannotEndInPeriod(x) } else { .identifierChain(x) } + case .comma: .comma + } + } + } + } + + static func parseTokens(_ tokens: [Token]) throws -> [Pass1Node] { + var result = [IntermediateNode]() + tokenLoop: for token in tokens { + switch token { + case .lparen: + if case let .identifierChain(x) = result.last { + result.removeLast() + result.append(.openedFunctionCall(x)) + } else { + result.append(.openedParenthesizedExpression) + } + case .rparen: + for i in (0.. [Pass2Node] { + var result = [Pass2Node]() + + for node in input { + switch node { + case let .functionCall(name, args): + if case .comma = args.first { throw Pass2Error.unexpectedComma(args) } + if case .comma = args.last { throw Pass2Error.unexpectedComma(args) } + + var splitArgs: [[Pass1Node]] = [[]] + for arg in args { + switch arg { + case .comma: + if splitArgs.last!.isEmpty { throw Pass2Error.unexpectedComma(args) } + splitArgs.append([]) + default: splitArgs[splitArgs.count - 1].append(arg) + } + } + if splitArgs.last!.isEmpty { splitArgs.removeLast() } + let mappedSplitArgs = try splitArgs.map { try parseNodes($0) } + result.append(.functionCall(name, mappedSplitArgs.map { .parenthesizedExpression($0) })) + + case let .parenthesizedExpression(nodes): + result.append(.parenthesizedExpression(try parseNodes(nodes))) + + case let .operator_(op): result.append(.operator_(op)) + case let .identifierChain(x): result.append(.identifierChain(x)) + case .comma: throw Pass2Error.unexpectedComma(input) + } + } + + return result + } + } + + indirect enum Pass3Node: Sendable { + case identifierChain(String) + case functionCall(String, [Pass3Node]) + case parenthesizedExpression([Pass3Node]) + case infixOperator(InfixOperator) + case prefixOperator(PrefixOperator, Pass3Node) + + enum PrefixOperator { + case not + } + enum InfixOperator { + case and + case or + } + + enum Pass3Error: Error { + case unexpectedPrefixOperator(Pass2Node, [Pass2Node]) + case unknownOperator(Pass2Node, [Pass2Node]) + case unexpectedRecurseResult([Pass3Node]) + } + + static func parseNodes(_ input: [Pass2Node]) throws -> [Pass3Node] { + var result = [Pass3Node]() + + var i = 0 + while i < input.count { + let node = input[i] + switch node { + case let .identifierChain(x): result.append(.identifierChain(x)) + case let .functionCall(name, args): result.append(.functionCall(name, try parseNodes(args))) + case let .parenthesizedExpression(exprs): result.append(.parenthesizedExpression(try parseNodes(exprs))) + case let .operator_(x): + if x == "!" { + if i + 1 >= input.count { + throw Pass3Error.unexpectedPrefixOperator(node, input) + } else { + let recurseResult = try parseNodes([input[i + 1]]) + guard recurseResult.count == 1 else { throw Pass3Error.unexpectedRecurseResult(recurseResult) } + result.append(.prefixOperator(.not, recurseResult.first!)) + i += 1 + } + } else if x == "&&" { + result.append(.infixOperator(.and)) + } else if x == "||" { + result.append(.infixOperator(.or)) + } else { + throw Pass3Error.unknownOperator(node, input) + } + } + i += 1 + } + return result + } + } + + indirect enum FinalPassNode: Sendable { + case identifierChain(String) + case functionCall(String, [FinalPassNode]) + case infixOperator(FinalPassNode, Pass3Node.InfixOperator, FinalPassNode) + case prefixOperator(Pass3Node.PrefixOperator, FinalPassNode) + + enum IntermediateNode { + case finalPassNode(FinalPassNode) + case openInfixOperator(FinalPassNode, Pass3Node.InfixOperator) + } + + enum FinalPassError: Error { + case easyMapReturnedNil(Pass3Node) + case expectedOperator(Pass3Node, [Pass3Node]) + case unexpectedOperator(Pass3Node, [Pass3Node]) + case expectedOneNode([IntermediateNode], [Pass3Node]) + case expectedOperand([IntermediateNode]) + } + + static func parseNodes(_ input: [Pass3Node]) throws -> FinalPassNode { + func easyMap(_ x: Pass3Node) throws -> FinalPassNode? { + switch x { + case let .identifierChain(x): return .identifierChain(x) + case let .functionCall(name, args): + let mappedArgs = try args.map { try parseNodes([$0]) } + return .functionCall(name, mappedArgs) + + case let .parenthesizedExpression(x): return try parseNodes(x) + case .infixOperator: return nil + case let .prefixOperator(op, expr): return .prefixOperator(op, try parseNodes([expr])) + } + } + + var result = [IntermediateNode]() + // Not doing full blown C precedence for && vs ||, but reverse so we have left associativity. + for node in input.reversed() { + switch node { + case .identifierChain, .functionCall, .parenthesizedExpression, .prefixOperator: + guard let easyMapped = try easyMap(node) else { + throw FinalPassError.easyMapReturnedNil(node) + } + if case .openInfixOperator(let left, let op) = result.last { + result.removeLast() + result.append(.finalPassNode(.infixOperator(left, op, easyMapped))) + } else if result.isEmpty { + result.append(.finalPassNode(easyMapped)) + } else { + throw FinalPassError.expectedOperator(node, input) + } + + case let .infixOperator(op): + guard case let .finalPassNode(x) = result.last else { + throw FinalPassError.unexpectedOperator(node, input) + } + result.removeLast() + result.append(.openInfixOperator(x, op)) + } + } + + guard result.count == 1 else { throw FinalPassError.expectedOneNode(result, input) } + switch result.first! { + case let .finalPassNode(x): return x + case .openInfixOperator: throw FinalPassError.expectedOperand(result) + } + } + + func augmented(for purpose: ExpressionEvaluator.EvaluationPurpose) -> FinalPassNode { + switch purpose { + case .default_: return self + case .jobIf, .stepIf: break + } + + let statusCheckFunctionNames = ["success", "always", "cancelled", "failure"] + func hasStatusCheckFunction(_ x: FinalPassNode) -> Bool { + switch x { + case .identifierChain: false + case .functionCall(let name, let args): statusCheckFunctionNames.contains(name) || args.contains(where: { hasStatusCheckFunction($0) }) + case .infixOperator(let left, _, let right): hasStatusCheckFunction(left) || hasStatusCheckFunction(right) + case .prefixOperator(_, let expr): hasStatusCheckFunction(expr) + } + } + + if hasStatusCheckFunction(self) { return self } + return .infixOperator(self, .and, .functionCall("success", [])) + } + } + + static func parse(_ tokens: [Token]) throws -> FinalPassNode { + let pass1 = try Pass1Node.parseTokens(tokens) + let pass2 = try Pass2Node.parseNodes(pass1) + let pass3 = try Pass3Node.parseNodes(pass2) + let finalPass = try FinalPassNode.parseNodes(pass3) + return finalPass + } +} + +fileprivate enum Interpreter { + enum InterpreterError: Error { + case expressionEvaluatedToNonIntValue(Parser.FinalPassNode) + case badContextLookup(String) + case unexpectedArgumentCount(String, Int, [Expression]) + case unknownFunction(String) + case fromJsonRequiresString(Expression) + case fromJsonRequiresStringValue(String, Expression) + } + + static func interpret(_ node: Parser.FinalPassNode, in context: borrowing Context, for purpose: ExpressionEvaluator.EvaluationPurpose, logger: Logger) throws -> Expression { + switch node { + case let .identifierChain(x): return try lookup(string: x, in: context) + case let .functionCall(name, args): + let interpretedArgs = try args.map { try interpret($0, in: context, for: purpose, logger: logger) } + return try computeFunction(name, interpretedArgs, in: context, for: purpose, logger: logger) + case let .infixOperator(left, op, right): + guard let interpretedLeft = try interpret(left, in: context, for: purpose, logger: logger).asInt else { + throw InterpreterError.expressionEvaluatedToNonIntValue(left) + } + // Short circuit + switch op { + case .and: if interpretedLeft == 0 { return .int(0) } + case .or: if interpretedLeft != 0 { return .int(1) } + } + return try interpret(right, in: context, for: purpose, logger: logger) + + case let .prefixOperator(op, expr): + guard let interpretedExpr = try interpret(expr, in: context, for: purpose, logger: logger).asInt else { + throw InterpreterError.expressionEvaluatedToNonIntValue(expr) + } + switch op { + case .not: return .int(interpretedExpr == 0 ? 1 : 0) + } + } + } + + static func lookup(string s: String, in context: borrowing Context) throws -> Expression { + guard let result = context[s] else { + throw InterpreterError.badContextLookup(s) + } + return result + } + + static func computeFunction(_ name: String, _ args: [Expression], in context: borrowing Context, for purpose: ExpressionEvaluator.EvaluationPurpose, logger: Logger) throws -> Expression { + func guardArgCount(_ expected: Int) throws { + guard args.count == expected else { throw InterpreterError.unexpectedArgumentCount(name, expected, args) } + } + + switch name { + case "always": + try guardArgCount(0) + return .int(1) + + case "success": + try guardArgCount(0) + let hadFailure = context.hasFailures(for: purpose, logger: logger) + return hadFailure ? .int(0) : .int(1) + + case "failure": + try guardArgCount(0) + let hadFailure = context.hasFailures(for: purpose, logger: logger) + return hadFailure ? .int(1) : .int(0) + + case "cancelled": + try guardArgCount(0) + return .int(0) + + case "fromJson": + try guardArgCount(1) + guard let argAsString = args[0].asString else { throw InterpreterError.fromJsonRequiresString(args[0]) } + return try Expression.fromJson(argAsString) + + default: + throw InterpreterError.unknownFunction(name) + } + } +} + + diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/FileLogHandler.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/FileLogHandler.swift new file mode 100644 index 0000000000..0bedd42865 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/FileLogHandler.swift @@ -0,0 +1,246 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Logging +import Synchronization + +/// Log handler for logging to a file +internal struct FileLogHandler: LogHandler { + private let label: String + public var logLevel: Logger.Level = .info + public var metadata: Logger.Metadata = [:] + public var metadataProvider: Logger.MetadataProvider? + private var fileHandle: FileHandle + + public init(label: String, outputFile: URL) { + self.label = label + + // FileHandle(forWritingAtPath:) returns nil if the file doesn't already exist + try? FileManager.default.createDirectory(at: outputFile.deletingLastPathComponent(), withIntermediateDirectories: true) + FileManager.default.createFile(atPath: outputFile.path(percentEncoded: false), contents: nil) + self.fileHandle = FileHandle(forWritingAtPath: outputFile.path(percentEncoded: false))! + try! self.fileHandle.seekToEnd() + } + + internal func log( + level: Logger.Level, + message: Logger.Message, + metadata: Logger.Metadata?, + source: String, + file: String, + function: String, + line: UInt + ) { + let now = Date() + let timestamp = now.ISO8601Format(.iso8601WithTimeZone(includingFractionalSeconds: true)) + let levelString = level.rawValue.uppercased() + + // Merge handler metadata with message metadata + let combinedMetadata = Self.prepareMetadata( + base: self.metadata, + provider: self.metadataProvider, + explicit: metadata + ) + + // Format metadata + let metadataString = if let combinedMetadata { "[" + combinedMetadata.map { "\($0.key)=\($0.value)" }.joined(separator: ",") + "]" } + else { "" } + + // Create log line and print to console + let logLine = "\(label) \(timestamp) \(levelString) [\(metadataString)]: \(message)\n" + + fileHandle.write(Data(logLine.utf8)) + } + + internal subscript(metadataKey key: String) -> Logger.Metadata.Value? { + get { + return self.metadata[key] + } + set { + self.metadata[key] = newValue + } + } + + + static func prepareMetadata( + base: Logger.Metadata, + provider: Logger.MetadataProvider?, + explicit: Logger.Metadata? + ) -> Logger.Metadata? { + var metadata = base + + + let provided = provider?.get() ?? [:] + + + guard !provided.isEmpty || !((explicit ?? [:]).isEmpty) else { + // all per-log-statement values are empty + return metadata + } + + + if !provided.isEmpty { + metadata.merge(provided, uniquingKeysWith: { _, provided in provided }) + } + + + if let explicit = explicit, !explicit.isEmpty { + metadata.merge(explicit, uniquingKeysWith: { _, explicit in explicit }) + } + + + return metadata + } +} + +internal func fileLogger(label: String, outputFile: URL) -> Logger { + var result = Logger(label: label.replacingOccurrences(of: " ", with: "-"), + factory: { + let fileLogHandler = FileLogHandler(label: $0, outputFile: outputFile) + if InProcessLogNotificationHandler.enabled { + let inProcessHandler = InProcessLogNotificationHandler(label: $0, outputFile: outputFile) + return MultiplexLogHandler([inProcessHandler, fileLogHandler]) + } else { + return fileLogHandler + } + }) + result.logLevel = .trace + return result +} + +/// Log handler for treating log messages as notifications that other parts of the process can respond to. +/// Makes the UI more performant because it doesn't have to repeatedly parse log files from disk +public struct InProcessLogNotificationHandler: LogHandler { + static nonisolated(unsafe) public var enabled: Bool = false + + private struct Storage { + var messages: [URL : [Message]] = [:] + var subscribers: [URL : [([Message]) -> ()]] = [:] + } + + static private let storage: Mutex = .init(.init()) + + private let label: String + private let outputFile: URL + public var logLevel: Logger.Level = .info + public var metadata: Logger.Metadata = [:] + public var metadataProvider: Logger.MetadataProvider? + + public init(label: String, outputFile: URL) { + self.label = label + self.outputFile = outputFile + } + + public func log(level: Logger.Level, message: Logger.Message, metadata: Logger.Metadata?, source: String, file: String, function: String, line: UInt) { + let now = Date() + + // Merge handler metadata with message metadata + let combinedMetadata = Self.prepareMetadata( + base: self.metadata, + provider: self.metadataProvider, + explicit: metadata + ) + + // Format metadata + let metadataString = if let combinedMetadata { "[" + combinedMetadata.map { "\($0.key)=\($0.value)" }.joined(separator: ",") + "]" } + else { "" } + + let message = Message(label: label, timestamp: now, level: level, metadata: metadataString, message: message.description) + Self.post(message, to: outputFile) + } + + public subscript(metadataKey key: String) -> Logger.Metadata.Value? { + get { + return self.metadata[key] + } + set { + self.metadata[key] = newValue + } + } + + private static func prepareMetadata( + base: Logger.Metadata, + provider: Logger.MetadataProvider?, + explicit: Logger.Metadata? + ) -> Logger.Metadata? { + var metadata = base + + + let provided = provider?.get() ?? [:] + + + guard !provided.isEmpty || !((explicit ?? [:]).isEmpty) else { + // all per-log-statement values are empty + return metadata + } + + + if !provided.isEmpty { + metadata.merge(provided, uniquingKeysWith: { _, provided in provided }) + } + + + if let explicit = explicit, !explicit.isEmpty { + metadata.merge(explicit, uniquingKeysWith: { _, explicit in explicit }) + } + + + return metadata + } + + /// The structured message/notification sent by InProcessLogNotificationHandler + public struct Message: Sendable, Equatable { + public let label: String + public let timestamp: Date + public let level: Logger.Level + public let metadata: String + public let message: String + } + + public static func clearAllStorage() { + storage.withLock { $0 = .init() } + } + + public static func subscribe(url: URL, _ code: @escaping @Sendable ([Message]) -> ()) { + storage.withLock { storage in + if storage.subscribers[url] == nil { + storage.subscribers[url] = [] + } + storage.subscribers[url]!.append(code) + if let messages = storage.messages[url] { + code(messages) + } + } + } + + public static func post(_ message: Message, to url: URL) { + storage.withLock { storage in + if storage.messages[url] == nil { + storage.messages[url] = [] + } + storage.messages[url]!.append(message) + for subscriber in storage.subscribers[url] ?? [] { + subscriber([message]) + } + } + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/FileSystemHelper.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/FileSystemHelper.swift new file mode 100644 index 0000000000..c7fba7c57e --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/FileSystemHelper.swift @@ -0,0 +1,188 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import WorkflowDescription +import Logging + +/// A helper for working with the file system, used by orchestrators and runners +internal struct FileSystemHelper: ~Copyable { + // All + var cacheDirectory: URL { yamlConfig.cacheDirectory } + var artifactDirectory: URL { yamlConfig.artifactDirectory } + private var loggingDirectory: URL { + var result = yamlConfig.loggingDirectory + guard let workflow else { return result } + result = result.appending(path: "workflow-\(workflow.id)") + guard let job else { return result } + result = result.appending(path: "job-\(job.id)") + guard let matrixIndex else { return result } + result = result.appending(path: "matrix-\(matrixIndex)") + guard let step else { return result } + result = result.appending(path: "step-\(stepIndex!)-\(step.name)") + return result + } + var swiftUsdSrcDirectory: URL { yamlConfig.swiftUsdSrcDirectory } + var swiftUsdTestsSrcDirectory: URL { yamlConfig.swiftUsdTestsSrcDirectory } + + static func topLevelLogFile(yamlConfig: YamlConfig) -> URL { + yamlConfig.loggingDirectory.appending(path: "log.txt") + } + + // MatrixIndex + var runnerRootDirectory: URL { yamlConfig.runnerRootDirectory.appending(path: "runner-\(matrixRunnerId!.uuidString)") } + + var githubWorkspaceDirectory: URL { runnerRootDirectory.appending(path: "workspace") } + var runnerTempDirectory: URL { githubWorkspaceDirectory.appending(path: ".temp") } + var swiftUsdWorkspaceDirectory: URL { githubWorkspaceDirectory.appending(path: "SwiftUsd") } + var swiftUsdTestsWorkspaceDirectory: URL { githubWorkspaceDirectory.appending(path: "SwiftUsd-Tests") } + + + // Step + var githubOutputFile: URL { + guard step != nil else { fatalError() } + return loggingDirectory.appending(path: ".githuboutput.txt") + } + var githubStepSummaryFile: URL { + guard step != nil else { fatalError() } + return loggingDirectory.appending(path: ".githubstepsummary.txt") + } + var subprocessPathEnvironmentVariable: String { + guard step != nil else { fatalError() } + let existing = ProcessInfo.processInfo.environment["PATH"] ?? "" + return yamlConfig.pathPrepend + ":" + existing + } + + init(yamlConfig: YamlConfig, workflow: Workflow?, job: Job?, matrixIndex: Int?, matrixRunnerId: UUID?, stepIndex: Int?, step: Step?) { + self.yamlConfig = yamlConfig + self.workflow = workflow + self.job = job + self.matrixIndex = matrixIndex + self.matrixRunnerId = matrixRunnerId + self.stepIndex = stepIndex + self.step = step + + let label: String = + if let step { "step-\(step.name)" } + else if let matrixIndex { "matrix-\(matrixIndex)" } + else if let job { "job-\(job.id)" } + else if let workflow { "workflow-\(workflow.id)" } + else { "TopLevelLogger" } + + self.logger = fileLogger(label: label, outputFile: loggingDirectory.appending(path: "log.txt")) + self.expressionEvaluator = ExpressionEvaluator(logger: logger) + } + + private let yamlConfig: YamlConfig + private(set) var logger: Logger! + private(set) var expressionEvaluator: ExpressionEvaluator! + private let workflow: Workflow? + private let job: Job? + private let matrixIndex: Int? + private let matrixRunnerId: UUID? + private let stepIndex: Int? + private let step: Step? + + func logLoggingDirectory() { + logger.info("Log file: '\(loggingDirectory.appending(path: "log.txt").absoluteURL.path(percentEncoded: false))'") + } + + func ensureEmptyFileExists(url: URL) throws { + logger.trace("FileSystemHelper.ensureEmptyFileExists", metadata: ["url" : url.loggerRepresentation]) + if FileManager.default.fileExists(atPath: url.path(percentEncoded: false)) { + try FileManager.default.trashItem(at: url, resultingItemURL: nil) + // try FileManager.default.removeItem(at: url) + } + if !FileManager.default.fileExists(atPath: url.deletingLastPathComponent().path(percentEncoded: false)) { + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + } + try "".data(using: .utf8)!.write(to: url) + } + + func ensureEmptyDirectoryExists(url: URL) throws { + logger.trace("FileSystemHelper.ensureEmptyDirectoryExists", metadata: ["url" : url.loggerRepresentation]) + if FileManager.default.fileExists(atPath: url.path(percentEncoded: false)) { + try FileManager.default.trashItem(at: url, resultingItemURL: nil) + // try FileManager.default.removeItem(at: url) + } + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + } + + func ensureDirectoryExists(url: URL) throws { + logger.trace("FileSystemHelper.ensureDirectoryExists", metadata: ["url" : url.loggerRepresentation]) + var isDirectory = ObjCBool(false) + if FileManager.default.fileExists(atPath: url.path(percentEncoded: false), isDirectory: &isDirectory) { + if !isDirectory.boolValue { try ensureEmptyDirectoryExists(url: url) } + } else { + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + } + } + + func hasCacheEntry(cache: Step.Cache, context: borrowing Context) -> Bool { + logger.trace("FileSystemHelper.hasCacheEntry") + guard step != nil else { fatalError() } + do { + let (key, _) = try expressionEvaluator.evaluateCacheEntry(cache: cache, in: context, fileSystemHelper: self) + return FileManager.default.fileExists(atPath: key.path(percentEncoded: false)) + } catch { + logger.error("While evaluating cache entry: \(error)") + return false + } + } + + func restoreCacheEntry(cache: Step.Cache, context: borrowing Context) throws { + logger.trace("FileSystemHelper.restoreCacheEntry") + guard step != nil else { fatalError() } + let (key, path) = try expressionEvaluator.evaluateCacheEntry(cache: cache, in: context, fileSystemHelper: self) + do { + if !FileManager.default.fileExists(atPath: path.deletingLastPathComponent().path(percentEncoded: false)) { + try FileManager.default.createDirectory(at: path.deletingLastPathComponent(), withIntermediateDirectories: true) + } + + if cache.requiresIndependentCopy { + try FileManager.default.copyItem(at: key, to: path) + } else { + try FileManager.default.createSymbolicLink(at: path, withDestinationURL: key) + } + } catch { + throw FileSystemHelperError.restoreCache(error, cache) + } + } + + func saveCacheEntry(cache: Step.Cache, context: borrowing Context) throws { + logger.trace("FileSystemHelper.saveCacheEntry") + guard step != nil else { fatalError() } + let (key, path) = try expressionEvaluator.evaluateCacheEntry(cache: cache, in: context, fileSystemHelper: self) + do { + try FileManager.default.moveItem(at: path, to: key) + try FileManager.default.createSymbolicLink(at: path, withDestinationURL: key) + } catch { + throw FileSystemHelperError.saveCache(error, cache) + } + } + + private enum FileSystemHelperError: Error, Sendable { + case restoreCache(Error, Step.Cache) + case saveCache(Error, Step.Cache) + } +} + + diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/JobOrchestrator.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/JobOrchestrator.swift new file mode 100644 index 0000000000..dd11e08922 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/JobOrchestrator.swift @@ -0,0 +1,177 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import WorkflowDescription +import Logging +import Synchronization + + +/// A job-level orchestrator, responsible for kicking off matrix instances in parallel +// Class because it needs to be copied into the TaskGroup closures, +// and Mutex means it would have to be a noncopyable struct +internal final class JobOrchestrator: Sendable, OrchestratorProtocol { + internal static func run(job: Job, workflow: Workflow, context: inout Context, logger: Logger) async throws -> Bool { + logger.debug("JobOrchestrator.run", metadata: [ + "job": job.loggerRepresentation, + "workflow": workflow.loggerRepresentation, + "context": context.loggerRepresentation, + ]) + let instance = JobOrchestrator(workflow: workflow, job: job, context: context) + defer { + instance.sharedContext.withLock { context.merge(other: $0, logger: logger) } + logger.debug("JobOrchestrator.run returning", metadata: ["job" : job.loggerRepresentation]) + } + + do { + try await instance.run() + return instance.sharedContext.withLock { $0.everyMatrixSucceded } + } catch { + instance.logger.error(.init(stringLiteral: String(describing: error))) + logger.error(.init(stringLiteral: String(describing: error))) + throw error + } + } + + private let workflow: Workflow + private let job: Job + private let sharedContext: Mutex + private let maxMatrixParallelism: Int + let fileSystemHelper: FileSystemHelper + + private init(workflow: Workflow, job: Job, context: borrowing Context) { + self.workflow = workflow + self.job = job + self.maxMatrixParallelism = context.yamlConfig.maxMatrixParallelism + self.sharedContext = Mutex(context.detachedCopy()) + self.fileSystemHelper = FileSystemHelper(yamlConfig: context.yamlConfig, workflow: workflow, job: job, matrixIndex: nil, matrixRunnerId: nil, stepIndex: nil, step: nil) + } + + private func run() async throws { + logger.info("run() start") + defer { logger.info("run() end"); fileSystemHelper.logLoggingDirectory() } + + let jobName = if let nameExpr = job.name { + sharedContext.withLock { try? expressionEvaluator.evaluateAsString(nameExpr, in: $0) } ?? job.id + } else { + job.id + } + logger.info("name: \(jobName)") + + let (matrixList, strategyMaxParallel) = try sharedContext.withLock { try expressionEvaluator.evaluateAsMatrixList(from: job, in: $0) } + + // withThrowingTaskGroup requires a Copyable value + final class ContextRef: Sendable { + let value: Context + + init(value: consuming Context) { + self.value = value + } + } + + try await withThrowingTaskGroup(of: ContextRef.self) { taskGroup in + var matrixIndex = 0 + + let activeExclusivityKeys = Mutex(Set()) + + func queueNextMatrix() { + if matrixIndex >= matrixList.count { return } + let matrix = matrixList[matrixIndex] + logger.debug("Starting new matrix index \(matrixIndex + 1) of \(matrixList.count)") + _ = taskGroup.addTaskUnlessCancelled { @Sendable [job, workflow, matrixIndex, detachedContext = sharedContext.withLock { $0.detachedCopy() }] in + do { + var detachedContext = detachedContext.detachedCopy() + + guard let instanceExclusivityKeys = matrix["exclusivity_keys", default: []].asArray?.compactMap(\.asString), + instanceExclusivityKeys.count == matrix["exclusivity_keys", default: []].asArray?.count else { + throw JobOrchestratorError.invalidExclusivityKeys(matrix) + } + self.logger.debug("Instance \(matrixIndex + 1) has exclusivity keys \(instanceExclusivityKeys)") + + var hasLoggedAboutBeingBlocked = false + while true { + // Spin-lock, waiting 5 seconds. (Easier than setting up a proper semaphore system) + let canRun = activeExclusivityKeys.withLock { activeExclusivityKeys in + if activeExclusivityKeys.intersection(instanceExclusivityKeys).isEmpty { + self.logger.debug("Instance \(matrixIndex + 1) is inserting exclusivity keys \(instanceExclusivityKeys)") + activeExclusivityKeys.formUnion(instanceExclusivityKeys) + return true + } else { + return false + } + } + if canRun { + if hasLoggedAboutBeingBlocked { + self.logger.debug("Instance \(matrixIndex + 1) with exclusivity keys \(instanceExclusivityKeys) is newly unblocked") + } + break + } + if !hasLoggedAboutBeingBlocked { + hasLoggedAboutBeingBlocked = true + self.logger.debug("Instance \(matrixIndex + 1) with exclusivity keys \(instanceExclusivityKeys) is blocked") + } + try await Task.sleep(for: .seconds(5)) + } + // Make sure that even if an error is thrown later, + // we clear the activeExclusivityKeys + defer { + activeExclusivityKeys.withLock { + self.logger.debug("Instance \(matrixIndex + 1) is removing exclusivity keys \(instanceExclusivityKeys)") + $0.subtract(instanceExclusivityKeys) + } + } + + try await MatrixInstanceRunner.run(matrix: matrix, matrixIndex: matrixIndex, job: job, workflow: workflow, context: &detachedContext, logger: self.logger) + return ContextRef(value: detachedContext) + } catch { + self.logger.error(.init(stringLiteral: String(describing: error))) + throw error + } + } + matrixIndex += 1 + } + + // Start by trying to queue everything + var numberToQueueInitially = matrixList.count + if strategyMaxParallel != 0 { + // If the strategy has a limit, make sure we respect that + numberToQueueInitially = min(strategyMaxParallel, numberToQueueInitially) + } + if maxMatrixParallelism != 0 { + // If the yaml has a limit, make sure we respect that + numberToQueueInitially = min(maxMatrixParallelism, numberToQueueInitially) + } + + while matrixIndex < numberToQueueInitially { + queueNextMatrix() + } + + for try await returnedContext in taskGroup { + sharedContext.withLock { $0.merge(other: returnedContext.value, logger: logger) } + queueNextMatrix() + } + } + } + + enum JobOrchestratorError: Error { + case invalidExclusivityKeys([String : Expression]) + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/MatrixInstanceRunner.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/MatrixInstanceRunner.swift new file mode 100644 index 0000000000..d1116f1bc5 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/MatrixInstanceRunner.swift @@ -0,0 +1,144 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import WorkflowDescription +import Logging + +/// A matrix-instance level runner, responsible for running individual steps in order +internal struct MatrixInstanceRunner: ~Copyable, RunnerProtocol { + internal static func run(matrix: [String : Expression], matrixIndex: Int, job: Job, workflow: Workflow, context: inout Context, logger: Logger) async throws { + logger.debug("MatrixInstanceRunner.run \(matrixIndex + 1)", metadata: [ + "matrix" : matrix.loggerRepresentation, + "matrixIndex" : matrixIndex.loggerRepresentation, + "job": job.loggerRepresentation, + "workflow": workflow.loggerRepresentation, + "context": context.loggerRepresentation + ]) + + var instance = MatrixInstanceRunner(matrix: matrix, matrixIndex: matrixIndex, job: job, workflow: workflow, context: context) + defer { + context.merge(other: instance.context, logger: logger) + logger.debug("MatrixInstanceRunner.run \(matrixIndex + 1) returning") + } + + do { + try await instance.run() + } catch { + instance.logger.error(.init(stringLiteral: String(describing: error))) + logger.error(.init(stringLiteral: String(describing: error))) + throw error + } + } + + private let runnerID: UUID + private let matrixIndex: Int + private let job: Job + private let workflow: Workflow + private var context: Context + let fileSystemHelper: FileSystemHelper + + init(matrix: [String : Expression], matrixIndex: Int, job: Job, workflow: Workflow, context: borrowing Context) { + self.runnerID = UUID() + self.matrixIndex = matrixIndex + self.job = job + self.workflow = workflow + self.fileSystemHelper = FileSystemHelper(yamlConfig: context.yamlConfig, workflow: workflow, job: job, matrixIndex: matrixIndex, matrixRunnerId: runnerID, stepIndex: nil, step: nil) + self.context = context.detachedCopy() + self.context.augment(matrixInclude: matrix, logger: logger) + } + + mutating func run() async throws { + logger.info("run() start", metadata: ["runnerID" : .string(runnerID.uuidString)]) + logger.info("Runner workspace: \(fileSystemHelper.githubWorkspaceDirectory.path(percentEncoded: false))") + defer { logger.info("run() end"); fileSystemHelper.logLoggingDirectory() } + + let jobName = if let nameExpr = job.name { + try expressionEvaluator.evaluateAsString(nameExpr, in: context) + } else { + job.id + " (\(matrixIndex))" + } + logger.debug("name: \(jobName)") + + try fileSystemHelper.ensureEmptyDirectoryExists(url: fileSystemHelper.runnerRootDirectory) + try fileSystemHelper.ensureEmptyDirectoryExists(url: fileSystemHelper.githubWorkspaceDirectory) + try fileSystemHelper.ensureEmptyDirectoryExists(url: fileSystemHelper.runnerTempDirectory) + context["runner.swiftusd-path"] = .string(fileSystemHelper.swiftUsdWorkspaceDirectory.absoluteURL.path(percentEncoded: false)) + context["github.workspace"] = .string(fileSystemHelper.githubWorkspaceDirectory.absoluteURL.path(percentEncoded: false)) + + switch job.kind { + case .workflow(let workflow): + logger.debug("Will run workflow '\(workflow.id)'", metadata: [ + "workflow" : workflow.loggerRepresentation, + ]) + let runJobSucceeded = try await WorkflowOrchestrator.run(workflow: workflow, context: &context, logger: logger, runPrecheckouts: false) + if !runJobSucceeded { + context.everyMatrixSucceded = false + } + logger.debug("Workflow status: \(runJobSucceeded ? "success" : "failure")") + + case .steps(let steps): + logger.debug("Will run steps") + context.everyStepSucceeded = true + + for (stepIndex, step) in steps.enumerated() { + let ifIsSatisfied = try expressionEvaluator.evaluateAsBool(step.if_, in: context, purpose: .stepIf) + if !ifIsSatisfied { + logger.debug("Skipping step \(step.name) because step.if_ evaluated to false") + continue + } + + logger.debug("Will run step '\(step.name)' (\(stepIndex + 1)/\(steps.count))", metadata: [ + "step": step.loggerRepresentation, + ]) + let runStepSucceeded: Bool + do { + runStepSucceeded = try await StepRunner.run(stepIndex: stepIndex, step: step, matrixIndex: matrixIndex, matrixRunnerId: runnerID, job: job, workflow: workflow, context: &context, logger: logger) + } catch { + runStepSucceeded = false + logger.error("Got uncaught error from StepRunner.run: \(error)") + } + if !runStepSucceeded { + context.everyMatrixSucceded = false + context.everyStepSucceeded = false + } + logger.debug("Step '\(step.name)' status: \(runStepSucceeded ? "success" : "failure")") + } + } + + if !context.everyMatrixSucceded { + if job.continueOnError { + logger.debug("Had matrix failures, but continuing because job.continueOnError is set") + } else { + logger.debug("Had matrix failures, cancelling matrix run") + throw CancellationError() + } + } + + logger.trace("Will augment context with job outputs") + // Augment context + var jobOutputs = [String : Expression]() + for (k, v) in job.outputs { + jobOutputs[k] = try expressionEvaluator.evaluate(expression: v, in: context, purpose: .default_) + } + context.augment(jobId: job.id, jobOutputs: jobOutputs, logger: logger) + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/PrecheckoutRunner.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/PrecheckoutRunner.swift new file mode 100644 index 0000000000..5a1dd1845a --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/PrecheckoutRunner.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Logging +import WorkflowDescription +import Subprocess +import System + +/// Small runner for "precheckouts", i.e. cloning and checking out +/// git repositories before the bulk of workflow running begins +internal enum PrecheckoutRunner { + /* + example precheckouts section in yaml: + ``` + precheckouts: + - remote: git@github.com:apple/SwiftUsd + ref: 6.1.0 + path: precheckouts/SwiftUsd + - remote: git@github.com:apple/SwiftUsd-Tests + ref: 6.1.0 + path: precheckouts/SwiftUsd-Tests + ``` + */ + + static func run(yamlConfig: YamlConfig, logger: Logger) async throws { + // Important: don't use `run() start` and `run() end` in the logs, + // because that confuses the UI which searches for those strings for measuring + // run times + logger.trace("PrecheckoutRunner.run start") + defer { logger.trace("PrecheckoutRunner.run end") } + + for (i, p) in yamlConfig.precheckouts.enumerated() { + logger.debug("Processing precheckout \(p.remote) (\(i + 1) of \(yamlConfig.precheckouts.count))") + + if FileManager.default.fileExists(atPath: p.path.absoluteURL.path(percentEncoded: false)) { + logger.debug("\(p.path.absoluteURL.path(percentEncoded: false)) already exists, skipping clone") + + } else { + logger.info("git clone '\(p.remote)' '\(p.path.absoluteURL.path(percentEncoded: false))'") + let cloneResult = try await Subprocess.run(.name("git"), arguments: ["clone", p.remote, p.path.absoluteURL.path(percentEncoded: false)], output: .discarded) + guard cloneResult.terminationStatus.isSuccess else { + logger.error("Failed to clone") + throw PrecheckoutRunnerError.clone(cloneResult.terminationStatus) + } + } + + logger.info("cd '\(p.path.absoluteURL.path(percentEncoded: false))'; git checkout '\(p.ref)'") + let checkoutResult = try await Subprocess.run(.name("git"), arguments: ["checkout", p.ref], workingDirectory: FilePath(p.path), output: .discarded) + guard checkoutResult.terminationStatus.isSuccess else { + logger.error("Failed to checkout") + throw PrecheckoutRunnerError.checkout(checkoutResult.terminationStatus) + } + } + } + + enum PrecheckoutRunnerError: Error { + case clone(TerminationStatus) + case checkout(TerminationStatus) + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/RunnerProtocol.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/RunnerProtocol.swift new file mode 100644 index 0000000000..e7d0b9a421 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/RunnerProtocol.swift @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Logging + +// Common behavior for runners and orchestrators + +protocol RunnerProtocol: ~Copyable { + var fileSystemHelper: FileSystemHelper { get } +} + +extension RunnerProtocol where Self: ~Copyable { + var logger: Logger { fileSystemHelper.logger } + var expressionEvaluator: ExpressionEvaluator { .init(logger: logger) } +} + + +protocol OrchestratorProtocol { + var fileSystemHelper: FileSystemHelper { get } +} + +extension OrchestratorProtocol { + var logger: Logger { fileSystemHelper.logger } + var expressionEvaluator: ExpressionEvaluator { .init(logger: logger) } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/StepRunner.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/StepRunner.swift new file mode 100644 index 0000000000..17869f4912 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/StepRunner.swift @@ -0,0 +1,413 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Subprocess +import WorkflowDescription +import System +import Logging +import Synchronization + +/// A step level runner, responsible for executing a single step +internal struct StepRunner: ~Copyable, RunnerProtocol { + internal static func run(stepIndex: Int, step: Step, matrixIndex: Int, matrixRunnerId: UUID, job: Job, workflow: Workflow, context: inout Context, logger: Logger) async throws -> Bool { + logger.trace("StepRunner.run", metadata: [ + "step": step.loggerRepresentation, + "matrixIndex": matrixIndex.loggerRepresentation, + "matrixRunnerId": matrixRunnerId.loggerRepresentation, + "job": job.loggerRepresentation, + "workflow": workflow.loggerRepresentation, + "context": context.loggerRepresentation, + ]) + var instance = StepRunner(stepIndex: stepIndex, step: step, matrixIndex: matrixIndex, matrixRunnerId: matrixRunnerId, job: job, workflow: workflow, context: context) + defer { + context.merge(other: instance.context, logger: logger) + logger.trace("StepRunner.run returning") + } + + do { + return try await instance.run() + } catch { + instance.logger.error(.init(stringLiteral: String(describing: error))) + logger.error(.init(stringLiteral: String(describing: error))) + throw error + } + } + + private let step: Step + private let matrixIndex: Int + private let job: Job + private let workflow: Workflow + internal let fileSystemHelper: FileSystemHelper + private var context: Context + + init(stepIndex: Int, step: Step, matrixIndex: Int, matrixRunnerId: UUID, job: Job, workflow: Workflow, context: borrowing Context) { + self.step = step + self.matrixIndex = matrixIndex + self.job = job + self.workflow = workflow + self.fileSystemHelper = .init(yamlConfig: context.yamlConfig, workflow: workflow, job: job, matrixIndex: matrixIndex, matrixRunnerId: matrixRunnerId, stepIndex: stepIndex, step: step) + self.context = context.detachedCopy() + } + + internal mutating func run() async throws -> Bool { + logger.info("run() start") + logger.info("Runner workspace: \(fileSystemHelper.githubWorkspaceDirectory.path(percentEncoded: false))") + defer { logger.info("run() end"); fileSystemHelper.logLoggingDirectory() } + + logger.debug("name: \(step.name)") + + var hadCacheHit = false + if let cache = step.cache { + logger.debug("Checking cache", metadata: ["cache": cache.loggerRepresentation]) + if fileSystemHelper.hasCacheEntry(cache: cache, context: context) { + logger.debug("Cache hit, restoring") + try fileSystemHelper.restoreCacheEntry(cache: cache, context: context) + hadCacheHit = true + } else { + logger.debug("Cache miss, will save if step succeeds") + } + } + if hadCacheHit { return true } + + let stepSucceeded = switch step.kind { + case let .runShellCommand(runStep): try await run(runStep: runStep) + case let .saveArtifact(saveArtifact): try await run(saveArtifactStep: saveArtifact) + case let .restoreArtifact(restoreArtifact): try await run(restoreArtifactStep: restoreArtifact) + case let .sparseCheckout(sparseCheckout): try await run(sparseCheckoutStep: sparseCheckout) + case let .checkout(checkout): try await run(checkoutStep: checkout) + } + + if let cache = step.cache, stepSucceeded { + logger.debug("Saving to cache") + try fileSystemHelper.saveCacheEntry(cache: cache, context: context) + } + return stepSucceeded + } + + private mutating func run(runStep: [Expression]) async throws -> Bool { + logger.debug("run(runStep:) start") + defer { logger.debug("run(runStep:) end") } + try fileSystemHelper.ensureEmptyFileExists(url: fileSystemHelper.githubOutputFile) + try fileSystemHelper.ensureEmptyFileExists(url: fileSystemHelper.githubStepSummaryFile) + + var env = Subprocess.Environment.inherit + var envString = [String]() + + func updateEnv(_ k: Environment.Key, _ v: String) { + env = env.updating([k : v]) + if k == "PATH" { + envString.append("export \(k)='\(v)':$\(k)") + } else { + envString.append("export \(k)='\(v)'") + } + } + func updateEnv(_ k: Environment.Key, _ v: URL) { + updateEnv(k, v.absoluteURL.path(percentEncoded: false)) + } + + for (k, v) in job.env { + updateEnv(.init(stringLiteral: k), try expressionEvaluator.evaluateAsString(v, in: context)) + } + updateEnv("GITHUB_OUTPUT", fileSystemHelper.githubOutputFile) + updateEnv("GITHUB_STEP_SUMMARY", fileSystemHelper.githubStepSummaryFile) + updateEnv("GITHUB_WORKSPACE", fileSystemHelper.githubWorkspaceDirectory) + updateEnv("RUNNER_TEMP", fileSystemHelper.runnerTempDirectory) + updateEnv("PATH", fileSystemHelper.subprocessPathEnvironmentVariable) + + updateEnv("ATDESK_SWIFTBUILD_JOBS", String(context.yamlConfig.atDeskSwiftBuildJobs)) + updateEnv("ATDESK_XCODEBUILD_JOBS", String(context.yamlConfig.atDeskXcodebuildJobs)) + + if context.yamlConfig.atDeskIOSXcodebuildDestination != "" { + updateEnv("ATDESK_IOS_XCODEBUILD_DESTINATION", context.yamlConfig.atDeskIOSXcodebuildDestination) + } + if context.yamlConfig.atDeskVisionOSXcodebuildDestination != "" { + updateEnv("ATDESK_VISIONOS_XCODEBUILD_DESTINATION", context.yamlConfig.atDeskVisionOSXcodebuildDestination) + } + if context.yamlConfig.atDeskDevelopmentTeam != "" { + updateEnv("ATDESK_DEVELOPMENT_TEAM", context.yamlConfig.atDeskDevelopmentTeam) + } + + + logger.debug(.init(stringLiteral: runStep.map(\.coerceToString).joined(separator: " "))) + func globExpand(_ s: String) -> [String] { + guard s.hasSuffix("/*") else { return [s] } + let dirContents = (try? FileManager.default.contentsOfDirectory(atPath: String(s.dropLast(2)))) ?? [] + return dirContents.map { String(s.dropLast(1)) + $0 } + } + let evaluatedRunCommand = try runStep.map { try expressionEvaluator.evaluateAsString($0, in: context) } + .flatMap { globExpand($0) } + + logger.info(.init(stringLiteral: "cd \(FilePath(fileSystemHelper.swiftUsdWorkspaceDirectory)!)")) + for s in envString { + logger.info(.init(stringLiteral: s)) + } + logger.info(.init(stringLiteral: evaluatedRunCommand.joined(separator: " "))) + + + + + let runResult = try await withTimeout(step.timeout) { [env, logger, workingDirectory = FilePath(fileSystemHelper.swiftUsdWorkspaceDirectory)] in + try await Subprocess.run( + .name(evaluatedRunCommand.first!), + arguments: Arguments(Array(evaluatedRunCommand.dropFirst())), + environment: env, + workingDirectory: workingDirectory, + error: .combineWithOutput, + preferredBufferSize: 1, + ) { execution, output in + for try await line in output.lines() { + var line = line + if line.hasSuffix("\n") { line.removeLast() } + logger.debug(.init(stringLiteral: line)) + } + } + } onTimeout: { [logger, timeout = step.timeout] in + logger.error("Step timed out after \(timeout)") + } + + logger.debug("Subprocess run terminated with \(runResult.terminationStatus)") + if !runResult.terminationStatus.isSuccess { + throw StepRunnerError.runStepError(runResult.terminationStatus) + } + + logger.trace("Will augment context with step outputs") + // Augment context + var outputs = [String : String]() + let outputFileContents = try String(contentsOf: fileSystemHelper.githubOutputFile, encoding: .utf8) + for line in outputFileContents.components(separatedBy: .newlines) { + let line = line.trimmingCharacters(in: .whitespacesAndNewlines) + if line.isEmpty { continue } + let parts = line.split(separator: "=", maxSplits: 1) + guard parts.count == 2 else { + throw StepRunnerError.errorParsingOutputFile(line) + } + outputs[String(parts[0])] = String(parts[1]) + } + if let id = step.id { + context.augment(stepId: id, stepOutputs: outputs, logger: logger) + } else if !outputs.isEmpty { + logger.info("Warning! No step ID specified for step, but outputs were found.") + } + + return runResult.terminationStatus.isSuccess + } + + private func run(saveArtifactStep: Step.SaveArtifact) async throws -> Bool { + logger.debug("run(saveArtifactStep:)", metadata: ["input" : saveArtifactStep.loggerRepresentation]) + let (src, dest) = try _evaluateSaveArtifactStep(saveArtifactStep) + logger.debug("Saving \(src.path(percentEncoded: false)) to \(dest.path(percentEncoded: false))") + do { + // Move and symlink is faster than a copy + do { + try FileManager.default.moveItem(at: src, to: dest) + } catch { + if saveArtifactStep.allowNoFilesFound { /* pass */ } + else { throw error } + } + try FileManager.default.createSymbolicLink(at: src, withDestinationURL: dest) + // try FileManager.default.copyItem(at: src, to: dest) + return true + } catch { + throw StepRunnerError.saveArtifactError(error, saveArtifactStep) + } + } + + private func run(restoreArtifactStep: Step.RestoreArtifact) async throws -> Bool { + logger.debug("run(restoreArtifactStep:)", metadata: ["input" : restoreArtifactStep.loggerRepresentation]) + let (srcs, dest) = try _evaluateRestoreArtifactStep(restoreArtifactStep) + logger.debug("Restoring \(srcs.map { $0.path(percentEncoded: false) }) to \(dest.path(percentEncoded: false))") + do { + if srcs.count == 1 { + let src = srcs.first! + if !FileManager.default.fileExists(atPath: dest.deletingLastPathComponent().path(percentEncoded: false)) { + try FileManager.default.createDirectory(at: dest.deletingLastPathComponent(), withIntermediateDirectories: true) + } + + if restoreArtifactStep.requiresIndependentCopy { + try FileManager.default.copyItem(at: src, to: dest) + } else { + try FileManager.default.createSymbolicLink(at: dest, withDestinationURL: src) + } + } else { + for src in srcs { + if !FileManager.default.fileExists(atPath: dest.path(percentEncoded: false)) { + try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: true) + } + if restoreArtifactStep.requiresIndependentCopy { + try FileManager.default.copyItem(at: src, to: dest.appending(path: src.lastPathComponent)) + } else { + try FileManager.default.createSymbolicLink(at: dest.appending(path: src.lastPathComponent), withDestinationURL: src) + } + } + } + return true + } catch { + throw StepRunnerError.restoreArtifactError(error, restoreArtifactStep) + } + } + + private func run(sparseCheckoutStep: [String]) async throws -> Bool { + logger.debug("run(sparseCheckoutStep:)", metadata: ["input" : .array(sparseCheckoutStep.map { $0.loggerRepresentation })]) + do { + if !FileManager.default.fileExists(atPath: fileSystemHelper.swiftUsdWorkspaceDirectory.path(percentEncoded: false)) { + try FileManager.default.createDirectory(at: fileSystemHelper.swiftUsdWorkspaceDirectory, withIntermediateDirectories: true) + } + for path in sparseCheckoutStep { + let src = fileSystemHelper.swiftUsdSrcDirectory.appending(path: path) + let dest = fileSystemHelper.swiftUsdWorkspaceDirectory.appending(path: path) + if !FileManager.default.fileExists(atPath: dest.deletingLastPathComponent().path(percentEncoded: false)) { + try FileManager.default.createDirectory(at: dest.deletingLastPathComponent(), withIntermediateDirectories: true) + } + try FileManager.default.copyItem(at: src, to: dest) + } + return true + } catch { + throw StepRunnerError.sparseCheckoutError(error, sparseCheckoutStep) + } + } + + private func run(checkoutStep: Step.Kind.CheckoutKind) async throws -> Bool { + logger.debug("run(checkoutStep:)", metadata: ["input" : checkoutStep.loggerRepresentation]) + do { + switch checkoutStep { + case .swiftUsd: + // SwiftUsd repo can be pretty big on dev machines, and an exact copy has a lot of extra files that aren't needed and make the copy take longer + + let arguments: Subprocess.Arguments = [ + "-r", // rsync recursively + "--filter=:- .gitignore", // don't copy anything in gitignore + fileSystemHelper.swiftUsdSrcDirectory.absoluteURL.path(percentEncoded: false), + // rsync into the workspace dir + fileSystemHelper.swiftUsdWorkspaceDirectory.deletingLastPathComponent().absoluteURL.path(percentEncoded: false), + "--exclude=.git", // don't care about git history + "--exclude=docs", // don't care about compiled docs + "--exclude=SwiftUsd.doccarchive", // don't care about compiled docs + "--exclude=SwiftUsd.docc/.docc-build", // don't care about compiled docs, + "-l" // copy symlinks as symlinks + ] + + let runResult = try await Subprocess.run( + .name("rsync"), + arguments: arguments, + output: .discarded) + if !runResult.terminationStatus.isSuccess { + throw StepRunnerError.rsyncError(runResult.terminationStatus) + } + + // try FileManager.default.copyItem(at: fileSystemHelper.swiftUsdSrcDirectory, to: fileSystemHelper.swiftUsdWorkspaceDirectory) + case .swiftUsd_tests: + try FileManager.default.copyItem(at: fileSystemHelper.swiftUsdTestsSrcDirectory, to: fileSystemHelper.swiftUsdTestsWorkspaceDirectory) + } + return true + } catch { + throw StepRunnerError.checkoutStep(error, checkoutStep) + } + } + + enum StepRunnerError: Error, Sendable { + case errorParsingOutputFile(String) + case runStepError(Subprocess.TerminationStatus) + case saveArtifactError(Error, Step.SaveArtifact) + case restoreArtifactError(Error, Step.RestoreArtifact) + case sparseCheckoutError(Error, [String]) + case checkoutStep(Error, Step.Kind.CheckoutKind) + case rsyncError(Subprocess.TerminationStatus) + } +} + +// MARK: Artifact evaluation +extension StepRunner { + private func _evaluateSaveArtifactStep(_ saveArtifactStep: Step.SaveArtifact) throws -> (src: URL, dest: URL) { + logger.trace("_evaluateSaveArtifactStep") + let src = try expressionEvaluator.evaluateAsString(saveArtifactStep.path, in: context) + let dest = try expressionEvaluator.evaluateAsString(saveArtifactStep.name, in: context) + return (fileSystemHelper.githubWorkspaceDirectory.appending(path: src), fileSystemHelper.artifactDirectory.appending(path: dest)) + } + + private func _evaluateRestoreArtifactStep(_ restoreArtifactStep: Step.RestoreArtifact) throws -> (srcs: [URL], dest: URL) { + logger.trace("_evaluateRestoreArtifactStep") + let srcPattern = try expressionEvaluator.evaluateAsString(restoreArtifactStep.pattern, in: context) + let dest = try expressionEvaluator.evaluateAsString(restoreArtifactStep.path, in: context) + + var srcs = [URL]() + for potentialFile in try FileManager.default.contentsOfDirectory(at: fileSystemHelper.artifactDirectory, includingPropertiesForKeys: nil) { + if Self._artifactPattern(pattern: srcPattern, matches: potentialFile.lastPathComponent) { + srcs.append(potentialFile) + } + } + return (srcs, fileSystemHelper.githubWorkspaceDirectory.appending(path: dest)) + } + + /// Glob-matches the `pattern` against a given input string `matches` + private static func _artifactPattern(pattern p: String, matches m: String) -> Bool { + // Keep empty subsequences so we know where glob stars were + let parts = p.split(separator: "*", omittingEmptySubsequences: false) + + var range = m.startIndex..= parts.count { return partRange.upperBound == m.endIndex } + + range = partRange.lowerBound..(_ d: Duration, code: @Sendable @escaping () async throws -> T, onTimeout: @Sendable @escaping () -> () = {}) async throws -> T { + try await withThrowingTaskGroup { group in + group.addTask { + try await code() + } + + group.addTask { + try await Task.sleep(for: d) + onTimeout() + throw TimeoutError() + } + + + let result = try await group.next()! + group.cancelAll() + return result + } +} + +extension Subprocess.ExecutionResult: @retroactive @unchecked Sendable where Result: Sendable {} + diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/WorkflowOrchestrator.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/WorkflowOrchestrator.swift new file mode 100644 index 0000000000..db54dd1ad4 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/WorkflowOrchestrator.swift @@ -0,0 +1,186 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Subprocess +import System +import Synchronization +import WorkflowDescription +import Logging + +/// A workflow level orchestrator, responsible for kicking off jobs in parallel +// Class because it needs to be copied into the TaskGroup closures, +// and Mutex means it would have to be a noncopyable struct +public final class WorkflowOrchestrator: Sendable, OrchestratorProtocol { + /// The public entry point for external targets to start a CI run + public static func run(configFile: URL, workflows: [Workflow]) async throws -> Bool { + let yamlConfig = try YamlConfig(configFile: configFile) + guard let workflow = workflows.first(where: { $0.id == yamlConfig.workflowId }) else { + throw WorkflowOrchestratorError.noWorkflowFound(yamlConfig.workflowId, workflows.map(\.id)) + } + + // Set up file system + let fileSystemHelper = FileSystemHelper(yamlConfig: yamlConfig, workflow: workflow, job: nil, matrixIndex: nil, matrixRunnerId: nil, stepIndex: nil, step: nil) + try fileSystemHelper.ensureEmptyDirectoryExists(url: fileSystemHelper.artifactDirectory) + try fileSystemHelper.ensureDirectoryExists(url: fileSystemHelper.cacheDirectory) + try fileSystemHelper.ensureEmptyDirectoryExists(url: yamlConfig.runnerRootDirectory) + try fileSystemHelper.ensureEmptyDirectoryExists(url: yamlConfig.loggingDirectory) + + // Set up context + var context = Context(yamlConfig: yamlConfig) + context["github.run_id"] = .string(UUID().uuidString) + context["inputs"] = .dictionary(yamlConfig.inputs.mapValues { .string($0) }) + context["skips"] = .dictionary(yamlConfig.skips.mapValues { .int($0) }) + + return try await WorkflowOrchestrator.run(workflow: workflow, context: &context, logger: fileSystemHelper.logger, runPrecheckouts: true) + } + + private init(workflow: Workflow, context: borrowing Context) { + // Set up stored properties + self.workflow = workflow + self.maxJobParallelism = context.yamlConfig.maxJobParallelism + + self.jobIdsToStates = Mutex([String : JobState]()) + self.sharedContext = Mutex(context.detachedCopy()) + self.sharedContext.withLock { $0.everyJobSucceeded = true } + self.fileSystemHelper = .init(yamlConfig: context.yamlConfig, workflow: workflow, job: nil, matrixIndex: nil, matrixRunnerId: nil, stepIndex: nil, step: nil) + self.logger.trace("WorkflowOrchestrator.init") + } + + internal static func run(workflow: Workflow, context: inout Context, logger: Logger, runPrecheckouts: Bool) async throws -> Bool { + logger.trace("WorkflowOrchestrator.run() start", metadata: ["workflow" : workflow.loggerRepresentation, + "context" : context.loggerRepresentation]) + let instance = WorkflowOrchestrator(workflow: workflow, context: context) + defer { + instance.sharedContext.withLock { context.merge(other: $0, logger: logger) } + logger.trace("WorkflowOrchestrator.run() end") + } + + do { + return try await instance.run(runPrecheckouts: runPrecheckouts) + } catch { + instance.logger.error(.init(stringLiteral: String(describing: error))) + logger.error(.init(stringLiteral: String(describing: error))) + throw error + } + } + + private func run(runPrecheckouts: Bool) async throws -> Bool { + logger.info("run() start") + defer { logger.info("run() end"); fileSystemHelper.logLoggingDirectory() } + + logger.info("name: \(workflow.id)") + + if runPrecheckouts { + try await PrecheckoutRunner.run(yamlConfig: sharedContext.withLock { $0.yamlConfig }, logger: fileSystemHelper.logger) + } + + try await withThrowingTaskGroup(of: Void.self) { taskGroup in + try queueMoreJobs(taskGroup: &taskGroup) + + for try await _ in taskGroup { + try queueMoreJobs(taskGroup: &taskGroup) + } + } + + return sharedContext.withLock { $0.everyJobSucceeded } + } + + private enum JobState { + case running + case finished + } + + private let workflow: Workflow + private let maxJobParallelism: Int + private let jobIdsToStates: Mutex<[String : JobState]> + private let sharedContext: Mutex + let fileSystemHelper: FileSystemHelper + + private func getJobsToStart() throws -> [Job] { + logger.debug("getJobsToStart") + let result = try jobIdsToStates.withLock { jobIdsToStates in + let currentRunningCount = jobIdsToStates.count(where: { $0.value == .running }) + let maxAllowedToStart = maxJobParallelism == 0 ? Int.max : maxJobParallelism - currentRunningCount + + var toStart = [Job]() + for job in workflow.jobs { + if jobIdsToStates[job.id] != nil { continue } + if !job.needs.allSatisfy({ jobIdsToStates[$0] == .finished }) { continue } + toStart.append(job) + jobIdsToStates[job.id] = .running + if toStart.count == maxAllowedToStart { break } + } + + if currentRunningCount == 0 && toStart.isEmpty && workflow.jobs.count > jobIdsToStates.count { + throw WorkflowOrchestratorError.noJobsStart(workflow, jobIdsToStates) + } + + return toStart + } + logger.trace("getJobsToStart returning", metadata: [ + "jobs" : .array(result.map(\.loggerRepresentation)) + ]) + return result + } + + private func start(job: Job) async throws { + logger.debug("start(job:)", metadata: ["job" : job.loggerRepresentation]) + var detachedContext = sharedContext.withLock { $0.detachedCopy() } + defer { sharedContext.withLock { $0.merge(other: detachedContext, logger: logger) } } + + let ifIsSatisfied = try expressionEvaluator.evaluateAsBool(job.if_, in: detachedContext, purpose: .jobIf) + let runJobSucceeded: Bool + if ifIsSatisfied { + logger.trace("Will run job") + runJobSucceeded = try await JobOrchestrator.run(job: job, workflow: workflow, context: &detachedContext, logger: logger) + } else { + logger.trace("Skipping job because its if_ condition is false") + runJobSucceeded = true + } + + logger.trace("Marking job as finished", metadata: ["job" : job.loggerRepresentation]) + jobIdsToStates.withLock { $0[job.id] = .finished } + if !runJobSucceeded { + logger.info("Failing workflow \(workflow.id) because \(job.id) failed") + detachedContext.everyJobSucceeded = false + } + } + + private func queueMoreJobs(taskGroup: inout ThrowingTaskGroup) throws { + logger.debug("queueMoreJobs") + for job in try getJobsToStart() { + taskGroup.addTask { @Sendable in + do { + try await self.start(job: job) + } catch { + self.logger.error(.init(stringLiteral: String(describing: error))) + throw error + } + } + } + } + + private enum WorkflowOrchestratorError: Error { + case noJobsStart(Workflow, [String : JobState]) + case noWorkflowFound(String, [String]) + } +} diff --git a/scripts/ci-at-desk/Sources/WorkflowRunning/YamlConfig.swift b/scripts/ci-at-desk/Sources/WorkflowRunning/YamlConfig.swift new file mode 100644 index 0000000000..4c6ac56a80 --- /dev/null +++ b/scripts/ci-at-desk/Sources/WorkflowRunning/YamlConfig.swift @@ -0,0 +1,168 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Yams + +/// Holds the parsed contents of a YAML config file for ci-at-desk +public struct YamlConfig: Sendable { + public struct Precheckout: Sendable { + public let remote: String + public let ref: String + public let path: URL + + public init(remote: String, ref: String, path: URL) { + self.remote = remote + self.ref = ref + self.path = path + } + } + + // workflow: + public let workflowId: String + + // precheckouts: + public let precheckouts: [Precheckout] + + // requiredPaths: + public let runnerRootDirectory: URL + public let cacheDirectory: URL + public let artifactDirectory: URL + public let loggingDirectory: URL + public let swiftUsdSrcDirectory: URL + public let swiftUsdTestsSrcDirectory: URL + + // ci-inputs: + public let inputs: [String : String] + + // max-parallelism: + public let maxJobParallelism: Int + public let maxMatrixParallelism: Int + public let atDeskSwiftBuildJobs: Int + public let atDeskXcodebuildJobs: Int + + // skips: + public let skips: [String : Int] + + // optional: + public let pathPrepend: String + public let atDeskIOSXcodebuildDestination: String + public let atDeskVisionOSXcodebuildDestination: String + public let atDeskDevelopmentTeam: String + + private enum YamlConfigError: Error { + case missingWorkflow + case missingCacheDirectory + case missingRunnerRootDirectory + case missingArtifactDirectory + case missingSwiftUsdDirectory + case missingSwiftUsdTestsDirectory + case missingLoggingDirectory + case invalidPrecheckouts + } + + public init(configFile: URL) throws { + let yamlString = try String(contentsOf: configFile, encoding: .utf8) + let yamlBlob = try Yams.load(yaml: yamlString) + + func extract(_ key: String) -> Any? { + guard var top = yamlBlob as? [String : Any] else { return nil } + let parts = key.components(separatedBy: ".") + + for (i, part) in parts.enumerated() { + if i + 1 < parts.count { + guard let newTop = top[part] as? [String : Any] else { return nil } + top = newTop + } else { + return top[part] + } + } + + return nil + } + + func extract(_ key: String, as t: T.Type = T.self) -> T? { + extract(key) as? T + } + func extract(_ key: String, as t: T.Type = T.self, orThrow: YamlConfigError) throws -> T { + guard let result = extract(key, as: t) else { throw orThrow } + return result + } + + func formUrl(extractedString s: String) -> URL { + if s.starts(with: "~/") { + return FileManager.default.homeDirectoryForCurrentUser.appending(path: s.dropFirst(2)) + } else if s.starts(with: "/") { + return URL(fileURLWithPath: s) + } else { + return configFile.deletingLastPathComponent().appending(path: s) + } + } + + func extractUrl(_ key: String) -> URL? { + extract(key, as: String.self).map(formUrl(extractedString:)) + } + + func extractUrl(_ key: String, orThrow: YamlConfigError) throws -> URL { + guard let result = extractUrl(key) else { throw orThrow } + return result + } + + func extractPrecheckouts() throws -> [Precheckout] { + guard let raw = extract("precheckouts") else { return [] } + guard let casted = raw as? [[AnyHashable : Any]] else { throw YamlConfigError.invalidPrecheckouts } + + return try casted.map { blob in + guard let remote = blob["remote"] as? String else { throw YamlConfigError.invalidPrecheckouts } + guard let ref = blob["ref"] as? String else { throw YamlConfigError.invalidPrecheckouts } + guard let path = blob["path"] as? String else { throw YamlConfigError.invalidPrecheckouts } + return Precheckout(remote: remote, ref: ref, path: formUrl(extractedString: path)) + } + } + + self.workflowId = try extract("workflow", orThrow: .missingWorkflow) + + self.precheckouts = try extractPrecheckouts() + + self.cacheDirectory = try extractUrl("requiredPaths.cache", orThrow: .missingCacheDirectory) + self.runnerRootDirectory = try extractUrl("requiredPaths.runnerRoot", orThrow: .missingRunnerRootDirectory) + self.artifactDirectory = try extractUrl("requiredPaths.artifacts", orThrow: .missingArtifactDirectory) + self.loggingDirectory = try extractUrl("requiredPaths.logging", orThrow: .missingLoggingDirectory) + self.swiftUsdSrcDirectory = try extractUrl("requiredPaths.SwiftUsd", orThrow: .missingSwiftUsdDirectory) + self.swiftUsdTestsSrcDirectory = try extractUrl("requiredPaths.SwiftUsd-Tests", orThrow: .missingSwiftUsdTestsDirectory) + + + self.inputs = extract("ci-inputs") ?? [:] + + self.maxJobParallelism = extract("max-parallelism.jobs") ?? 0 + self.maxMatrixParallelism = extract("max-parallelism.matrices") ?? 0 + self.atDeskSwiftBuildJobs = extract("max-parallelism.ATDESK_SWIFTBUILD_JOBS") ?? 0 + self.atDeskXcodebuildJobs = extract("max-parallelism.ATDESK_XCODEBUILD_JOBS") ?? 0 + + self.skips = extract("skips") ?? [:] + + self.pathPrepend = extract("optional.PATH-prepend") ?? "" + self.atDeskIOSXcodebuildDestination = extract("optional.ATDESK_IOS_XCODEBUILD_DESTINATION") ?? "" + self.atDeskVisionOSXcodebuildDestination = extract("optional.ATDESK_VISIONOS_XCODEBUILD_DESTINATION") ?? "" + self.atDeskDevelopmentTeam = extract("optional.ATDESK_DEVELOPMENT_TEAM") ?? "" + } +} + diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/AppModel.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/AppModel.swift new file mode 100644 index 0000000000..052af09203 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/AppModel.swift @@ -0,0 +1,117 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + +import Foundation +import Observation +import Subprocess +import System +import Synchronization +import Logging +import WorkflowRunning + +/// The main app model for the SwiftUI version of ci-at-desk +@MainActor +@Observable class AppModel { + private var runner: Runner? + private(set) var parsedLogs: ParsedLogs? + + private(set) var synchronizedNow = Date() + private(set) var cancelTime: Date? + + // UI properties + var logLevel = Logger.Level.debug + var showMetadata: Bool = false + var showLabel: Bool = false + var timestampsMode: TimestampsMode = .none + var logLineDisplayLimit: Int = 60 + var rawLogs: Bool = false + var isShowingInvalidYamlAlert = false + var autoExpandLogs: AutoExpansionMode = .exceptOnSuccess + var autoExpandSteps: AutoExpansionMode = .exceptOnSuccess + + var isRunning: Bool { if let runner { runner.isRunning } else { false } } + + // Setting for how log timestamps are displayed in the UIp + enum TimestampsMode: String, CustomStringConvertible, CaseIterable, Identifiable, Hashable { + case none + case absolute + case relative + + var id: Self { self } + var description: String { rawValue } + } + + // Setting for when logs in disclosure groups are automatically expanded + enum AutoExpansionMode: CaseIterable, CustomStringConvertible, Identifiable, Hashable { + var id: Self { self } + + case never + case onErrorOnly + case onSuccessOnly + case whileInProgressOnly + case exceptOnError + case exceptOnSuccess + case exceptWhileInProgress + case always + + var description: String { + switch self { + case .never: "never" + case .onErrorOnly: "on error only" + case .onSuccessOnly: "on success only" + case .whileInProgressOnly: "while in progress only" + case .exceptOnError: "except on error" + case .exceptOnSuccess: "except on success" + case .exceptWhileInProgress: "except while in progress" + case .always: "always" + } + } + } + + init() { + Task { + while true { + parsedLogs?.refresh() + synchronizedNow = Date() + try? await Task.sleep(for: .seconds(0.5)) + } + } + } + + // MARK: UI actions + func run(yamlConfig: URL) { + InProcessLogNotificationHandler.clearAllStorage() + do { + runner = try Runner(yamlConfig: yamlConfig) + if let loggingDirectory = runner?.loggingDirectory { + parsedLogs = ParsedLogs(loggingDirectory) + } + cancelTime = nil + } catch { + isShowingInvalidYamlAlert = true + } + } + func cancel() { + runner?.cancel() + runner = nil + cancelTime = Date() + InProcessLogNotificationHandler.clearAllStorage() + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/ParsedLogs.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/ParsedLogs.swift new file mode 100644 index 0000000000..15f2e32e8e --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/ParsedLogs.swift @@ -0,0 +1,274 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import Logging +import WorkflowRunning + +// Finds directories under a given directory with a given prefix +fileprivate func dirs(under: URL, withPrefix: String) -> [URL] { + guard let contents = try? FileManager.default.contentsOfDirectory(at: under, includingPropertiesForKeys: nil) else { return [] } + let filteredContents = contents.filter { url in + guard url.lastPathComponent.hasPrefix(withPrefix) else { return false } + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path(percentEncoded: false), isDirectory: &isDirectory) else { return false } + guard isDirectory.boolValue else { return false } + return true + } + return filteredContents.sorted(by: { x, y in + // If the files look like an incrementing prefix, + // sort by the increment + let trimmedX = x.lastPathComponent.trimmingPrefix(withPrefix) + let trimmedY = y.lastPathComponent.trimmingPrefix(withPrefix) + + if let intX = Int(trimmedX), let intY = Int(trimmedY) { + return intX < intY + } + + // Fallback to lexicographic on the file name + return x.lastPathComponent < y.lastPathComponent + }) +} + +fileprivate func contentsOrEmpty(at: URL) -> String { + (try? String(contentsOf: at, encoding: .utf8)) ?? "" +} + +/// Top-level class containing logs for each workflow +@MainActor @Observable class ParsedLogs { + let url: URL + var workflows: [WorkflowLog] + + init(_ loggingDirectory: URL) { + url = loggingDirectory + workflows = dirs(under: url, withPrefix: "workflow-").map(WorkflowLog.init(_:)) + } + + var name: String { "ci-at-desk" } + var id: ObjectIdentifier { .init(self) } + + var isEnded: Bool = false + var containsErrors: Bool = false + var startTime: Date? = nil + var endTime: Date? = nil + + func refresh() { + let dirs = dirs(under: url, withPrefix: "workflow-") + let toAdd = dirs.filter { dir in !workflows.contains(where: { $0.url == dir })} + workflows.append(contentsOf: toAdd.map(WorkflowLog.init(_:))) + workflows = workflows.filter { dirs.contains($0.url) } + + for w in workflows { + w.refresh() + } + + if !isEnded && workflows.allSatisfy(\.isEnded) { isEnded = true } + if !containsErrors && workflows.contains(where: \.containsErrors) { containsErrors = true } + if startTime == nil { + startTime = workflows.compactMap(\.startTime).min() + } + if endTime == nil { + let endTimes = workflows.compactMap(\.endTime) + if endTimes.count == workflows.count { endTime = endTimes.max() } + } + } +} + +// Contains logs for a workflow and all its jobs +@MainActor @Observable class WorkflowLog: @MainActor Identifiable { + let url: URL + let logContents: LogContents + var jobs: [JobLog] + + init(_ workflowDirectory: URL) { + url = workflowDirectory + logContents = .init(workflowDirectory) + jobs = dirs(under: workflowDirectory, withPrefix: "job-").map(JobLog.init(_:)) + } + + var name: String { logContents.prettyName ?? url.lastPathComponent } + var id: ObjectIdentifier { .init(self) } + + var isEnded: Bool = false + var containsErrors: Bool = false + var startTime: Date? { logContents.startTime } + var endTime: Date? { logContents.endTime } + + func refresh() { + let dirs = dirs(under: url, withPrefix: "job-") + let toAdd = dirs.filter { dir in !jobs.contains(where: { $0.url == dir })} + jobs.append(contentsOf: toAdd.map(JobLog.init(_:))) + jobs = jobs.filter { dirs.contains($0.url) } + + for j in jobs { + j.refresh() + } + + if !isEnded && logContents.isEnded && jobs.allSatisfy(\.isEnded) { isEnded = true } + if !containsErrors && (logContents.containsErrors || jobs.contains(where: \.containsErrors)) { containsErrors = true } + } +} + +// Contains logs for a job and all its matrix instances +@MainActor @Observable class JobLog: @MainActor Identifiable { + let url: URL + let logContents: LogContents + var matrixInstances: [MatrixInstanceLog] + + init(_ jobDirectory: URL) { + url = jobDirectory + logContents = .init(jobDirectory) + matrixInstances = dirs(under: jobDirectory, withPrefix: "matrix-").map(MatrixInstanceLog.init(_:)) + } + + var name: String { logContents.prettyName ?? url.lastPathComponent } + var id: ObjectIdentifier { .init(self) } + + var isEnded: Bool = false + var containsErrors: Bool = false + var startTime: Date? { logContents.startTime } + var endTime: Date? { logContents.endTime } + + func refresh() { + let dirs = dirs(under: url, withPrefix: "matrix-") + let toAdd = dirs.filter { dir in !matrixInstances.contains(where: { $0.url == dir })} + matrixInstances.append(contentsOf: toAdd.map(MatrixInstanceLog.init(_:))) + matrixInstances = matrixInstances.filter { dirs.contains($0.url) } + + for m in matrixInstances { + m.refresh() + } + + if !isEnded && logContents.isEnded && matrixInstances.allSatisfy(\.isEnded) { isEnded = true } + if !containsErrors && (logContents.containsErrors || matrixInstances.contains(where: \.containsErrors)) { containsErrors = true } + } +} + +// Contains logs for a matrix instance and all its steps +@MainActor @Observable class MatrixInstanceLog: @MainActor Identifiable { + let url: URL + let logContents: LogContents + var steps: [StepLog] + + init(_ matrixInstanceDirectory: URL) { + url = matrixInstanceDirectory + logContents = .init(matrixInstanceDirectory) + steps = dirs(under: matrixInstanceDirectory, withPrefix: "step-").map(StepLog.init(_:)) + } + + var name: String { logContents.prettyName ?? url.lastPathComponent } + var id: ObjectIdentifier { .init(self) } + + var isEnded: Bool = false + var containsErrors: Bool = false + var startTime: Date? { logContents.startTime } + var endTime: Date? { logContents.endTime } + var runnerWorkspace: URL? { logContents.runnerWorkspace } + + func refresh() { + let dirs = dirs(under: url, withPrefix: "step-") + let toAdd = dirs.filter { dir in !steps.contains(where: { $0.url == dir })} + steps.append(contentsOf: toAdd.map(StepLog.init(_:))) + steps = steps.filter { dirs.contains($0.url) } + + for s in steps { + s.refresh() + } + + if !isEnded && logContents.isEnded && steps.allSatisfy(\.isEnded) { isEnded = true } + if !containsErrors && (logContents.containsErrors || steps.contains(where: \.containsErrors)) { containsErrors = true } + } +} + +// Contains logs, github outputs, and github summaries for a step +@MainActor @Observable class StepLog: @MainActor Identifiable { + let url: URL + let logContents: LogContents + var githubOutputContents: String + var githubSummaryContents: String + + init(_ stepDirectory: URL) { + url = stepDirectory + logContents = .init(stepDirectory) + githubOutputContents = "" + githubSummaryContents = "" + } + + var name: String { logContents.prettyName ?? url.lastPathComponent } + var id: ObjectIdentifier { .init(self) } + + var isEnded: Bool { logContents.isEnded } + var containsErrors: Bool { logContents.containsErrors } + var startTime: Date? { logContents.startTime } + var endTime: Date? { logContents.endTime } + var runnerWorkspace: URL? { logContents.runnerWorkspace } + + func refresh() { + githubOutputContents = contentsOrEmpty(at: url.appending(path: ".githuboutput.txt")) + githubSummaryContents = contentsOrEmpty(at: url.appending(path: ".githubstepsummary.txt")) + } +} + +// Contains structured log messages +@MainActor @Observable class LogContents { + var messages: [InProcessLogNotificationHandler.Message] = [] + var isEnded: Bool = false + var containsErrors: Bool = false + var startTime: Date? = nil + var endTime: Date? = nil + var prettyName: String? = nil + var runnerWorkspace: URL? + + typealias Line = InProcessLogNotificationHandler.Message + + init(_ directory: URL) { + InProcessLogNotificationHandler.subscribe(url: directory.appending(path: "log.txt")) { [weak self] messages in + Task { @MainActor in + self?.addMessages(messages) + } + } + } + + func addMessages(_ newMessages: [InProcessLogNotificationHandler.Message]) { + messages.append(contentsOf: newMessages) + + if !isEnded && newMessages.contains(where: { $0.message.contains("run() end") }) { isEnded = true } + if !containsErrors && newMessages.contains(where: { $0.level >= .error }) { containsErrors = true } + if startTime == nil, let m = newMessages.first(where: { $0.message.contains("run() start") }) { startTime = m.timestamp } + if endTime == nil, let m = newMessages.first(where: { $0.message.contains("run() end") }) { endTime = m.timestamp } + if prettyName == nil { + for m in newMessages { + if let wholeMatch = m.message.wholeMatch(of: #/name: (.*)/#) { + prettyName = String(wholeMatch.output.1) + break + } + } + } + if runnerWorkspace == nil { + for m in newMessages { + if let wholeMatch = m.message.wholeMatch(of: #/Runner workspace: (.*)/#) { + runnerWorkspace = URL(fileURLWithPath: String(wholeMatch.output.1)) + break + } + } + } + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/Runner.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/Runner.swift new file mode 100644 index 0000000000..427c5f1040 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Models/Runner.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import WorkflowRunning + +// UI class for asynchronously starting a WorkflowOrchestrator +// in a cancellable task +@MainActor +class Runner { + private var task: Task? + func cancel() { + task?.cancel() + task = nil + } + var isRunning: Bool { task != nil } + + deinit { + task?.cancel() + task = nil + } + + let loggingDirectory: URL + + init(yamlConfig: URL) throws { + self.loggingDirectory = try YamlConfig(configFile: yamlConfig).loggingDirectory + + task = Task.detached { + _ = try? await WorkflowOrchestrator.run(configFile: yamlConfig, workflows: CLIArgs.workflows) + Task { @MainActor in + self.task = nil + } + } + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/AutoExpandingDisclosureGroup.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/AutoExpandingDisclosureGroup.swift new file mode 100644 index 0000000000..d1d044745d --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/AutoExpandingDisclosureGroup.swift @@ -0,0 +1,40 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import SwiftUI + +/// A disclosure group with a customizable initial expansion state +struct InitialStateDisclosureGroup: View where Label: View, Content: View { + var label: Label + var content: () -> Content + @State private var isExpanded: Bool + + init(isInitiallyExpanded: Bool, @ViewBuilder content: @escaping () -> Content, @ViewBuilder label: () -> Label) { + self.label = label() + self._isExpanded = State(initialValue: isInitiallyExpanded) + self.content = content + } + + var body: some View { + DisclosureGroup(isExpanded: $isExpanded, content: content, label: { label }) + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/ContentView.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/ContentView.swift new file mode 100644 index 0000000000..df78cddab8 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/ContentView.swift @@ -0,0 +1,60 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import SwiftUI +import Logging + +/// The root view in ci-at-desk-ui +struct ContentView: View { + @State private var model = AppModel() + @State private var selection: HierarchicalTableView.Node? + + var body: some View { + NavigationSplitView { + VStack(alignment: .leading) { + Divider() + LogDisplayControls() + Divider() + HierarchicalTableView(parsedLogs: model.parsedLogs, selection: $selection) + } + } detail: { + VStack(alignment: .leading) { + if let selection { + NodeDetailView(node: selection) + } else { + SummaryView() + } + } + .padding(.leading, 4) + } + .navigationSplitViewStyle(.balanced) + .environment(model) + .onAppear { + if let initialConfigFile = ci_at_desk_UI.initialConfigFile { + model.run(yamlConfig: initialConfigFile) + } + } + .onDisappear { + model.cancel() + } + .alert("Invalid YAML config file", isPresented: $model.isShowingInvalidYamlAlert) {} + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/HierarchicalTableView.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/HierarchicalTableView.swift new file mode 100644 index 0000000000..4faaa44e39 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/HierarchicalTableView.swift @@ -0,0 +1,109 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import SwiftUI +import AppKit + +/// A table using OutlineGroups to display the workflows, jobs, matrix instances, and steps of a CI run +struct HierarchicalTableView: View { + let parsedLogs: ParsedLogs? + @Binding var selection: Node? + + enum Node: Hashable, Identifiable { + var id: Self { self } + + case none + case root(ParsedLogs) + case workflow(WorkflowLog) + case job(JobLog) + case matrixInstance(MatrixInstanceLog) + case step(StepLog) + + @MainActor var children: [Node]? { + switch self { + case .none: nil + case .root(let x): x.workflows.map { .workflow($0) } + case let .workflow(x): x.jobs.map { .job($0) } + case let .job(x): x.matrixInstances.map { .matrixInstance($0) } + case let .matrixInstance(x): x.steps.map { .step($0) } + case .step: nil + } + } + + func hash(into hasher: inout Hasher) { + switch self { + case .none: hasher.combine(0) + case let .root(x): hasher.combine(x.url) + case let .workflow(x): hasher.combine(x.url) + case let .job(x): hasher.combine(x.url) + case let .matrixInstance(x): hasher.combine(x.url) + case let .step(x): hasher.combine(x.url) + } + } + + static func ==(lhs: Self, rhs: Self) -> Bool { + switch (lhs, rhs) { + case (.none, .none): true + case let (.root(x), .root(y)) where x.url == y.url: true + case let (.workflow(x), .workflow(y)) where x.url == y.url: true + case let (.job(x), .job(y)) where x.url == y.url: true + case let (.matrixInstance(x), .matrixInstance(y)) where x.url == y.url: true + case let (.step(x), .step(y)) where x.url == y.url: true + default: false + } + } + } + + var body: some View { + List(selection: $selection) { + if let parsedLogs { + OutlineGroup(Node.root(parsedLogs), children: \.children) { node in + switch node { + case let .root(x): RunnerStatusLabel(x).showInFinderContextMenu(logs: x.url, runner: nil) + case let .workflow(x): RunnerStatusLabel(x).showInFinderContextMenu(logs: x.url.appending(path: "log.txt"), runner: nil) + case let .job(x): RunnerStatusLabel(x).showInFinderContextMenu(logs: x.url.appending(path: "log.txt"), runner: nil) + case let .matrixInstance(x): RunnerStatusLabel(x).showInFinderContextMenu(logs: x.url.appending(path: "log.txt"), runner: x.runnerWorkspace) + case let .step(x): RunnerStatusLabel(x).showInFinderContextMenu(logs: x.url.appending(path: "log.txt"), runner: x.runnerWorkspace) + case .none: Text("No parsed logs") + } + } + } + } + } +} + +fileprivate extension View { + func showInFinderContextMenu(logs: URL, runner: URL?) -> some View { + self.contextMenu { + Button("Show logs in Finder") { + NSWorkspace.shared.activateFileViewerSelecting([logs]) + } + if let runner { + Button("Show runner workspace in Finder") { + NSWorkspace.shared.activateFileViewerSelecting([runner]) + } + } + } + } +} + +// todo: add an inline CPU/memory usage chart under the HierarchicalTableView? diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/LogContentsView.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/LogContentsView.swift new file mode 100644 index 0000000000..f442f85305 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/LogContentsView.swift @@ -0,0 +1,256 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import SwiftUI +import Logging + +/// A view displaying the auto-updating contents of a log file +struct LogContentsView: View { + let logContents: LogContents + @Environment(AppModel.self) private var model + + var body: some View { + if model.rawLogs { + TextEditor(text: .constant(rawLogContents)) + } else { + NSTextViewRepresentable(logContents: logContents, model: model) + .frame(minWidth: minWidth) + .onChange(of: model.logLevel, updateTextViewSize) + .onChange(of: model.showMetadata, updateTextViewSize) + .onChange(of: model.showLabel, updateTextViewSize) + .onChange(of: model.timestampsMode, updateTextViewSize) + .onChange(of: model.logLineDisplayLimit, updateTextViewSize) + .onChange(of: logContents.messages, updateTextViewSize) + } + } + + @State private var minWidth = 1.0 + + // NSViewRepresentable.sizeThatFits lets SwiftUI ask AppKit views for their preferred size, + // but it doesn't let AppKit views proactively tell SwiftUI to call sizeThatFits again and + // layout the view again. So, we use `.frame(minWidth:)` on the NSViewRepresentable, + // changing the value within a small range to make SwiftUI redo layout when we know it needs + // to occur + func updateTextViewSize() { + minWidth += 1 + if minWidth > 10 { + minWidth = 1 + } + } + + var rawLogContents: String { + var toJoin = logContents.messages.compactMap { Self.buildLine(line: $0, model: model) } + + if model.logLineDisplayLimit > 0 && toJoin.count > model.logLineDisplayLimit { + toJoin = Array(toJoin.dropFirst(toJoin.count - model.logLineDisplayLimit)) + } + + return toJoin + .map { String($0.characters) } + .joined(separator: "\n") + } + + + fileprivate static func buildLine(line: LogContents.Line, model: AppModel) -> AttributedString? { + guard line.level >= model.logLevel else { return nil } + + var result = AttributedString() + if model.showLabel { + var label = AttributedString(line.label + " ") + label.foregroundColor = NSColor.tertiaryLabelColor + result.append(label) + } + + switch model.timestampsMode { + case .none: break + case .absolute: + let originalTimestamp = line.timestamp.ISO8601Format(.iso8601WithTimeZone(includingFractionalSeconds: true)) + var timestamp = AttributedString(originalTimestamp + " ") + timestamp.foregroundColor = NSColor.secondaryLabelColor + timestamp.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + result.append(timestamp) + case .relative: + let durationInSeconds = model.synchronizedNow.timeIntervalSince(line.timestamp) + let formattedTime = Duration.seconds(durationInSeconds).formatted( + .time(pattern: .hourMinuteSecond(padHourToLength: 2, fractionalSecondsLength: 1)) + ) + var timestamp = AttributedString("-" + formattedTime + " ") + timestamp.foregroundColor = NSColor.secondaryLabelColor + timestamp.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + result.append(timestamp) + } + + if model.showMetadata { + var metadata = AttributedString(line.metadata + " ") + metadata.foregroundColor = NSColor(.cyan) + result.append(metadata) + } + + // message + var message = AttributedString(line.message) + switch line.level { + case .trace: + message.foregroundColor = NSColor(.blue) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + case .debug: + message.foregroundColor = NSColor(.secondary) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + case .info: + message.foregroundColor = NSColor(.primary) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + case .notice: + message.foregroundColor = NSColor(.primary) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .bold) + case .warning: + message.foregroundColor = NSColor(.yellow) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + case .error: + message.foregroundColor = NSColor(.red) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .regular) + case .critical: + message.foregroundColor = NSColor(.red) + message.font = .monospacedSystemFont(ofSize: NSFont.systemFontSize, weight: .bold) + } + + result.append(message) + + return result + } +} + +// NSTextView wrapper for SwiftUI, for better performance than a single long Text instance +// in a scroll view +fileprivate struct NSTextViewRepresentable: NSViewRepresentable { + let logContents: LogContents + let model: AppModel + + func makeCoordinator() -> Coordinator { + .init(logContents: logContents, model: model) + } + + func makeNSView(context: Context) -> NSTextView { + context.coordinator.makeNSView() + } + + func updateNSView(_ textView: NSTextView, context: Context) { + context.coordinator.updateNSView(textView, logContents: logContents, model: model) + } + + func sizeThatFits(_ proposal: ProposedViewSize, nsView textView: NSTextView, context: Context) -> CGSize? { + context.coordinator.sizeThatFits(proposal, nsView: textView) + } + + static func dismantleNSView(_ nsView: NSTextView, coordinator: Coordinator) { + coordinator.dismantleNSView(nsView) + } +} + +extension NSTextViewRepresentable { + @MainActor class Coordinator: NSObject { + var logContents: LogContents + var model: AppModel + var textView: NSTextView! + + init(logContents: LogContents, model: AppModel) { + self.logContents = logContents + self.model = model + } + + @objc func frameDidChange() { + textView.textContainer?.size = textView.frame.size + textView.needsLayout = true + textView.needsDisplay = true + } + + func makeNSView() -> NSTextView { + textView = NSTextView() + textView.isEditable = false + textView.isSelectable = true + textView.backgroundColor = .clear + textView.textContainer?.lineBreakMode = .byWordWrapping + textView.postsFrameChangedNotifications = true + + NotificationCenter.default.addObserver(self, selector: #selector(frameDidChange), name: NSView.frameDidChangeNotification, object: textView) + + return textView + } + + func updateNSView(_ textView: NSTextView, logContents: LogContents, model: AppModel) { + self.logContents = logContents + self.model = model + + var toJoin = logContents.messages.compactMap { line in + LogContentsView.buildLine(line: line, model: model) + } + if model.logLineDisplayLimit > 0 && toJoin.count > model.logLineDisplayLimit { + toJoin = Array(toJoin.dropFirst(toJoin.count - model.logLineDisplayLimit)) + } + var attrString = AttributedString() + for (i, x) in toJoin.enumerated() { + attrString += x + if i + 1 < toJoin.count { + attrString += "\n" + } + } + + // Copy and restore the selected ranges because setAttributedString() clears the selection + let selectedRanges = textView.selectedRanges + textView.textStorage?.setAttributedString(NSAttributedString(attrString)) + textView.selectedRanges = selectedRanges + } + + func dismantleNSView(_ textView: NSTextView) { + NotificationCenter.default.removeObserver(self) + } + + func sizeThatFits(_ proposal: ProposedViewSize, nsView textView: NSTextView) -> CGSize? { + guard let textContainer = textView.textContainer else { return nil } + + let oldSize = textContainer.size + defer { textContainer.size = oldSize } + if let width = proposal.width { + if width == 0 { + textContainer.size.width = 10 + } else { + textContainer.size.width = width + } + } + textContainer.size.height = 10000000 + + textContainer.lineFragmentPadding = 0 + // Important, calling glyphRange makes the layoutManager do some internal calculations + // instead of just returning a default value for `usedRect(for:)` + _ = textView.layoutManager?.glyphRange(for: textContainer) + guard let usedRect = textView.layoutManager?.usedRect(for: textContainer) else { return nil } + + // TextKit uses 10000000 to represent an unrestricted size + var result = usedRect.size + if result.width == 10000000 { + result.width = proposal.width ?? 1000 + } + + return result + } + } +} + +extension NSFont: @unchecked @retroactive Sendable {} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/LogDisplayControls.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/LogDisplayControls.swift new file mode 100644 index 0000000000..c6f0f3b03d --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/LogDisplayControls.swift @@ -0,0 +1,127 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + +import Foundation +import SwiftUI +import Logging + +/// Controls for customizing how logs are displayed +struct LogDisplayControls: View { + @Environment(AppModel.self) private var model + @State private var isOpenPanelPresented = ci_at_desk_UI.initialConfigFile == nil + + private var openButton: some View { + Button { + isOpenPanelPresented = true + } label: { + Label("Open", systemImage: "square.and.arrow.down") + } + .fileImporter(isPresented: $isOpenPanelPresented, + allowedContentTypes: [.yaml], onCompletion: { result in + if let url = try? result.get() { + model.run(yamlConfig: url) + } else { + model.cancel() + } + }) + } + + private var stopButton: some View { + Button { + model.cancel() + } + label: { + Label("Stop", systemImage: "stop.fill") + } + .disabled(!model.isRunning) + } + + var body: some View { + @Bindable var model = model + + HStack { + Form { + if !model.isRunning { + openButton + } else { + stopButton + } + + Toggle("Raw logs", isOn: $model.rawLogs) + LogLevelPicker(logLevel: $model.logLevel) + TimestampsPicker(label: "Timestamp mode:", value: $model.timestampsMode) + AutoExpansionModePicker(label: "Auto-expand logs:", value: $model.autoExpandLogs) + } + + Form { + let nowString = ISO8601DateFormatter().string(from: model.synchronizedNow) + Text(nowString) + .monospaced() + + + TextField("Log line display limit:", value: $model.logLineDisplayLimit, format: .number) + Toggle("Show label", isOn: $model.showLabel) + Toggle("Show metadata", isOn: $model.showMetadata) + AutoExpansionModePicker(label: "Auto-expand steps:", value: $model.autoExpandSteps) + } + } + .padding() + } +} + +// MARK: Miscellaneous pickers + +fileprivate struct TimestampsPicker: View { + let label: String + @Binding var value: AppModel.TimestampsMode + + var body: some View { + Picker(label, selection: $value) { + ForEach(AppModel.TimestampsMode.allCases) { x in + Text("\(x.description)") + } + } + } +} + +fileprivate struct AutoExpansionModePicker: View { + let label: String + @Binding var value: AppModel.AutoExpansionMode + + var body: some View { + Picker(label, selection: $value) { + ForEach(AppModel.AutoExpansionMode.allCases) { x in + Text("\(x.description)") + } + } + } +} + +fileprivate struct LogLevelPicker: View { + @Binding var logLevel: Logger.Level + + var body: some View { + Picker("Log level:", selection: $logLevel) { + ForEach(Logger.Level.allCases, id: \.self) { x in + Text(x.description) + } + } + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/NodeDetailView.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/NodeDetailView.swift new file mode 100644 index 0000000000..c62ad7a68b --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/NodeDetailView.swift @@ -0,0 +1,165 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import SwiftUI + +/// View showing detail about a selected workflow, job, matrix instance, or step +struct NodeDetailView: View { + @Environment(AppModel.self) private var model + let node: HierarchicalTableView.Node + + // HierarchicalTableView.Node only compares the URLs and ignores the log contents, but we want + // the detail view to refresh if the log contents change, so reextract the logs from the model + private func extract(workflow: URL) -> WorkflowLog? { + model.parsedLogs?.workflows.first(where: { $0.url == workflow }) + } + private func extract(job: URL) -> JobLog? { + model.parsedLogs?.workflows.flatMap(\.jobs).first(where: { $0.url == job }) + } + private func extract(matrixInstance: URL) -> MatrixInstanceLog? { + model.parsedLogs?.workflows.flatMap(\.jobs).flatMap(\.matrixInstances).first(where: { $0.url == matrixInstance }) + } + private func extract(step: URL) -> StepLog? { + model.parsedLogs?.workflows.flatMap(\.jobs).flatMap(\.matrixInstances).flatMap(\.steps).first(where: { $0.url == step }) + } + + var body: some View { + switch node { + case .none, .root: + SummaryView() + + case let .workflow(x): + if let x = extract(workflow: x.url) { + RunnerStatusLabel(x) + ScrollView { + LogContentsView(logContents: x.logContents) + } + } + case let .job(x): + if let x = extract(job: x.url) { + RunnerStatusLabel(x) + ScrollView { + LogContentsView(logContents: x.logContents) + } + } + case let .matrixInstance(x): + if let x = extract(matrixInstance: x.url) { + GeometryReader { proxy in + ScrollView { + MatrixInstanceView(matrixInstance: x) + .frame(maxWidth: proxy.size.width) + } + } + } + + case let .step(x): + if let x = extract(step: x.url) { + RunnerStatusLabel(x) + GeometryReader { proxy in + ScrollView { + StepView(step: x) + .frame(maxWidth: proxy.size.width) + } + } + } + } + } +} + +fileprivate struct MatrixInstanceView: View { + let matrixInstance: MatrixInstanceLog + @Environment(AppModel.self) private var model + + func defaultExpansion(for x: StepLog) -> Bool { + switch model.autoExpandSteps { + case .never: false + case .onErrorOnly: x.containsErrors + case .onSuccessOnly: !x.containsErrors && !x.isEnded + case .whileInProgressOnly: !x.isEnded + case .exceptOnError: !x.containsErrors + case .exceptOnSuccess: x.containsErrors || !x.isEnded + case .exceptWhileInProgress: !x.isEnded + case .always: true + } + } + + func defaultExpansion(for x: MatrixInstanceLog) -> Bool { + switch model.autoExpandSteps { + case .never: false + case .onErrorOnly: x.containsErrors + case .onSuccessOnly: !x.containsErrors && !x.isEnded + case .whileInProgressOnly: !x.isEnded + case .exceptOnError: !x.containsErrors + case .exceptOnSuccess: x.containsErrors || !x.isEnded + case .exceptWhileInProgress: !x.isEnded + case .always: true + } + } + + var body: some View { + ForEach(matrixInstance.steps) { step in + InitialStateDisclosureGroup(isInitiallyExpanded: defaultExpansion(for: step)) { + StepView(step: step) + } label: { + RunnerStatusLabel(step) + } + } + + InitialStateDisclosureGroup(isInitiallyExpanded: defaultExpansion(for: matrixInstance)) { + GroupBox { + LogContentsView(logContents: matrixInstance.logContents) + } + } label: { + RunnerStatusLabel(name: "Log", isEnded: matrixInstance.isEnded, containsErrors: matrixInstance.containsErrors, startTime: matrixInstance.startTime, endTime: matrixInstance.endTime) + } + } +} + +fileprivate struct StepView: View { + let step: StepLog + + var body: some View { + TabView { + Tab { + LogContentsView(logContents: step.logContents) + } label: { + Text("Log") + } + + if !step.githubOutputContents.isEmpty { + Tab { + TextEditor(text: .constant(step.githubOutputContents)) + } label: { + Text("GitHub Output") + } + } + + if !step.githubSummaryContents.isEmpty { + Tab { + TextEditor(text: .constant(step.githubSummaryContents)) + } label: { + Text("GitHub Summary") + } + } + } + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/RunnerStatusLabel.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/RunnerStatusLabel.swift new file mode 100644 index 0000000000..f8c685595b --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/RunnerStatusLabel.swift @@ -0,0 +1,96 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import SwiftUI + +/// View showing the name, time, and status of a workflow, job, matrix instance, or step +struct RunnerStatusLabel: View { + @Environment(AppModel.self) private var model + + var name: String + var isEnded: Bool + var containsErrors: Bool + var startTime: Date? + var endTime: Date? + + init(name: String, isEnded: Bool, containsErrors: Bool, startTime: Date?, endTime: Date?) { + self.name = name + self.isEnded = isEnded + self.containsErrors = containsErrors + self.startTime = startTime + self.endTime = endTime + } + + init(_ x: ParsedLogs) { + self.init(name: x.name, isEnded: x.isEnded, containsErrors: x.containsErrors, startTime: x.startTime, endTime: x.endTime) + } + + init(_ x: WorkflowLog) { + self.init(name: x.name, isEnded: x.isEnded, containsErrors: x.containsErrors, startTime: x.startTime, endTime: x.endTime) + } + + init(_ x: JobLog) { + self.init(name: x.name, isEnded: x.isEnded, containsErrors: x.containsErrors, startTime: x.startTime, endTime: x.endTime) + } + + init(_ x: MatrixInstanceLog) { + self.init(name: x.name, isEnded: x.isEnded, containsErrors: x.containsErrors, startTime: x.startTime, endTime: x.endTime) + } + + init(_ x: StepLog) { + self.init(name: x.name, isEnded: x.isEnded, containsErrors: x.containsErrors, startTime: x.startTime, endTime: x.endTime) + } + + var body: some View { + let elapsedTimeInterval = (endTime ?? model.cancelTime ?? model.synchronizedNow).timeIntervalSince(startTime ?? model.synchronizedNow) + + + let elapsedDuration = Duration.seconds(elapsedTimeInterval) + let fractionStrategy: Duration.UnitsFormatStyle.FractionalPartDisplayStrategy = if elapsedDuration < .seconds(10) { .show(length: 1) } else { .hide } + + let formattedElapsedTime = elapsedDuration.formatted(.units(width: .narrow, fractionalPart: fractionStrategy)) + + emoji + Text(" " + name) + Text(" (\(formattedElapsedTime))") + } + + private var emoji: Text { + if isEnded { + if containsErrors { + Text("❌") + } else { + Text("✅") + } + } else if model.cancelTime != nil { + if containsErrors { + Text("⏹️❌") + } else { + Text("⏹️") + } + } else { + if containsErrors { + Text("⚙️❌") + } else { + Text("⚙️") + } + } + } +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/SummaryView.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/SummaryView.swift new file mode 100644 index 0000000000..d08f70a870 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/SummaryView.swift @@ -0,0 +1,276 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import Foundation +import SwiftUI + +/// View showing the overall summary as written into `$GITHUB_STEP_SUMMARY` by steps +struct SummaryView: View { + @Environment(AppModel.self) private var model + + var body: some View { + VStack(alignment: .leading) { + HStack { + Text("Summary").bold() + Spacer() + } + ScrollView { + if let parsedLogs = model.parsedLogs { + ForEach(parsedLogs.workflows) { workflow in + ForEach(workflow.jobs) { job in + jobView(job) + } + } + } + } + .textSelection(.enabled) + } + .padding(.top, 4) + } + + @ViewBuilder private func jobView(_ job: JobLog) -> some View { + let shouldShow = !job.matrixInstances.allSatisfy { $0.steps.allSatisfy(\.githubSummaryContents.isEmpty) } + if shouldShow { + GroupBox { + VStack(alignment: .leading) { + ForEach(job.matrixInstances) { matrixInstance in + ForEach(matrixInstance.steps) { step in + if !step.githubSummaryContents.isEmpty { + SummaryRenderer(text: step.githubSummaryContents) + } + } + } + Spacer() + } + } label: { + Text(job.name).bold() + } + } + } +} + +/// View showing the summary for an individual step +fileprivate struct SummaryRenderer: View { + let text: String + + var body: some View { + let items = Self.parse(text) + ForEach(Array(items.enumerated()), id: \.offset) { offset, element in + ItemView(item: element) + } + .textSelection(.enabled) + } + + enum Item { + case codeblock(AttributedString) + indirect case collapsibleSection(AttributedString, [Item]) + case attributedString(AttributedString) + } + + struct ItemView: View { + let item: Item + + var body: some View { + switch item { + case .codeblock(let x): + GroupBox { + HStack { + Text(x) + Spacer() + } + } + case .collapsibleSection(let name, let contents): + DisclosureGroup { + if contents.count == 1, case .codeblock = contents.first { + ItemView(item: contents.first!) + } else { + GroupBox { + HStack { + VStack(alignment: .leading) { + ForEach(Array(contents.enumerated()), id: \.offset) { (offset, element) in + HStack { + ItemView(item: element) + Spacer() + } + } + } + Spacer() + } + } + } + } label: { + Text(name) + } + case .attributedString(let x): Text(x) + } + } + } +} + +// MARK: Summary parsing + +extension SummaryRenderer { + static func parse(_ text: String) -> [Item] { + enum Pass1: Equatable { + case collapsibleSection([String]) + case line(String) + } + + func pass1(_ text: String) -> [Pass1] { + var result = [Pass1]() + + let lines = text.components(separatedBy: .newlines) + var i = 0 + + while i < lines.count { + if lines[i] == "
" { + let j = i + while i < lines.count && lines[i] != "
" { + i += 1 + } + result.append(.collapsibleSection(Array(lines[j + 1 ..< i - 1]))) + i += 1 + } else { + result.append(.line(lines[i])) + i += 1 + } + } + + if case .line("") = result.last { result.removeLast() } + + return result + } + + enum Pass2 { + case collapsibleSection(String, [Pass2]) + case line(String) + case codeblock([String]) + } + + func pass2(_ pass1: [Pass1]) -> [Pass2] { + var result = [Pass2]() + + var i = 0 + while i < pass1.count { + switch pass1[i] { + case .line("```"): + i += 1 + var blockContents = [String]() + while i < pass1.count && pass1[i] != .line("```") { + if case let .line(l) = pass1[i] { + blockContents.append(l) + } + i += 1 + } + result.append(.codeblock(blockContents)) + i += 1 + + case .line(let l): + result.append(.line(l)) + i += 1 + + case .collapsibleSection(let lines): + let summaryIndex = lines.firstIndex(where: { $0.wholeMatch(of: #/(.*)/#) != nil }) + let summary = if let summaryIndex { + String(lines[summaryIndex].wholeMatch(of: #/(.*)/#)!.output.1) + } else { "" } + + var recurseLines = lines.enumerated().filter { $0.offset != summaryIndex }.map(\.element) + if recurseLines.first == "" { recurseLines.removeFirst() } + if recurseLines.last == "" { recurseLines.removeLast() } + + result.append(.collapsibleSection(summary, pass2(recurseLines.map(Pass1.line)))) + i += 1 + } + } + return result + } + + enum Pass3 { + case collapsibleSection(String, [Pass3]) + case paragraph([String]) + case codeblock([String]) + } + + func pass3(_ pass2: [Pass2]) -> [Pass3] { + var result = [Pass3]() + + var i = 0 + while i < pass2.count { + switch pass2[i] { + case .codeblock(let l): + result.append(.codeblock(l)) + i += 1 + case .collapsibleSection(let name, let contents): + result.append(.collapsibleSection(name, pass3(contents))) + i += 1 + case .line: + var toJoin = [String]() + while i < pass2.count { + if case let .line(l) = pass2[i] { + toJoin.append(l) + if l.hasSuffix(" ") { toJoin.append("") } + } else { + break + } + i += 1 + } + result.append(.paragraph(toJoin)) + } + } + return result + } + + func pass4(_ pass3: [Pass3]) -> [Item] { + func formAttr(_ l: String) -> AttributedString { + (try? AttributedString(markdown: l)) ?? AttributedString(l) + } + + + return pass3.map { + switch $0 { + case .codeblock(let l): + var result = AttributedString(l.joined(separator: "\n")) + result.font = .body.monospaced() + return .codeblock(result) + + case .collapsibleSection(let name, let contents): + return .collapsibleSection(formAttr(name), pass4(contents)) + + case .paragraph(let lines): + return .attributedString(join(lines.map(formAttr(_:)))) + } + } + } + + return pass4(pass3(pass2(pass1(text)))) + } + + static func join(_ arr: [AttributedString]) -> AttributedString { + var result = AttributedString() + for (i, x) in arr.enumerated() { + if i + 1 < arr.count { result.append(x + "\n") } + else { result.append(x) } + } + return result + } + +} diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/ci-at-desk-UI.swift b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/ci-at-desk-UI.swift new file mode 100644 index 0000000000..d19ac25fdc --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/UI/Views/ci-at-desk-UI.swift @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import SwiftUI + +/// The SwiftUI App for the SwiftUI version of ci-at-desk. +// Launched in ci_at_desk.swift +struct ci_at_desk_UI: App { + @Environment(\.openWindow) var openWindow + + static var initialConfigFile: URL? + + var body: some Scene { + Window("ci-at-desk", id: "MainWindow") { + ContentView() + } + .onChange(of: 0, initial: true) { + // Make ci-at-desk-ui appear in the Cmd-tab app switcher + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate() + + // Open the main window when UI mode launches, + // required + openWindow(id: "MainWindow") + } + .windowToolbarStyle(.unifiedCompact) + } +} + diff --git a/scripts/ci-at-desk/Sources/ci-at-desk/ci_at_desk.swift b/scripts/ci-at-desk/Sources/ci-at-desk/ci_at_desk.swift new file mode 100644 index 0000000000..bb4248e082 --- /dev/null +++ b/scripts/ci-at-desk/Sources/ci-at-desk/ci_at_desk.swift @@ -0,0 +1,324 @@ +//===----------------------------------------------------------------------===// +// This source file is part of github.com/apple/SwiftUsd +// +// Copyright © 2025 Apple Inc. and the SwiftUsd project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//===----------------------------------------------------------------------===// + + +import ArgumentParser +import Foundation +import WorkflowDescription +import WorkflowRunning + +/// Entry point for ci-at-desk +@main +struct CLIArgs: MainActorAsyncParsableCommand { + // todo: after making ReleaseNewVersion.yml work correctly, take workflow name as argument + // specified in yamlConfig so ci-at-desk can be used to run different workflows + + @Argument(transform: URL.init(fileURLWithPath:)) var configFile: URL? + + @Flag var ui: Bool = false + + mutating func run() async throws { + #if os(macOS) + if ui { + InProcessLogNotificationHandler.enabled = true + ci_at_desk_UI.initialConfigFile = configFile + ci_at_desk_UI.main() + return + } + #else + if ui { throw ValidationError("--ui is only supported on macOS") } + #endif + + guard let configFile else { + throw ValidationError("configFile is required if not running in UI mode") + } + + let success = try await WorkflowOrchestrator.run(configFile: configFile, workflows: Self.workflows) + throw success ? ExitCode.success : ExitCode.failure + } +} + +// MARK: Workflow definitions +extension CLIArgs { + static let workflows = [buildSwiftUsd, runTests, releaseNewVersion] + + static var buildSwiftUsd: Workflow { + Workflow( + id: "Build-SwiftUsd", + jobs: [ + Job(id: "Define-Build-Matrix", + name: "Define build matrix", + outputs: ["matrix" : "${{ steps.matrix.outputs.matrix }}"], + steps: [ + Step(name: "Sparse checkout repository code", + sparseCheckout: [".github/scripts"]), + + Step(name: "Define matrix", + id: "matrix", + run: "python3", "-u", "./.github/scripts/define-build-matrix.py", + "--targets", "${{ inputs.build-targets }}"), + ]), + + Job(id: "Cache-cloning-OpenUSD-for-at-desk-builds", + if_: "${{ !skips.build-openusd }}", + steps: [ + Step(name: "Sparse checkout repository code", + sparseCheckout: []), + + Step(name: "Cache cloning OpenUSD for at-desk-builds", + cache: .init(key: "openusd-source", path: "${{ github.workspace }}/openusd-source"), + run: "git", "clone", "https://github.com/PixarAnimationStudios/OpenUSD.git", "${{ github.workspace }}/openusd-source"), + ]), + + Job(id: "Build-OpenUSD", + if_: "${{ !skips.build-openusd }}", + name: "Build OpenUSD (${{ matrix.target }})", + needs: ["Define-Build-Matrix", "Cache-cloning-OpenUSD-for-at-desk-builds"], + env: [ + "OPENUSD_REF" : "${{ inputs.openusd-ref }}", + "TARGET_PLATFORM" : "${{ matrix.target }}", + "OPENUSD_PATH" : "${{ github.workspace }}/openusd-source", + "SWIFTUSD_PATH" : "${{ runner.swiftusd-path }}", + ], + strategy: Job.Strategy(matrix: "${{ fromJson(needs.Define-Build-Matrix.outputs.matrix) }}"), + steps: [ + Step(name: "Sparse checkout repository code", + sparseCheckout: [".github/scripts", "openusd-patch.patch"]), + + Step(name: "Cache cloning OpenUSD for at-desk-builds", + cache: .init(key: "openusd-source", path: "${{ github.workspace }}/openusd-source", requiresIndependentCopy: true), + run: "git", "clone", "https://github.com/PixarAnimationStudios/OpenUSD.git", "${{ github.workspace }}/openusd-source"), + + Step(name: "Compute cache key", + id: "compute-cache-key", + run: "python3", "-u", "./.github/scripts/compute-openusd-build-cache-key.py"), + + Step(name: "Build OpenUSD on cache miss", + cache: .init(key: "${{ steps.compute-cache-key.outputs.cache-key }}", + path: "${{ github.workspace }}/openusd-builds/${{ matrix.target }}"), + run: "python3", "-u", "./.github/scripts/build-openusd.py"), + + Step(name: "Save OpenUSD build artifact", + if_: "${{ always() }}", + saveArtifact: .init(name: "openusd-builds-${{ github.run_id }}-${{ matrix.target }}", + path: "openusd-builds/${{ matrix.target }}")), + ]), + + Job(id: "Make-Swift-Package", + if_: "${{ !skips.make-swift-package }}", + name: "Make Swift Package", + needs: ["Build-OpenUSD"], + steps: [ + Step(name: "Checkout repository code", + checkout: .swiftUsd), + + Step(name: "Restore OpenUSD build artifacts", + if_: "${{ !skips.build-openusd }}", + restoreArtifact: .init(path: "openusd-builds", + pattern: "openusd-builds-${{ github.run_id }}-*")), + + Step(name: "Restore OpenUSD build artifacts when skipping building OpenUSD", + if_: "${{ skips.build-openusd }}", + sparseCheckout: ["openusd-builds"]), + + Step(name: "Clean make-swift-package", + run: "swift", "package", "--package-path", "./scripts/make-swift-package", + "clean"), + + Step(name: "Run make-swift-package", + run: "swift", "run", "--package-path", "./scripts/make-swift-package", + "make-swift-package", "${{ github.workspace }}/openusd-builds/*", "--force"), + + Step(name: "Save package artifact", + saveArtifact: .init(name: "SwiftUsd-package-${{ github.run_id }}", + path: "SwiftUsd")) + ]) + ] + ) + } + + static var runTests: Workflow { + Workflow( + id: "Run-Tests", + jobs: [ + Job(id: "Build-SwiftUsd", + if_: "${{ !skips.make-swift-package }}", + name: "Build SwiftUsd", + workflow: buildSwiftUsd), + + Job(id: "Define-Test-Matrix", + name: "Define Test Matrix", + outputs: ["matrix": "${{ steps.matrix.outputs.matrix }}", + "max-parallel": "${{ steps.matrix.outputs.max-parallel }}"], + steps: [ + Step(name: "Sparse checkout repository code", + sparseCheckout: [".github/scripts"]), + + Step(name: "Define matrix", + id: "matrix", + run: "python3", "-u", "./.github/scripts/define-test-matrix.py") + ]), + + Job(id: "Run-Tests-Once", + name: "Run tests (${{ matrix.target_platform }}, ${{ matrix.config }}, ${{ matrix.build_system }})", + needs: ["Build-SwiftUsd", "Define-Test-Matrix"], + env: [ + "TARGET_PLATFORM": "${{ matrix.target_platform }}", + "XCODEBUILD_DESTINATION": "${{ matrix.xcodebuild_destination }}", + "BUILD_SYSTEM": "${{ matrix.build_system }}", + "CONFIG": "${{ matrix.config }}", + "SWIFTUSD_REF": "${{ inputs.swiftusd-ref }}", + "OPENUSD_REF": "${{ inputs.openusd-ref }}", + "SWIFTUSD_TESTS_REF": "${{ inputs.swiftusd-tests-ref }}", + "GITHUB_RUN_ID": "${{ github.run_id }}", + "SWIFTUSD_TESTS_PATH": "${{ github.workspace }}/SwiftUsd-Tests", + "SWIFTUSD_PATH": "${{ github.workspace }}/SwiftUsd", + "RESULT_BUNDLE_PATH": "${{ github.workspace }}/SwiftUsd-Tests.xcresult", + "MATRIX_RESULT_PATH": "${{ github.workspace }}/matrix-result.json", + ], + strategy: Job.Strategy(failFast: false, + maxParallel: "${{ fromJson(needs.Define-Test-Matrix.outputs.max-parallel) }}", + matrix: ["include" : "${{ fromJson(needs.Define-Test-Matrix.outputs.matrix) }}"]), + continueOnError: true, + steps: [ + Step(name: "Restore package artifact", + if_: "${{ !skips.make-swift-package }}", + restoreArtifact: .init(path: "SwiftUsd", + pattern: "SwiftUsd-package-${{ github.run_id }}", + requiresIndependentCopy: true)), + + Step(name: "Restore package artifact when skipping make-swift-package", + if_: "${{ skips.make-swift-package }}", + checkout: .swiftUsd), + + Step(name: "Check out unit tests", + checkout: .swiftUsd_tests), + + Step(name: "Compute artifact name", + id: "compute-artifact-name", + run: "python3", "-u", "./.github/scripts/compute-test-artifact-name.py"), + + Step(name: "Build Tests", + timeout: .minutes(15), + run: "python3", "-u", "./.github/scripts/run-tests-helper.py", "build-tests"), + + Step(name: "Run Tests", + timeout: .minutes(15), + run: "python3", "-u", "./.github/scripts/run-tests-helper.py", "run-tests"), + + Step(name: "Save xcresult artifact", + if_: "${{ always() }}", + saveArtifact: .init(name: "${{ steps.compute-artifact-name.outputs.artifact_name }}", + path: "./SwiftUsd-Tests.xcresult", + allowNoFilesFound: true)), + + Step(name: "Save matrix result artifact", + if_: "${{ always() }}", + saveArtifact: .init(name: "matrix-result-${{ github.run_id }}-${{ steps.compute-artifact-name.outputs.artifact_name }}.json", + path: "./matrix-result.json")), + ]), + + Job(id: "Summarize-Test-Results", + if_: "${{ always() }}", + name: "Summarize Test Results", + needs: ["Run-Tests-Once"], + env: [ + "MATRIX_RESULTS_PATH" : "${{ github.workspace }}/../matrix-results" + ], + steps: [ + Step(name: "Sparse checkout repository code", + sparseCheckout: [".github/scripts"]), + + Step(name: "Restore matrix result artifacts", + restoreArtifact: .init(path: "../matrix-results", + pattern: "matrix-result-${{ github.run_id }}-*")), + + Step(name:"Summarize matrix results", + run: "python3", "-u", "./.github/scripts/summarize-test-matrix-results.py") + ]) + ] + ) + } + + static var releaseNewVersion: Workflow { + Workflow(id: "Release-New-Version", + jobs: [ + Job(id: "Build-SwiftUsd", + name: "Build SwiftUsd", + workflow: buildSwiftUsd), + + Job(id: "Release-New-Version", + name: "Release New Version", + needs: ["Build-SwiftUsd"], + env: [ + "SWIFTUSD_PATH" : "${{ runner.swiftusd-path }}", + ], + steps: [ + Step(name: "Restore package artifact", + restoreArtifact: .init(path: "SwiftUsd", + pattern: "SwiftUsd-package-${{ github.run_id }}", + requiresIndependentCopy: true)), + Step(name: "Add back git remote", + sparseCheckout: [".git"]), + + Step(name: "Build symbol graphs", + run: "swift", "run", "--package-path", "scripts/docc", "build-documentation"), + Step(name: "Check for documentation warnings", + run: "swift", "run", "--package-path", "scripts/docc", "check-documentation"), + Step(name: "Update documentation", + run: "swift", "run", "--package-path", "scripts/docc", "update-documentation"), + + Step(name: "Commit updated swift-package and documentation", + run: "python3", "-u", "./.github/scripts/release-new-version.py", "${{ inputs.swiftusd-tag }}"), + ]) + ]) + } +} + + +// MARK: MainActorAsyncParsableCommand + +// In order for UI mode to launch properly, the call to +// `ci_at_desk_UI.main()` has to occur when there's a single +// thread in the process. Otherwise, open panels won't work +// and the current timestamp won't update in real time +@MainActor +protocol MainActorAsyncParsableCommand: AsyncParsableCommand { + mutating func run() async throws + static func main(_ arguments: [String]?) async + static func main() async throws +} +extension MainActorAsyncParsableCommand { + static func main(_ arguments: [String]?) async { + do { + var command = try parseAsRoot(arguments) + if var asyncCommand = command as? AsyncParsableCommand { + try await asyncCommand.run() + } else { + try command.run() + } + } catch { + exit(withError: error) + } + } + static func main() async throws { + await Self.main(nil) + } +}