diff --git a/examples/puzzletron/README.md b/examples/puzzletron/README.md index b7be3aa0c5a..4d9aab4e058 100644 --- a/examples/puzzletron/README.md +++ b/examples/puzzletron/README.md @@ -22,7 +22,7 @@ campaign report. The configs below are the exact current-code entry points for the completed campaigns. Each verified report is a self-contained HTML file that embeds all -sanity-check outputs, stage manifests, and evaluation results — they can be +sanity-check outputs, stage manifests, and evaluation results; they can be 100s of MB. Download them to disk and open locally rather than previewing in a browser tab. @@ -80,10 +80,12 @@ local defaults-versus-customize decisions at every section: ```bash python examples/puzzletron/puzzletron_setup_v2.py \ - --defaults nv-internal/sepehr_defaults.yaml + --defaults examples/puzzletron/configs/setup/defaults.example.yaml ``` -The defaults file is loaded only when passed explicitly. Selection prompts have +The example defaults use only repository-relative values. Copy the file and add +site-specific data, scheduler, and container settings before selecting it. +Defaults are loaded only when passed explicitly. Selection prompts have a visible **← Back** action; text and numeric prompts accept `:back`. Every accepted answer and the exact navigation frame are saved in `answers_v2.yaml`, so an interrupted session can resume with: diff --git a/examples/puzzletron/configs/families/nemotron3/nano_30b_a3b_bf16/model.yaml b/examples/puzzletron/configs/families/nemotron3/nano_30b_a3b_bf16/model.yaml index 672dd813da8..0020cb6acbe 100644 --- a/examples/puzzletron/configs/families/nemotron3/nano_30b_a3b_bf16/model.yaml +++ b/examples/puzzletron/configs/families/nemotron3/nano_30b_a3b_bf16/model.yaml @@ -1,9 +1,12 @@ # @package _global_ display_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 -input_hf_model_path: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/hf/hub/models--nvidia--NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/snapshots/cbd3fa9f933d55ef16a84236559f4ee2a0526848 +# Runnable public Hugging Face repository ID; this is not a placeholder. +# Setup-generated bundles replace it with the model source selected by the user. +input_hf_model_path: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 model_info: + # Canonical public identity used for model metadata and revision checks. hf_repo: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 hf_revision: cbd3fa9f933d55ef16a84236559f4ee2a0526848 model_type: nemotron_h diff --git a/examples/puzzletron/configs/families/qwen3_5/qwen3p5_9b/model.yaml b/examples/puzzletron/configs/families/qwen3_5/qwen3p5_9b/model.yaml index b81ac8c7d98..07f53ca4091 100644 --- a/examples/puzzletron/configs/families/qwen3_5/qwen3p5_9b/model.yaml +++ b/examples/puzzletron/configs/families/qwen3_5/qwen3p5_9b/model.yaml @@ -1,9 +1,13 @@ # @package _global_ +# Runnable public Hugging Face repository ID; this is not a placeholder. +# Setup-generated bundles replace it with the model source selected by the user. input_hf_model_path: Qwen/Qwen3.5-9B model_info: + # Canonical public identity used for model metadata and revision checks. hf_repo: Qwen/Qwen3.5-9B + hf_revision: 21eca8a083a2121a92fba681f4a7c72cf20ff1a7 model_type: qwen3_5_text architectures: [Qwen3_5ForConditionalGeneration] num_hidden_layers: 32 diff --git a/examples/puzzletron/configs/families/qwen3_5/qwen3p6_35b_a3b/model.yaml b/examples/puzzletron/configs/families/qwen3_5/qwen3p6_35b_a3b/model.yaml index b3c25d14704..737ff41e219 100644 --- a/examples/puzzletron/configs/families/qwen3_5/qwen3p6_35b_a3b/model.yaml +++ b/examples/puzzletron/configs/families/qwen3_5/qwen3p6_35b_a3b/model.yaml @@ -1,12 +1,15 @@ # @package _global_ display_name: Qwen/Qwen3.6-35B-A3B +# Runnable public Hugging Face repository ID; this is not a placeholder. +# Setup-generated bundles replace it with the model source selected by the user. input_hf_model_path: Qwen/Qwen3.6-35B-A3B # Required by depth RPC / scoring stages; unlike dense Qwen configs this is not # inherited from a family-level descriptor key. descriptor: qwen3_5_moe model_info: + # Canonical public identity used for model metadata and revision checks. hf_repo: Qwen/Qwen3.6-35B-A3B hf_revision: 995ad96eacd98c81ed38be0c5b274b04031597b0 model_type: qwen3_5_moe diff --git a/examples/puzzletron/configs/orchestration/execution.example.yaml b/examples/puzzletron/configs/orchestration/execution.example.yaml index 0e764993bc2..562d0cd5b84 100644 --- a/examples/puzzletron/configs/orchestration/execution.example.yaml +++ b/examples/puzzletron/configs/orchestration/execution.example.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example runner and execution configs for the v2 orchestrator.""" +# Example execution config for the v2 orchestrator. execution: defaults: diff --git a/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml b/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml index b9a4e840e6d..4bfd30b80bf 100644 --- a/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml +++ b/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml @@ -4,21 +4,31 @@ runner: kind: slurm slurm: - account: coreai_dlalgo_llm + # Required placeholder. Replace with the Slurm account for your site. + account: REPLACE_WITH_SLURM_ACCOUNT + # Replace these generic partition names if your site uses different names. partition: batch partition_interactive: interactive partition_batch: batch + # Optional. CPU/IO stages use the regular one-node partition when unset. + partition_cpu: null interactive_max_nodes: 2 max_nodes: 20 time_limit: "4:00:00" - log_dir: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/engineering/puzzle_runs/qwen-moe/logs + log_dir: puzzle_runs/qwen-moe/logs execution_contract: - repository: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/engineering/modelopt_qwen - venv: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/engineering/modelopt_qwen/.venv_new - container: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/enroot/pytorch_25p05.sqsh - container_mounts: /lustre:/lustre + # Replace with the ModelOpt checkout path visible on every worker and in the container. + repository: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_CHECKOUT + # Replace with the virtual environment path to source on every worker. + venv: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_VENV + # Replace with an image or path accepted by the site's srun container plugin. + container: REPLACE_WITH_SLURM_CONTAINER_IMAGE + # Replace with the host and container paths required by the campaign. + container_mounts: "REPLACE_WITH_HOST_PATH:REPLACE_WITH_CONTAINER_PATH" + # Replace these values with the site's setup script and source checkout paths. prerun_commands: - - source /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/setup-envs.sh - - export VLLM_ROOT=/lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/engineering/vllm_new - - export AUTOMODEL_ROOT=/lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/engineering/Automodel_new + - source REPLACE_WITH_SITE_SETUP_SCRIPT + - export VLLM_ROOT=REPLACE_WITH_WORKER_VISIBLE_VLLM_CHECKOUT + - export AUTOMODEL_ROOT=REPLACE_WITH_WORKER_VISIBLE_AUTOMODEL_CHECKOUT + # Optional shell commands run when the stage payload exits. postrun_commands: [] diff --git a/examples/puzzletron/configs/orchestration/runner.baremetal.example.yaml b/examples/puzzletron/configs/orchestration/runner.baremetal.example.yaml index fa020ad36b3..03f91ffac8f 100644 --- a/examples/puzzletron/configs/orchestration/runner.baremetal.example.yaml +++ b/examples/puzzletron/configs/orchestration/runner.baremetal.example.yaml @@ -1,19 +1,27 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example bare-metal runner config for the v2 orchestrator.""" +# Example bare-metal runner config for the v2 orchestrator. runner: kind: baremetal inventory: - rendezvous_host: node-a + # Required placeholders. Replace both host values with SSH hosts reachable + # from the controller; the rendezvous host must match one inventory hostname. + rendezvous_host: REPLACE_WITH_PRIMARY_HOST rendezvous_port_base: 29500 hosts: - - hostname: node-a + - hostname: REPLACE_WITH_PRIMARY_HOST gpus: 8 - - hostname: node-b + - hostname: REPLACE_WITH_SECONDARY_HOST gpus: 8 execution_contract: - repository: /shared/modelopt - venv: /shared/modelopt/.venv - setup_env: "" + # Runnable when the same checkout is visible on every SSH host. + # Otherwise, replace "." with the repository path shared by those hosts. + repository: . + # Sourced as /bin/activate after changing into the repository. + # Use a repository-relative path or an absolute path visible on every host. + venv: .venv + # Optional script sourced before virtualenv activation on every host. + # Leave null when the worker login environment needs no additional setup. + setup_env: null diff --git a/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml b/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml index 49eec6b49f3..84942376cf6 100644 --- a/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml +++ b/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml @@ -1,25 +1,37 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example Slurm runner config for the v2 orchestrator.""" +# Example Slurm runner config for the v2 orchestrator. runner: kind: slurm slurm: - account: your_account + # Required placeholder. Replace with the Slurm account for your site. + account: REPLACE_WITH_SLURM_ACCOUNT + # Replace these generic partition names if your site uses different names. partition: batch partition_interactive: interactive partition_batch: batch - # Optional. CPU/IO stages fall back to one GPU when omitted. + # Optional. CPU/IO stages use the regular one-node partition when unset. partition_cpu: null interactive_max_nodes: 2 time_limit: "4:00:00" log_dir: puzzle_runs/logs execution_contract: - repository: /shared/modelopt - venv: /shared/modelopt/.venv - container: /path/to/container.sqsh - container_mounts: /shared:/shared - prerun_commands: - - source /path/to/site-setup.sh + # Runnable when orchestration starts from a checkout visible on every worker. + # Otherwise, replace "." with the worker-visible repository path. + repository: . + # Sourced as /bin/activate after changing into the repository. + # Use a repository-relative path or an absolute path visible on every worker. + venv: .venv + # Optional. Set to an image/path accepted by the cluster's srun container plugin. + # Leave null to execute directly in the worker environment. + container: null + # Optional and used only with a container. Use comma-separated + # /host/path:/container/path entries, for example /data:/data,/models:/models. + container_mounts: null + # Optional shell commands run in order before virtualenv activation. + # Example: ["module load cuda", "export HF_HOME=/data/huggingface"] + prerun_commands: [] + # Optional shell commands run when the stage payload exits. postrun_commands: [] diff --git a/examples/puzzletron/configs/setup/defaults.example.yaml b/examples/puzzletron/configs/setup/defaults.example.yaml new file mode 100644 index 00000000000..e1fb42f4203 --- /dev/null +++ b/examples/puzzletron/configs/setup/defaults.example.yaml @@ -0,0 +1,25 @@ +schema_version: 1 +infrastructure: + execution_contract: + # Runnable when setup and orchestration start from a checkout visible on every worker. + # Otherwise, replace "." with the worker-visible repository path. + repository: . + # Sourced as /bin/activate after changing into the repository. + # Use a repository-relative path or an absolute path visible on every worker. + venv: .venv + # Optional. Set to an image/path accepted by the cluster's srun container plugin. + # Leave null to execute directly in the worker environment. + container: null + # Optional and used only with a container. Use comma-separated + # /host/path:/container/path entries, for example /data:/data,/models:/models. + container_mounts: null + # Optional shell commands run in order before virtualenv activation. + # Example: ["module load cuda", "export HF_HOME=/data/huggingface"] + prerun_commands: [] + runner: + kind: slurm + slurm: + # Add the required Slurm account for your site before using this file. + # account: REPLACE_WITH_SLURM_ACCOUNT + # Optional. CPU/IO stages use the regular partition when unset. + partition_cpu: null diff --git a/nv-internal/sepehr_defaults.yaml b/nv-internal/sepehr_defaults.yaml deleted file mode 100644 index 80eab3bbe44..00000000000 --- a/nv-internal/sepehr_defaults.yaml +++ /dev/null @@ -1,15 +0,0 @@ -schema_version: 1 -data: - source: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/engineering/Puzzle-KD-Nemotron-Post-Training-Dataset-v2/ -infrastructure: - execution_contract: - venv: .venv_new - container: /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/enroot/pytorch_25p05.sqsh - container_mounts: /lustre:/lustre - prerun_commands: - - source /lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_llm/users/ssameni/setup-envs.sh - runner: - kind: slurm - slurm: - account: coreai_dlalgo_llm - partition_cpu: cpu_interactive diff --git a/puzzletron_setup/wizard.py b/puzzletron_setup/wizard.py index 52a9e9374f5..6c5f9f59b76 100644 --- a/puzzletron_setup/wizard.py +++ b/puzzletron_setup/wizard.py @@ -1211,18 +1211,42 @@ def _ask_infrastructure( [("Slurm", "slurm"), ("SSH bare metal", "baremetal")], default="slurm", ) - repository = prompts.text("Repository path on workers:", default=str(Path.cwd())) - venv = prompts.text("Python virtual environment on workers:", default=".venv") - container = prompts.text("Container image/path (blank for none):", default="").strip() + repository = prompts.text( + "Repository path on workers:", + default=".", + description=( + "Use the checkout path visible on every worker and inside the container, if used." + ), + ) + venv = prompts.text( + "Python virtual environment on workers:", + default=".venv", + description=( + "Use a repository-relative path or an absolute worker-visible path containing " + "bin/activate." + ), + ) + container = prompts.text( + "Container image/path (blank for none):", + default="", + description=( + "Use an image/path accepted by the cluster's srun container plugin, or leave " + "blank to run directly in the worker environment." + ), + ).strip() mounts = prompts.text( "Container mounts (blank for none):", default="", description=( "Use /host/path:/container/path; separate multiple mounts with commas, for " - "example /lustre:/lustre,/datasets:/datasets." + "example /data:/data,/models:/models." ), ).strip() - prerun = prompts.text("Pre-run commands separated by ';;' (blank for none):", default="") + prerun = prompts.text( + "Pre-run commands separated by ';;' (blank for none):", + default="", + description="Commands run in order before virtualenv activation.", + ) gpus_per_node = prompts.integer("GPUs per node:", default=8) _print_mesh_guidance(model) mesh_checkpoint = prompts.checkpoint() diff --git a/tests/unit/torch/puzzletron/test_portable_configs.py b/tests/unit/torch/puzzletron/test_portable_configs.py new file mode 100644 index 00000000000..084e65087fc --- /dev/null +++ b/tests/unit/torch/puzzletron/test_portable_configs.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for portable Puzzletron example configuration.""" + +import re +from pathlib import Path + +import yaml + +from puzzletron_orchestrator.compiler import load_execution_config, load_runner_config +from puzzletron_setup.v2.defaults import load_defaults + +REPOSITORY_ROOT = Path(__file__).resolve().parents[4] + + +def test_slurm_runner_example_is_portable() -> None: + slurm = load_runner_config( + REPOSITORY_ROOT / "examples/puzzletron/configs/orchestration/runner.slurm.example.yaml" + ) + assert not Path(slurm.contract.repository).is_absolute() + assert not Path(slurm.contract.venv).is_absolute() + assert slurm.contract.container is None + assert slurm.contract.container_mounts is None + assert not slurm.contract.prerun_commands + assert slurm.slurm is not None + assert slurm.slurm.account.startswith("REPLACE_WITH_") + assert slurm.slurm.partition_cpu is None + + +def test_baremetal_runner_example_is_portable() -> None: + baremetal = load_runner_config( + REPOSITORY_ROOT / "examples/puzzletron/configs/orchestration/runner.baremetal.example.yaml" + ) + assert not Path(baremetal.contract.repository).is_absolute() + assert not Path(baremetal.contract.venv).is_absolute() + assert baremetal.contract.setup_env is None + assert baremetal.baremetal is not None + hostnames = [host.hostname for host in baremetal.baremetal.hosts] + assert hostnames + assert baremetal.baremetal.rendezvous_host in hostnames + assert all(hostname.startswith("REPLACE_WITH_") for hostname in hostnames) + + +def test_qwen_slurm_runner_preserves_portable_environment_contract() -> None: + runner = load_runner_config( + REPOSITORY_ROOT / "examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml" + ) + + contract_values = ( + runner.contract.repository, + runner.contract.venv, + runner.contract.container, + runner.contract.container_mounts, + ) + assert all(value and value.startswith("REPLACE_WITH_") for value in contract_values) + assert runner.contract.prerun_commands + assert all("REPLACE_WITH_" in command for command in runner.contract.prerun_commands) + assert runner.slurm is not None + assert runner.slurm.account.startswith("REPLACE_WITH_") + assert runner.slurm.partition_cpu is None + + +def test_execution_example_is_loadable() -> None: + path = REPOSITORY_ROOT / "examples/puzzletron/configs/orchestration/execution.example.yaml" + + execution = load_execution_config(path) + + assert set(execution) >= {"defaults", "stages"} + + +def test_setup_defaults_example_is_portable() -> None: + path = REPOSITORY_ROOT / "examples/puzzletron/configs/setup/defaults.example.yaml" + + defaults = load_defaults(path) + + contract = defaults["infrastructure"]["execution_contract"] + assert not Path(contract["repository"]).is_absolute() + assert not Path(contract["venv"]).is_absolute() + assert contract["container"] is None + assert contract["container_mounts"] is None + assert not contract["prerun_commands"] + + slurm = defaults["infrastructure"]["runner"]["slurm"] + assert "account" not in slurm + assert slurm["partition_cpu"] is None + + +def test_model_examples_use_public_hugging_face_identities() -> None: + paths = ( + "examples/puzzletron/configs/families/nemotron3/nano_30b_a3b_bf16/model.yaml", + "examples/puzzletron/configs/families/qwen3_5/qwen3p5_9b/model.yaml", + "examples/puzzletron/configs/families/qwen3_5/qwen3p6_35b_a3b/model.yaml", + ) + + for relative_path in paths: + config = yaml.safe_load((REPOSITORY_ROOT / relative_path).read_text()) + + assert config["input_hf_model_path"] == config["model_info"]["hf_repo"] + assert not config["input_hf_model_path"].startswith("REPLACE_WITH_") + assert re.fullmatch(r"[0-9a-f]{40}", config["model_info"]["hf_revision"])