diff --git a/.codespellrc b/.codespellrc index c554845f3..d9ab6d83c 100644 --- a/.codespellrc +++ b/.codespellrc @@ -56,7 +56,9 @@ # ans - bash and powershell variable short for answer -ignore-words-list = numer,wit,aks,edn,ser,ois,gir,rouge,categor,aline,ative,afterall,deques,dateA,dateB,TE,FillIn,alle,vai,LOD,InOut,pixelX,aNULL,Wee,Sherif,queston,Vertexes,nin,FO,CAF,Parth,ans +# Vally/vally - Name of product + +ignore-words-list = numer,wit,aks,edn,ser,ois,gir,rouge,categor,aline,ative,afterall,deques,dateA,dateB,TE,FillIn,alle,vai,LOD,InOut,pixelX,aNULL,Wee,Sherif,queston,Vertexes,nin,FO,CAF,Parth,ans,Vally,vally # Skip certain files and directories diff --git a/.github/workflows/external-plugin-pr-quality-gates.yml b/.github/workflows/external-plugin-pr-quality-gates.yml index f59e5190f..0a9c10aa4 100644 --- a/.github/workflows/external-plugin-pr-quality-gates.yml +++ b/.github/workflows/external-plugin-pr-quality-gates.yml @@ -95,6 +95,9 @@ jobs: - name: Install GitHub Copilot CLI run: npm install -g @github/copilot + - name: Install @microsoft/vally + run: npm install @microsoft/vally + - name: Run external plugin PR quality gates id: quality env: @@ -205,7 +208,7 @@ jobs: const sourceUrl = String(entry?.source_tree_url || ''); const locator = String(entry?.source?.sha || entry?.source?.ref || 'repository'); const sourceCell = sourceUrl ? `[${locator}](${sourceUrl})` : locator; - return `| ${name} | ${quality.skill_validator_status || 'not_run'} | ${quality.smoke_status || 'not_run'} | ${quality.overall_status || 'not_run'} | ${sourceCell} |`; + return `| ${name} | ${quality.vally_lint_status || 'not_run'} | ${quality.smoke_status || 'not_run'} | ${quality.overall_status || 'not_run'} | ${sourceCell} |`; }) : ['| _none_ | not_run | not_run | not_run | _n/a_ |']; @@ -218,7 +221,7 @@ jobs: '', '### Per-plugin quality summary', '', - '| Plugin | skill-validator | install smoke test | overall | source tree |', + '| Plugin | vally lint | install smoke test | overall | source tree |', '|---|---|---|---|---|', ...rows, '', diff --git a/.github/workflows/external-plugin-quality-gates.yml b/.github/workflows/external-plugin-quality-gates.yml index 95e27dc4b..b88a02797 100644 --- a/.github/workflows/external-plugin-quality-gates.yml +++ b/.github/workflows/external-plugin-quality-gates.yml @@ -36,6 +36,9 @@ jobs: - name: Install GitHub Copilot CLI run: npm install -g @github/copilot + - name: Install @microsoft/vally + run: npm install @microsoft/vally + - name: Run external plugin quality gates id: quality env: diff --git a/.github/workflows/skill-check-comment.yml b/.github/workflows/skill-check-comment.yml index f427cc711..3302f70d0 100644 --- a/.github/workflows/skill-check-comment.yml +++ b/.github/workflows/skill-check-comment.yml @@ -1,12 +1,12 @@ -name: Skill Validator — PR Comment +name: Vally Lint — PR Comment -# Posts results from the "Skill Validator — PR Gate" workflow. +# Posts results from the "Vally Lint — PR Gate" workflow. # Runs with write permissions but never checks out PR code, # so it is safe for fork PRs. on: workflow_run: - workflows: ["Skill Validator — PR Gate"] + workflows: ["Vally Lint — PR Gate"] types: [completed] permissions: @@ -22,7 +22,7 @@ jobs: - name: Download results artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: skill-validator-results + name: vally-lint-results run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} @@ -34,11 +34,11 @@ jobs: const managedLabels = { 'skill-check-warning': { color: 'FBCA04', - description: 'Skill validator reported warnings' + description: 'Vally lint reported warnings' }, 'skill-check-error': { color: 'B60205', - description: 'Skill validator reported errors' + description: 'Vally lint reported errors' } }; @@ -85,9 +85,9 @@ jobs: const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10); const totalChecked = skillCount + agentCount; - const marker = ''; - const rawOutput = fs.existsSync('sv-output.txt') - ? fs.readFileSync('sv-output.txt', 'utf8') + const marker = ''; + const rawOutput = fs.existsSync('vally-output.txt') + ? fs.readFileSync('vally-output.txt', 'utf8') : ''; const output = rawOutput.replace(/\x1b\[[0-9;]*m/g, '').trim(); @@ -151,7 +151,7 @@ jobs: ]; const findingsTable = summaryLines.length === 0 - ? ['_No findings were emitted by the validator._'] + ? ['_No findings were emitted by the linter._'] : [ '| Level | Finding |', '|---|---|', @@ -170,7 +170,7 @@ jobs: const body = [ marker, - '## 🔍 Skill Validator Results', + '## 🔍 Vally Lint Results', '', `**${verdict}**`, '', @@ -183,16 +183,16 @@ jobs: ...findingsTable, '', '
', - 'Full validator output', + 'Full linter output', '', '```text', - output || 'No validator output captured.', + output || 'No linter output captured.', '```', '', '
', '', exitCode !== '0' - ? '> **Note:** The validator returned a non-zero exit code. Please review the findings above before merge.' + ? '> **Note:** Vally lint returned a non-zero exit code. Please review the findings above before merge.' : '', ].join('\n'); diff --git a/.github/workflows/skill-check.yml b/.github/workflows/skill-check.yml index 7948fc866..879a7c4d2 100644 --- a/.github/workflows/skill-check.yml +++ b/.github/workflows/skill-check.yml @@ -1,4 +1,4 @@ -name: Skill Validator — PR Gate +name: Vally Lint — PR Gate on: pull_request: @@ -22,37 +22,10 @@ jobs: with: fetch-depth: 0 - # ── Download & cache skill-validator ────────────────────────── - - name: Get cache key date - id: cache-date - run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" - - - name: Restore skill-validator from cache - id: cache-sv - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - name: Setup Node.js + uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0 with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} - restore-keys: | - skill-validator-linux-x64- - - - name: Download skill-validator - if: steps.cache-sv.outputs.cache-hit != 'true' - run: | - mkdir -p .skill-validator - curl -fsSL \ - "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \ - -o .skill-validator/skill-validator-linux-x64.tar.gz - tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator - rm .skill-validator/skill-validator-linux-x64.tar.gz - chmod +x .skill-validator/skill-validator - - - name: Save skill-validator to cache - if: steps.cache-sv.outputs.cache-hit != 'true' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + node-version: 20 # ── Detect changed skills & agents ──────────────────────────── - name: Detect changed skills and agents @@ -111,8 +84,8 @@ jobs: echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check." - # ── Run skill-validator check ───────────────────────────────── - - name: Run skill-validator check + # ── Run vally lint check ─────────────────────────────────────── + - name: Run vally lint check id: check if: steps.detect.outputs.total != '0' env: @@ -134,53 +107,58 @@ jobs: done <<< "$AGENT_FILES_RAW" fi - CMD=(.skill-validator/skill-validator check --verbose) + EXIT_CODE=0 + : > vally-output.txt - if [ ${#SKILL_DIRS[@]} -gt 0 ]; then - CMD+=(--skills "${SKILL_DIRS[@]}") + if [ ${#SKILL_DIRS[@]} -eq 0 ] && [ ${#AGENT_FILES[@]} -eq 0 ]; then + echo "No skills or agents to validate." | tee -a vally-output.txt fi - if [ ${#AGENT_FILES[@]} -gt 0 ]; then - CMD+=(--agents "${AGENT_FILES[@]}") - fi + for skill_dir in "${SKILL_DIRS[@]}"; do + echo "### Linting ${skill_dir}" | tee -a vally-output.txt + set +e + OUTPUT=$(npx --yes @microsoft/vally-cli lint "$skill_dir" --verbose 2>&1) + CMD_EXIT=$? + set -e + echo "$OUTPUT" | tee -a vally-output.txt + echo "" >> vally-output.txt - printf 'Running: ' - printf '%q ' "${CMD[@]}" - echo + if [ "$CMD_EXIT" -ne 0 ]; then + EXIT_CODE=1 + fi + done - # Capture output; don't fail the workflow (warn-only mode) - set +e - OUTPUT=$("${CMD[@]}" 2>&1) - EXIT_CODE=$? - set -e + if [ ${#AGENT_FILES[@]} -gt 0 ]; then + { + echo "### Agent files detected (not linted by vally)" + echo "ℹ️ Vally currently lints SKILL.md content. Agent files were detected but skipped:" + printf '%s\n' "${AGENT_FILES[@]}" + echo "" + } | tee -a vally-output.txt + fi echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" - # Save output to file (multi-line safe) - echo "$OUTPUT" > sv-output.txt - - echo "$OUTPUT" - # ── Upload results for the commenting workflow ──────────────── - name: Save metadata if: always() run: | - mkdir -p sv-results - echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt - echo "${{ steps.detect.outputs.total }}" > sv-results/total.txt - echo "${{ steps.detect.outputs.skill_count }}" > sv-results/skill-count.txt - echo "${{ steps.detect.outputs.agent_count }}" > sv-results/agent-count.txt - echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt - if [ -f sv-output.txt ]; then - cp sv-output.txt sv-results/sv-output.txt + mkdir -p vally-results + echo "${{ github.event.pull_request.number }}" > vally-results/pr-number.txt + echo "${{ steps.detect.outputs.total }}" > vally-results/total.txt + echo "${{ steps.detect.outputs.skill_count }}" > vally-results/skill-count.txt + echo "${{ steps.detect.outputs.agent_count }}" > vally-results/agent-count.txt + echo "${{ steps.check.outputs.exit_code }}" > vally-results/exit-code.txt + if [ -f vally-output.txt ]; then + cp vally-output.txt vally-results/vally-output.txt fi - name: Upload results if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: - name: skill-validator-results - path: sv-results/ + name: vally-lint-results + path: vally-results/ retention-days: 1 - name: Post skip notice if no skills changed diff --git a/.github/workflows/skill-quality-report.yml b/.github/workflows/skill-quality-report.yml index 75db829da..70a3209b3 100644 --- a/.github/workflows/skill-quality-report.yml +++ b/.github/workflows/skill-quality-report.yml @@ -19,71 +19,37 @@ jobs: with: fetch-depth: 0 # full history for git-log author fallback - # ── Download & cache skill-validator ────────────────────────── - - name: Get cache key date - id: cache-date - run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT" - - - name: Restore skill-validator from cache - id: cache-sv - uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - name: Setup Node.js + uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0 with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} - restore-keys: | - skill-validator-linux-x64- - - - name: Download skill-validator - if: steps.cache-sv.outputs.cache-hit != 'true' - run: | - mkdir -p .skill-validator - curl -fsSL \ - "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \ - -o .skill-validator/skill-validator-linux-x64.tar.gz - tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator - rm .skill-validator/skill-validator-linux-x64.tar.gz - chmod +x .skill-validator/skill-validator - - - name: Save skill-validator to cache - if: steps.cache-sv.outputs.cache-hit != 'true' - uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 - with: - path: .skill-validator - key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }} + node-version: 20 # ── Run full scan ───────────────────────────────────────────── - - name: Run skill-validator check on all skills + - name: Run vally lint on all skills id: check-skills run: | set +e set -o pipefail - .skill-validator/skill-validator check \ - --skills ./skills \ - --verbose \ - 2>&1 | tee sv-skills-output.txt + npx --yes @microsoft/vally-cli lint ./skills --verbose 2>&1 | tee vally-skills-output.txt echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" set +o pipefail set -e - - name: Run skill-validator check on all agents + - name: Note agent scan status id: check-agents run: | - set +e - set -o pipefail AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null | tr '\n' ' ') if [ -n "$AGENT_FILES" ]; then - .skill-validator/skill-validator check \ - --agents $AGENT_FILES \ - --verbose \ - 2>&1 | tee sv-agents-output.txt - echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + { + echo "ℹ️ Vally currently lints SKILL.md content." + echo "ℹ️ Agent files are detected but excluded from this scan:" + echo "$AGENT_FILES" + } > vally-agents-output.txt else echo "No agent files found." - echo "" > sv-agents-output.txt + echo "" > vally-agents-output.txt echo "exit_code=0" >> "$GITHUB_OUTPUT" fi - set +o pipefail - set -e # ── Build report with author attribution ────────────────────── - name: Build quality report @@ -147,18 +113,18 @@ jobs: } } - // ── Parse skill-validator output ────────────────────── + // ── Parse vally lint output ─────────────────────────── // The output is a text report; we preserve it as-is and // augment it with author info in the summary. - const skillsOutput = fs.readFileSync('sv-skills-output.txt', 'utf8').trim(); - const agentsOutput = fs.existsSync('sv-agents-output.txt') - ? fs.readFileSync('sv-agents-output.txt', 'utf8').trim() + const skillsOutput = fs.readFileSync('vally-skills-output.txt', 'utf8').trim(); + const agentsOutput = fs.existsSync('vally-agents-output.txt') + ? fs.readFileSync('vally-agents-output.txt', 'utf8').trim() : ''; const codeowners = parseCodeowners(); // Count findings - // The skill-validator uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories + // Vally lint uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories const combined = skillsOutput + '\n' + agentsOutput; const errorCount = (combined.match(/❌/g) || []).length; const warningCount = (combined.match(/⚠/g) || []).length; @@ -179,7 +145,7 @@ jobs: } catch {} // ── Build author-attributed summary ─────────────────── - // Extract per-resource blocks from output. The validator + // Extract per-resource blocks from output. The linter // prints skill names as headers — we annotate them with // the resolved owner. function annotateWithAuthors(output, kind) { @@ -238,10 +204,10 @@ jobs: `| ℹ️ Advisories | ${advisoryCount} |`, '', '---', ]; - const footer = `\n---\n\n_Generated by the [Skill Validator nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`; + const footer = `\n---\n\n_Generated by the [Vally lint nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`; - const skillsBlock = makeDetailsBlock('Skills', 'Full skill-validator output for skills', annotatedSkills); - const agentsBlock = makeDetailsBlock('Agents', 'Full skill-validator output for agents', annotatedAgents); + const skillsBlock = makeDetailsBlock('Skills', 'Full vally lint output for skills', annotatedSkills); + const agentsBlock = makeDetailsBlock('Agents', 'Agent scan notes', annotatedAgents); // Try full inline body first const fullBody = summaryLines.join('\n') + '\n\n' + skillsBlock + '\n\n' + agentsBlock + footer; diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f2c54fd0..70c8d680f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -231,7 +231,7 @@ The public-submission policy builds on those rules and also requires `license` p 1. **Open an issue** using the external plugin issue form. Automation applies the `external-plugin` and `awaiting-review` labels. 2. **Automated intake validation** checks that the required fields are present and correctly formatted for a GitHub-hosted plugin. Invalid submissions are labeled `requires-submitter-fixes` with a comment explaining what must be fixed before maintainer review. 3. **Automated quality gates** run after metadata validation: - - `skill-validator check --plugin` against the submitted plugin path/ref/sha + - `vally lint` against the submitted plugin path/ref/sha - install smoke test via Copilot CLI against an ephemeral marketplace entry generated from the submission 4. **Ready for maintainer review**: if metadata validation and quality gates pass, automation removes `awaiting-review` and adds `ready-for-review`. 5. **Submitter-fix blocker**: if metadata is valid but quality gates fail, automation applies `requires-submitter-fixes` instead of advancing to human review. @@ -246,7 +246,7 @@ The public-submission policy builds on those rules and also requires `license` p When a pull request updates `plugins/external.json` (for example, version updates for a previously approved listing), automation runs PR quality checks and posts the result directly on the PR: 1. **Detect changed entries**: automation identifies added/updated external plugin entries in the PR. -2. **Run quality gates**: automation runs install smoke tests and `skill-validator` checks against each changed plugin source ref/SHA/path. +2. **Run quality gates**: automation runs install smoke tests and `vally lint` checks against each changed plugin source ref/SHA/path. 3. **Post source links**: automation updates a bot comment with per-plugin results and direct GitHub tree links to each plugin source location. 4. **Sync workflow-state labels on the PR**: - `ready-for-review` when all checks pass diff --git a/eng/external-plugin-intake.mjs b/eng/external-plugin-intake.mjs index ade914ec4..22d551ce9 100644 --- a/eng/external-plugin-intake.mjs +++ b/eng/external-plugin-intake.mjs @@ -423,11 +423,11 @@ export function parseMarkReadyForReviewCommand(body) { function normalizeQualityGateResult(rawResult) { const defaults = { overall_status: "not_run", - skill_validator_status: "not_run", + vally_lint_status: "not_run", smoke_status: "not_run", failure_class: "none", summary: "", - skill_validator_output: "", + vally_lint_output: "", smoke_output: "", }; @@ -442,7 +442,7 @@ function normalizeQualityGateResult(rawResult) { } function buildQualityGatesCommentSection(qualityResult) { - const skillState = qualityResult.skill_validator_status || "not_run"; + const vallyState = qualityResult.vally_lint_status || "not_run"; const smokeState = qualityResult.smoke_status || "not_run"; const summaryText = String(qualityResult.summary || "").trim() || "_No quality gate details were provided._"; @@ -451,21 +451,21 @@ function buildQualityGatesCommentSection(qualityResult) { "", "| Gate | Status |", "|---|---|", - `| skill-validator | ${skillState} |`, + `| vally lint | ${vallyState} |`, `| install smoke test | ${smokeState} |`, "", summaryText, ]; - const skillOutput = String(qualityResult.skill_validator_output || "").trim(); - if (skillOutput) { + const vallyOutput = String(qualityResult.vally_lint_output || "").trim(); + if (vallyOutput) { sections.push( "", "
", - "skill-validator output", + "vally lint output", "", "```text", - skillOutput, + vallyOutput, "```", "", "
", diff --git a/eng/external-plugin-pr-quality-gates.mjs b/eng/external-plugin-pr-quality-gates.mjs index 44158322f..e72928c50 100644 --- a/eng/external-plugin-pr-quality-gates.mjs +++ b/eng/external-plugin-pr-quality-gates.mjs @@ -66,27 +66,27 @@ function aggregateResultStatus(pluginResults) { }; } -export function runExternalPluginPrQualityGates(plugins) { +export async function runExternalPluginPrQualityGates(plugins) { if (!Array.isArray(plugins)) { throw new Error("plugins must be an array"); } - const checkedPlugins = plugins.map((plugin) => { - const quality = runExternalPluginQualityGates(plugin); + const checkedPlugins = await Promise.all(plugins.map(async (plugin) => { + const quality = await runExternalPluginQualityGates(plugin); return { name: plugin?.name ?? "unknown", source: plugin?.source ?? {}, source_tree_url: buildSourceTreeUrl(plugin), quality, }; - }); + })); const aggregate = aggregateResultStatus(checkedPlugins); const summary = checkedPlugins.length === 0 ? "No changed external plugin entries were detected in plugins/external.json." : checkedPlugins .map((entry) => - `- ${entry.name}: skill-validator=${entry.quality.skill_validator_status}, install-smoke=${entry.quality.smoke_status}, overall=${entry.quality.overall_status}` + `- ${entry.name}: vally-lint=${entry.quality.vally_lint_status}, install-smoke=${entry.quality.smoke_status}, overall=${entry.quality.overall_status}` ) .join("\n"); @@ -120,6 +120,6 @@ if (import.meta.url === `file://${process.argv[1]}`) { } const plugins = JSON.parse(args["plugins-json"]); - const result = runExternalPluginPrQualityGates(plugins); + const result = await runExternalPluginPrQualityGates(plugins); process.stdout.write(`${JSON.stringify(result)}\n`); } diff --git a/eng/external-plugin-quality-gates.mjs b/eng/external-plugin-quality-gates.mjs index 06edfcd32..ce866331b 100644 --- a/eng/external-plugin-quality-gates.mjs +++ b/eng/external-plugin-quality-gates.mjs @@ -3,10 +3,11 @@ import fs from "fs"; import os from "os"; import path from "path"; +import { Writable } from "stream"; import { spawnSync } from "child_process"; +import { runLint, LintConsoleReporter } from "@microsoft/vally"; const MAX_OUTPUT_LENGTH = 12000; -const SKILL_VALIDATOR_ARCHIVE_URL = "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz"; const INFRA_ERROR_PATTERNS = [ /\b401\b/, @@ -132,35 +133,10 @@ function cloneSubmissionRepository(workDir, plugin) { return repoDir; } -function downloadSkillValidator(workDir) { - const validatorDir = path.join(workDir, "skill-validator"); - ensureDirectory(validatorDir); - const archivePath = path.join(validatorDir, "skill-validator-linux-x64.tar.gz"); - - const download = runCommand("curl", ["-fsSL", SKILL_VALIDATOR_ARCHIVE_URL, "-o", archivePath]); - if (download.exitCode !== 0) { - throw new Error(`Failed to download skill-validator: ${download.output}`); - } - - const untar = runCommand("tar", ["-xzf", archivePath, "-C", validatorDir]); - if (untar.exitCode !== 0) { - throw new Error(`Failed to extract skill-validator: ${untar.output}`); - } - - const binaryPath = path.join(validatorDir, "skill-validator"); - if (!fs.existsSync(binaryPath)) { - throw new Error("skill-validator binary was not found after extraction"); - } - - runCommand("chmod", ["+x", binaryPath]); - return binaryPath; -} - // Ordered list of candidate locations for plugin.json, from most to least specific. -// The skill-validator --plugin mode expects plugin.json at the plugin root, but -// both the Copilot CLI and many external repos use nested conventions. We read the -// manifest ourselves so skill/agent paths can be resolved from the plugin root -// consistently, regardless of where the manifest lives. +// Both the Copilot CLI and many external repos use nested conventions. We read the +// manifest ourselves so skill paths can be resolved from the plugin root consistently, +// regardless of where the manifest lives. // NOTE: Keep in sync with EXTERNAL_PLUGIN_ROOT_MANIFEST_PATHS in external-plugin-validation.mjs const PLUGIN_JSON_CANDIDATES = [ [".github", "plugin", "plugin.json"], @@ -178,72 +154,66 @@ function findPluginJson(pluginRoot) { return null; } -function buildSkillValidatorArgs(pluginRoot) { +function buildVallyLintArgs(pluginRoot) { const pluginJsonPath = findPluginJson(pluginRoot); if (!pluginJsonPath) { - // No recognised plugin.json location found — let the validator fail with its - // own diagnostic (covers exotic layouts and surfaces the real error to submitters). - return ["check", "--verbose", "--plugin", pluginRoot]; + // No recognised plugin.json location — lint the whole plugin root and let + // vally surface the real error to the submitter. + return [pluginRoot]; } let pluginJson; try { pluginJson = JSON.parse(fs.readFileSync(pluginJsonPath, "utf8")); } catch { - // Malformed plugin.json — let the validator surface the parse error. - return ["check", "--verbose", "--plugin", pluginRoot]; + // Malformed plugin.json — fall back to linting the full root. + return [pluginRoot]; } - const args = ["check", "--verbose"]; - - // Paths in plugin.json are relative to the plugin root regardless of where - // plugin.json itself lives. Use [].concat() to accept both string and array values. + // Collect skill directory paths from plugin.json. const skillPaths = [].concat(pluginJson.skills ?? []) .map((s) => path.resolve(pluginRoot, s)) - .filter((p) => fs.existsSync(p)); - - // Agent entries may be directory paths or explicit file paths; normalise to directories - // so AgentDiscovery.DiscoverAgentsInDirectory can discover agents within them. - // Deduplicate in case multiple file entries share the same parent directory. - const agentPaths = [...new Set( - [].concat(pluginJson.agents ?? []) - .map((a) => { - const resolved = path.resolve(pluginRoot, a); - if (fs.existsSync(resolved) && fs.statSync(resolved).isFile()) { - return path.dirname(resolved); - } - return resolved; - }) - .filter((p) => fs.existsSync(p)) - )]; + .filter((p) => fs.existsSync(p) && fs.statSync(p).isDirectory()); if (skillPaths.length > 0) { - args.push("--skills", ...skillPaths); - } - if (agentPaths.length > 0) { - args.push("--agents", ...agentPaths); - } - - if (skillPaths.length === 0 && agentPaths.length === 0) { - // plugin.json found but no resolvable skills/agents — fall back to --plugin so the - // validator can surface the specific validation error to the submitter. - return ["check", "--verbose", "--plugin", pluginRoot]; + return skillPaths; } - return args; + // No resolvable skill directories — lint the full plugin root so vally can + // surface the specific validation error to the submitter. + return [pluginRoot]; } -function runSkillValidatorGate(workDir, pluginRoot) { +async function runVallyLintGate(pluginRoot) { try { - const validatorBinary = downloadSkillValidator(workDir); - const args = buildSkillValidatorArgs(pluginRoot); - const check = runCommand(validatorBinary, args); - - if (check.exitCode === 0) { - return { status: "pass", output: check.output }; + const targets = buildVallyLintArgs(pluginRoot); + + let combinedOutput = ""; + let anyFailure = false; + + for (const target of targets) { + const chunks = []; + const captureStream = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(chunk.toString()); + callback(); + }, + }); + + const result = await runLint({ rootPath: target }); + const reporter = new LintConsoleReporter({ verbose: true, stream: captureStream }); + await reporter.report(result); + + combinedOutput += chunks.join("") + "\n"; + if (!result.passed) { + anyFailure = true; + } } - return { status: "fail", output: check.output }; + return { + status: anyFailure ? "fail" : "pass", + output: truncateOutput(combinedOutput), + }; } catch (error) { return { status: "infra_error", @@ -358,15 +328,15 @@ function toFailureClass(overallStatus) { return "none"; } -export function runExternalPluginQualityGates(plugin) { +export async function runExternalPluginQualityGates(plugin) { const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "external-plugin-quality-")); const result = { overall_status: "not_run", - skill_validator_status: "not_run", + vally_lint_status: "not_run", smoke_status: "not_run", failure_class: "none", summary: "", - skill_validator_output: "", + vally_lint_output: "", smoke_output: "", }; @@ -376,7 +346,7 @@ export function runExternalPluginQualityGates(plugin) { const pluginRoot = normalizedPluginPath ? path.join(repoDir, normalizedPluginPath) : repoDir; if (!fs.existsSync(pluginRoot) || !fs.statSync(pluginRoot).isDirectory()) { - result.skill_validator_status = "fail"; + result.vally_lint_status = "fail"; result.smoke_status = "fail"; result.overall_status = "fail"; result.failure_class = "submitter_fixes"; @@ -384,18 +354,18 @@ export function runExternalPluginQualityGates(plugin) { return result; } - const skillResult = runSkillValidatorGate(workDir, pluginRoot); - result.skill_validator_status = skillResult.status; - result.skill_validator_output = skillResult.output; + const vallyResult = await runVallyLintGate(pluginRoot); + result.vally_lint_status = vallyResult.status; + result.vally_lint_output = vallyResult.output; const smokeResult = runInstallSmokeGate(workDir, plugin); result.smoke_status = smokeResult.status; result.smoke_output = smokeResult.output; - result.overall_status = toOverallStatus(result.skill_validator_status, result.smoke_status); + result.overall_status = toOverallStatus(result.vally_lint_status, result.smoke_status); result.failure_class = toFailureClass(result.overall_status); result.summary = [ - `- skill-validator: ${result.skill_validator_status}`, + `- vally lint: ${result.vally_lint_status}`, `- install smoke test: ${result.smoke_status}`, `- overall: ${result.overall_status}`, ].join("\n"); @@ -405,7 +375,7 @@ export function runExternalPluginQualityGates(plugin) { result.overall_status = "infra_error"; result.failure_class = "infra"; result.summary = truncateOutput(error.message); - result.skill_validator_output = truncateOutput(error.stack || error.message); + result.vally_lint_output = truncateOutput(error.stack || error.message); return result; } finally { fs.rmSync(workDir, { recursive: true, force: true }); @@ -434,6 +404,6 @@ if (import.meta.url === `file://${process.argv[1]}`) { } const plugin = JSON.parse(args["plugin-json"]); - const result = runExternalPluginQualityGates(plugin); + const result = await runExternalPluginQualityGates(plugin); process.stdout.write(`${JSON.stringify(result)}\n`); } diff --git a/package-lock.json b/package-lock.json index bcd194d18..b4cb2c982 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "vfile-matter": "^5.0.1" }, "devDependencies": { + "@microsoft/vally": "^0.6.0", "all-contributors-cli": "^6.26.1" } }, @@ -27,6 +28,193 @@ "node": ">=6.9.0" } }, + "node_modules/@github/copilot": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.64-3.tgz", + "integrity": "sha512-Q9nBMYEHX1bJLXzzJocQx2nZvORJ0E9gvK6ly/FCtmtA7ad96BWZvf4EHzkCNDsn56aI3zNaUSfKHUKcmIAzSg==", + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2" + }, + "bin": { + "copilot": "npm-loader.js" + }, + "optionalDependencies": { + "@github/copilot-darwin-arm64": "1.0.64-3", + "@github/copilot-darwin-x64": "1.0.64-3", + "@github/copilot-linux-arm64": "1.0.64-3", + "@github/copilot-linux-x64": "1.0.64-3", + "@github/copilot-linuxmusl-arm64": "1.0.64-3", + "@github/copilot-linuxmusl-x64": "1.0.64-3", + "@github/copilot-win32-arm64": "1.0.64-3", + "@github/copilot-win32-x64": "1.0.64-3" + } + }, + "node_modules/@github/copilot-darwin-arm64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.64-3.tgz", + "integrity": "sha512-wlV6mRoAd/wG2V08TG+BOJ0nyOjroya24FSyA5A49z7PnUUuQXYRpa/GljvI5j3PM8aUl0DyBkXuB/DcFU818g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-arm64": "copilot" + } + }, + "node_modules/@github/copilot-darwin-x64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.64-3.tgz", + "integrity": "sha512-mk48PIESL2keeemX7tLRmWRDxKwl0q3cFI1ORD2QcrieNK7pSqI3eVbfoB7MqoUUI27yzIkl67xqgl8Qq28IUQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ], + "bin": { + "copilot-darwin-x64": "copilot" + } + }, + "node_modules/@github/copilot-linux-arm64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.64-3.tgz", + "integrity": "sha512-rCgtK3/rofQW5StSbeU0TwDUlOl2bvS2HGKyapVxow1Nvz3Q/TDB+eFRQc5ocBdv5tNSor+Caw2JGkRx5v508w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linux-x64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.64-3.tgz", + "integrity": "sha512-FAiBMw1h07mURSBLi3ztj5yzbP+uTbo9mhxOym1Xysut5LDpO2kYUzTYk2DlIyLGZhmH/HDOZE+b6U7lOUQy0g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linux-x64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.64-3.tgz", + "integrity": "sha512-vn8P6grPf0y2mNskkdVbAz0i46b1sP9uSXv7z6kgycjprl0CdIYPDf3WEkG60vpyopfQna+iCqCLMWRnNyCk3g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.64-3.tgz", + "integrity": "sha512-atdHimNd6nzRRwHybXUY6/84bYzXeKbDOeYN/N/DsX23+AQOPSu5BD8MD8166I/5kNHui0XOmeTSydVNBUwcJw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, + "node_modules/@github/copilot-sdk": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-1.0.3.tgz", + "integrity": "sha512-ujnH2QVw3+xvjgo9cbpY0wik4fNxAmdMDSFnxGScDSvRuK2vUCL2xWW4V2ANc9pWwRHPBpEpMuNJMtmydmLCIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@github/copilot": "^1.0.64-1", + "vscode-jsonrpc": "^8.2.1", + "zod": "^4.3.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@github/copilot-win32-arm64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.64-3.tgz", + "integrity": "sha512-jUTS9meoHEXQR8nMDOjwC0baqV273lYtLxY46W7TiOFszhsqhbhWxQMkNQBfT3GEfPp+40igzMPq3reaUTuvag==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-arm64": "copilot.exe" + } + }, + "node_modules/@github/copilot-win32-x64": { + "version": "1.0.64-3", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.64-3.tgz", + "integrity": "sha512-gHUhS500Q91hjtH9fqKDblaIs0mO09G4ifpZ1woDPXbkdKe/W29uwB7g2fn0+KczNRyPxFSWlqjnOon4Fe8svA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ], + "bin": { + "copilot-win32-x64": "copilot.exe" + } + }, + "node_modules/@microsoft/vally": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally/-/vally-0.6.0.tgz", + "integrity": "sha512-b283YRDFZXUkKNKY3+1EfMBVbHrBLIs5jfUi7lIQ8N0Y10lVsNnNGkRbtTbd7tLYZajr6AhtnpIroc4RFzo1cQ==", + "dev": true, + "dependencies": { + "@github/copilot-sdk": "^1.0.0", + "js-tiktoken": "^1.0.21", + "picomatch": "^4.0.4", + "yaml": "^2.9.0", + "zod": "^4.4.3" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -116,6 +304,27 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -215,6 +424,16 @@ "node": ">=0.10.0" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/didyoumean": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", @@ -352,6 +571,16 @@ "node": ">=8" } }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-yaml": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", @@ -535,6 +764,19 @@ "node": ">=0.10" } }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pify": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", @@ -783,6 +1025,16 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -881,6 +1133,16 @@ "engines": { "node": ">=6" } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index b7faf9d4b..ec46fbbde 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "author": "GitHub", "license": "MIT", "devDependencies": { + "@microsoft/vally": "^0.6.0", "all-contributors-cli": "^6.26.1" }, "dependencies": {