Switch skill CI validation workflows to Vally lint

.github/workflows/skill-check-comment.yml

@@ -1,12 +1,12 @@
-name: Skill Validator — PR Comment
+name: Vally Lint — PR Comment
 
-# Posts results from the "Skill Validator — PR Gate" workflow.
+# Posts results from the "Vally Lint — PR Gate" workflow.
 # Runs with write permissions but never checks out PR code,
 # so it is safe for fork PRs.
 
 on:
   workflow_run:
-    workflows: ["Skill Validator — PR Gate"]
+    workflows: ["Vally Lint — PR Gate"]
     types: [completed]
 
 permissions:
@@ -22,7 +22,7 @@
       - name: Download results artifact
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: skill-validator-results
+          name: vally-lint-results
           run-id: ${{ github.event.workflow_run.id }}
           github-token: ${{ github.token }}
 
@@ -34,11 +34,11 @@
             const managedLabels = {
               'skill-check-warning': {
                 color: 'FBCA04',
-                description: 'Skill validator reported warnings'
+                description: 'Vally lint reported warnings'
               },
               'skill-check-error': {
                 color: 'B60205',
-                description: 'Skill validator reported errors'
+                description: 'Vally lint reported errors'
               }
             };
 
@@ -105,9 +105,9 @@
             const agentCount = parseInt(fs.readFileSync('agent-count.txt', 'utf8').trim(), 10);
             const totalChecked = skillCount + agentCount;
 
-            const marker = '<!-- skill-validator-results -->';
-            const rawOutput = fs.existsSync('sv-output.txt')
-              ? fs.readFileSync('sv-output.txt', 'utf8')
+            const marker = '<!-- vally-lint-results -->';
+            const rawOutput = fs.existsSync('vally-output.txt')
+              ? fs.readFileSync('vally-output.txt', 'utf8')
               : '';
             const output = rawOutput.replace(/\x1b\[[0-9;]*m/g, '').trim();
 
@@ -170,8 +170,8 @@
               `| ℹ️ Advisories | ${advisoryCount} |`,
             ];
 
             const findingsTable = summaryLines.length === 0
-              ? ['_No findings were emitted by the validator._']
+              ? ['_No findings were emitted by the linter._']
               : [
                   '| Level | Finding |',
                   '|---|---|',
@@ -190,9 +190,9 @@
 
             const body = [
               marker,
-              '## 🔍 Skill Validator Results',
+              '## 🔍 Vally Lint Results',
               '',
               `**${verdict}**`,
               '',
               ...scopeTable,
               '',
@@ -203,16 +203,16 @@
               ...findingsTable,
               '',
               '<details>',
-              '<summary>Full validator output</summary>',
+              '<summary>Full linter output</summary>',
               '',
               '```text',
-              output || 'No validator output captured.',
+              output || 'No linter output captured.',
               '```',
               '',
               '</details>',
               '',
               exitCode !== '0'
-                ? '> **Note:** The validator returned a non-zero exit code. Please review the findings above before merge.'
+                ? '> **Note:** Vally lint returned a non-zero exit code. Please review the findings above before merge.'
                 : '',
             ].join('\n');
 

.github/workflows/skill-check.yml

@@ -1,4 +1,4 @@
-name: Skill Validator — PR Gate
+name: Vally Lint — PR Gate
 
 on:
   pull_request:
@@ -22,37 +22,10 @@
         with:
           fetch-depth: 0
 
-      # ── Download & cache skill-validator ──────────────────────────
-      - name: Get cache key date
-        id: cache-date
-        run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
-
-      - name: Restore skill-validator from cache
-        id: cache-sv
-        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+      - name: Setup Node.js
+        uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0
         with:
-          path: .skill-validator
-          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
-          restore-keys: |
-            skill-validator-linux-x64-
-
-      - name: Download skill-validator
-        if: steps.cache-sv.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p .skill-validator
-          curl -fsSL \
-            "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
-            -o .skill-validator/skill-validator-linux-x64.tar.gz
-          tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
-          rm .skill-validator/skill-validator-linux-x64.tar.gz
-          chmod +x .skill-validator/skill-validator
-
-      - name: Save skill-validator to cache
-        if: steps.cache-sv.outputs.cache-hit != 'true'
-        uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
-        with:
-          path: .skill-validator
-          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+          node-version: 20
 
       # ── Detect changed skills & agents ────────────────────────────
       - name: Detect changed skills and agents
@@ -111,8 +84,8 @@
 
           echo "Found $SKILL_COUNT skill dir(s) and $AGENT_COUNT agent file(s) to check."
 
-      # ── Run skill-validator check ─────────────────────────────────
-      - name: Run skill-validator check
+      # ── Run vally lint check ───────────────────────────────────────
+      - name: Run vally lint check
         id: check
         if: steps.detect.outputs.total != '0'
         env:
@@ -134,53 +107,58 @@
             done <<< "$AGENT_FILES_RAW"
           fi
 
-          CMD=(.skill-validator/skill-validator check --verbose)
+          EXIT_CODE=0
+          : > vally-output.txt
 
-          if [ ${#SKILL_DIRS[@]} -gt 0 ]; then
-            CMD+=(--skills "${SKILL_DIRS[@]}")
+          if [ ${#SKILL_DIRS[@]} -eq 0 ] && [ ${#AGENT_FILES[@]} -eq 0 ]; then
+            echo "No skills or agents to validate." | tee -a vally-output.txt
           fi
 
-          if [ ${#AGENT_FILES[@]} -gt 0 ]; then
-            CMD+=(--agents "${AGENT_FILES[@]}")
-          fi
+          for skill_dir in "${SKILL_DIRS[@]}"; do
+            echo "### Linting ${skill_dir}" | tee -a vally-output.txt
+            set +e
+            OUTPUT=$(npx --yes @microsoft/vally-cli lint "$skill_dir" --verbose 2>&1)
+            CMD_EXIT=$?
+            set -e
+            echo "$OUTPUT" | tee -a vally-output.txt
+            echo "" >> vally-output.txt
 
-          printf 'Running: '
-          printf '%q ' "${CMD[@]}"
-          echo
+            if [ "$CMD_EXIT" -ne 0 ]; then
+              EXIT_CODE=1
+            fi
+          done
 
-          # Capture output; don't fail the workflow (warn-only mode)
-          set +e
-          OUTPUT=$("${CMD[@]}" 2>&1)
-          EXIT_CODE=$?
-          set -e
+          if [ ${#AGENT_FILES[@]} -gt 0 ]; then
+            {
+              echo "### Agent files detected (not linted by vally)"
+              echo "ℹ️ Vally currently lints SKILL.md content. Agent files were detected but skipped:"
+              printf '%s\n' "${AGENT_FILES[@]}"
+              echo ""
+            } | tee -a vally-output.txt
+          fi
 
           echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
 
-          # Save output to file (multi-line safe)
-          echo "$OUTPUT" > sv-output.txt
-
-          echo "$OUTPUT"
-
       # ── Upload results for the commenting workflow ────────────────
       - name: Save metadata
         if: always()
         run: |
-          mkdir -p sv-results
-          echo "${{ github.event.pull_request.number }}" > sv-results/pr-number.txt
-          echo "${{ steps.detect.outputs.total }}" > sv-results/total.txt
-          echo "${{ steps.detect.outputs.skill_count }}" > sv-results/skill-count.txt
-          echo "${{ steps.detect.outputs.agent_count }}" > sv-results/agent-count.txt
-          echo "${{ steps.check.outputs.exit_code }}" > sv-results/exit-code.txt
-          if [ -f sv-output.txt ]; then
-            cp sv-output.txt sv-results/sv-output.txt
+          mkdir -p vally-results
+          echo "${{ github.event.pull_request.number }}" > vally-results/pr-number.txt
+          echo "${{ steps.detect.outputs.total }}" > vally-results/total.txt
+          echo "${{ steps.detect.outputs.skill_count }}" > vally-results/skill-count.txt
+          echo "${{ steps.detect.outputs.agent_count }}" > vally-results/agent-count.txt
+          echo "${{ steps.check.outputs.exit_code }}" > vally-results/exit-code.txt
+          if [ -f vally-output.txt ]; then
+            cp vally-output.txt vally-results/vally-output.txt
           fi
 
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
         with:
-          name: skill-validator-results
-          path: sv-results/
+          name: vally-lint-results
+          path: vally-results/
           retention-days: 1
 
       - name: Post skip notice if no skills changed

.github/workflows/skill-quality-report.yml

@@ -19,71 +19,37 @@ jobs:
         with:
           fetch-depth: 0 # full history for git-log author fallback
 
-      # ── Download & cache skill-validator ──────────────────────────
-      - name: Get cache key date
-        id: cache-date
-        run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
-
-      - name: Restore skill-validator from cache
-        id: cache-sv
-        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+      - name: Setup Node.js
+        uses: actions/setup-node@3235b876344febd2b5f2414c5edc3a01b7f10a06 # v4.2.0
         with:
-          path: .skill-validator
-          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
-          restore-keys: |
-            skill-validator-linux-x64-
-
-      - name: Download skill-validator
-        if: steps.cache-sv.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p .skill-validator
-          curl -fsSL \
-            "https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
-            -o .skill-validator/skill-validator-linux-x64.tar.gz
-          tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
-          rm .skill-validator/skill-validator-linux-x64.tar.gz
-          chmod +x .skill-validator/skill-validator
-
-      - name: Save skill-validator to cache
-        if: steps.cache-sv.outputs.cache-hit != 'true'
-        uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
-        with:
-          path: .skill-validator
-          key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
+          node-version: 20
 
       # ── Run full scan ─────────────────────────────────────────────
-      - name: Run skill-validator check on all skills
+      - name: Run vally lint on all skills
         id: check-skills
         run: |
           set +e
           set -o pipefail
-          .skill-validator/skill-validator check \
-            --skills ./skills \
-            --verbose \
-            2>&1 | tee sv-skills-output.txt
+          npx --yes @microsoft/vally-cli lint ./skills --verbose 2>&1 | tee vally-skills-output.txt
           echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
           set +o pipefail
           set -e
 
-      - name: Run skill-validator check on all agents
+      - name: Note agent scan status
         id: check-agents
         run: |
-          set +e
-          set -o pipefail
           AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null | tr '\n' ' ')
           if [ -n "$AGENT_FILES" ]; then
-            .skill-validator/skill-validator check \
-              --agents $AGENT_FILES \
-              --verbose \
-              2>&1 | tee sv-agents-output.txt
-            echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+            {
+              echo "ℹ️ Vally currently lints SKILL.md content."
+              echo "ℹ️ Agent files are detected but excluded from this scan:"
+              echo "$AGENT_FILES"
+            } > vally-agents-output.txt
           else
             echo "No agent files found."
-            echo "" > sv-agents-output.txt
+            echo "" > vally-agents-output.txt
             echo "exit_code=0" >> "$GITHUB_OUTPUT"
           fi
-          set +o pipefail
-          set -e
 
       # ── Build report with author attribution ──────────────────────
       - name: Build quality report
@@ -147,18 +113,18 @@ jobs:
               }
             }
 
-            // ── Parse skill-validator output ──────────────────────
+            // ── Parse vally lint output ───────────────────────────
             // The output is a text report; we preserve it as-is and
             // augment it with author info in the summary.
-            const skillsOutput = fs.readFileSync('sv-skills-output.txt', 'utf8').trim();
-            const agentsOutput = fs.existsSync('sv-agents-output.txt')
-              ? fs.readFileSync('sv-agents-output.txt', 'utf8').trim()
+            const skillsOutput = fs.readFileSync('vally-skills-output.txt', 'utf8').trim();
+            const agentsOutput = fs.existsSync('vally-agents-output.txt')
+              ? fs.readFileSync('vally-agents-output.txt', 'utf8').trim()
               : '';
 
             const codeowners = parseCodeowners();
 
             // Count findings
-            // The skill-validator uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories
+            // Vally lint uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories
             const combined = skillsOutput + '\n' + agentsOutput;
             const errorCount = (combined.match(/❌/g) || []).length;
             const warningCount = (combined.match(/⚠/g) || []).length;
@@ -179,7 +145,7 @@ jobs:
             } catch {}
 
             // ── Build author-attributed summary ───────────────────
-            // Extract per-resource blocks from output.  The validator
+            // Extract per-resource blocks from output.  The linter
             // prints skill names as headers — we annotate them with
             // the resolved owner.
             function annotateWithAuthors(output, kind) {
@@ -238,10 +204,10 @@ jobs:
               `| ℹ️ Advisories | ${advisoryCount} |`, '',
               '---',
             ];
-            const footer = `\n---\n\n_Generated by the [Skill Validator nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`;
+            const footer = `\n---\n\n_Generated by the [Vally lint nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`;
 
-            const skillsBlock = makeDetailsBlock('Skills', 'Full skill-validator output for skills', annotatedSkills);
-            const agentsBlock = makeDetailsBlock('Agents', 'Full skill-validator output for agents', annotatedAgents);
+            const skillsBlock = makeDetailsBlock('Skills', 'Full vally lint output for skills', annotatedSkills);
+            const agentsBlock = makeDetailsBlock('Agents', 'Agent scan notes', annotatedAgents);
 
             // Try full inline body first
             const fullBody = summaryLines.join('\n') + '\n\n' + skillsBlock + '\n\n' + agentsBlock + footer;