diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index dd95a7f..823ebc5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -212,8 +212,9 @@ frontier. This keeps file reads and parsing parallel without reading files that the dependency graph has not reached. Parsing uses `SourceType::from_path` so the extension controls whether a file is -parsed as TypeScript, JSX, ESM, or CommonJS-flavored source. Each file is parsed -at most once per analysis run. +parsed as TypeScript, ESM, or CommonJS-flavored source. JavaScript-family files +are parsed with JSX enabled because large Babel-based codebases commonly keep +JSX in `.js` files. Each file is parsed at most once per analysis run. The parse pool is capped by `CODESCYTHE_PARSE_THREADS` when set, then `RAYON_NUM_THREADS`, then the host's available parallelism. Test files may be diff --git a/benchmarks/BUILD.bazel b/benchmarks/BUILD.bazel index 707ea01..b8c5b02 100644 --- a/benchmarks/BUILD.bazel +++ b/benchmarks/BUILD.bazel @@ -31,6 +31,7 @@ fixture_functional_test( js_binary( name = "fixture_conformance_snapshot_bin", data = [ + ":kibana_pr270237_knip.jsonc", "//:node_modules/benchmark", "//:node_modules/knip", "//crates/codescythe_cli:codescythe", diff --git a/benchmarks/README.md b/benchmarks/README.md index 61fa0dd..16ddb9d 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -40,13 +40,15 @@ Codescythe and Knip, except Codescythe disables its default test-file leaf patterns for these whole-corpus benchmark configs. VS Code uses source-root entry/project globs for `src`, `build`, and `extensions`. Grafana uses source-root entry/project globs for `public`, `packages`, `scripts`, and the -root TypeScript config files. Kibana uses source-root entry/project globs for -`src`, `x-pack`, `packages`, `examples`, and `oas_docs`. Renovate uses the -source-side CLI, config-validator, `tsdown` package entries, TypeScript tooling -scripts, JavaScript/MJS tooling entrypoints, and test entrypoints instead of -generated `dist/` package bins. -The repo installs Knip as a dev dependency; set `KNIP_BIN` to compare against a -different Knip binary. +root TypeScript config files. Kibana uses the root `knip.jsonc` introduced in +`elastic/kibana#270237`; Codescythe uses the equivalent source graph (`src`, +`x-pack`, `packages`, `examples`, and `plugins` with `ts`, `tsx`, `js`, and +`jsx` files), while Knip runs with the imported +`benchmarks/kibana_pr270237_knip.jsonc`. Renovate uses the source-side CLI, +config-validator, `tsdown` package entries, TypeScript tooling scripts, +JavaScript/MJS tooling entrypoints, and test entrypoints instead of generated +`dist/` package bins. The repo installs Knip as a dev dependency; set `KNIP_BIN` +to compare against a different Knip binary. The Kibana source snapshot expects `@kbn/tsconfig-base` to exist in `node_modules`, but the Bazel-fetched fixture only contains source files. The @@ -63,7 +65,8 @@ unset. ## Current Numbers -Local run on May 22, 2026 with the checked-in fixture configs: +Local run on May 22, 2026 with the checked-in fixture configs. The Kibana rows +use the imported `elastic/kibana#270237` Knip config and a one-run smoke sample. ```sh bazel build -c opt //crates/codescythe_cli:codescythe @@ -77,8 +80,8 @@ vscode codescythe 1111.9ms +/-1.36% 5 0.90 vscode knip 4223.1ms +/-4.89% 3 0.24 grafana codescythe 833.2ms +/-4.11% 5 1.20 grafana knip 9513.4ms +/-56.75% 3 0.11 -kibana codescythe 12963.8ms +/-25.85% 3 0.08 -kibana knip 53327.5ms +/-33.69% 3 0.02 +kibana codescythe 13609.0ms +/-0.00% 1 0.07 +kibana knip 43044.0ms +/-0.00% 1 0.02 renovate codescythe 154.5ms +/-4.79% 18 6.47 renovate knip 900.5ms +/-6.09% 5 1.11 ``` diff --git a/benchmarks/kibana_pr270237_knip.jsonc b/benchmarks/kibana_pr270237_knip.jsonc new file mode 100644 index 0000000..6ecff28 --- /dev/null +++ b/benchmarks/kibana_pr270237_knip.jsonc @@ -0,0 +1,107 @@ +// Configuration for `knip` (https://knip.dev). +// +// Kibana uses a single root `package.json` and links packages via `link:` rather than yarn workspaces, +// so this file is the source of truth for what knip should analyze. Add more workspaces here as +// additional areas of the monorepo are brought under knip-managed lint coverage. +// +// Usage: +// node scripts/knip [...args] +// +// Check for unused root dependencies (ignores @kbn/* and @elastic/*): +// node scripts/knip --workspace . --include dependencies,devDependencies +// +// Scope to a single workspace: +// node scripts/knip --workspace src/platform/packages/shared/kbn-safer-lodash-set --include unlisted +{ + "$schema": "https://unpkg.com/knip@6/schema-jsonc.json", + // @kbn/* and @elastic/* are internal linked packages managed outside of knip. + // @babel/plugin-* and babel-plugin-* are consumed via require.resolve() in + // packages/kbn-babel-preset/ which knip cannot trace (it only detects require/import). + "ignoreDependencies": [ + "^@kbn/", + "^@elastic/", + "^@babel/plugin-", + "^babel-plugin-", + "^eslint-plugin-", + // This is required transitively via EUI's testing utilities; knip can't trace this. + "@testing-library/react-hooks", + // @types/d3-color is used by @elastic/kibana-d3-color; knip can't trace this. + "@types/d3-color", + // used by our FTR; knip can't trace this. + "geckodriver", + // these are used as Cypress reporters via reporter_config.json; knip can't trace the + // cypress-multi-reporters -> configFile -> reporterEnabled chain. + "buildkite-test-collector", + "cypress-multi-reporters", + "marge", + "mocha-junit-reporter", + "mocha-multi-reporters", + "mochawesome", + "mochawesome-merge", + // nyc is invoked from shell scripts (.buildkite/scripts/steps/code_coverage/) + // and osquery's package.json scripts; knip can't trace shell usage. + "nyc", + // oxlint is invoked as a binary via procRunner in @kbn/lint-cli; + // knip can't trace dynamic cmd invocations. + "oxlint" + ], + "workspaces": { + ".": { + "entry": [ + "src/**/*.{ts,tsx,js,jsx}", + "x-pack/**/*.{ts,tsx,js,jsx}", + "packages/**/*.{ts,tsx,js,jsx}", + "examples/**/*.{ts,tsx,js,jsx}", + "plugins/**/*.{ts,tsx,js,jsx}" + ], + "project": [ + "src/**/*.{ts,tsx,js,jsx}", + "x-pack/**/*.{ts,tsx,js,jsx}", + "packages/**/*.{ts,tsx,js,jsx}", + "examples/**/*.{ts,tsx,js,jsx}", + "plugins/**/*.{ts,tsx,js,jsx}" + ], + "ignore": [ + "**/node_modules/**", + "**/target/**", + "**/build/**" + ], + "storybook": { + "config": [ + "src/platform/packages/shared/kbn-storybook/src/lib/default_config.ts", + "**/.storybook/main.{js,ts}" + ] + }, + "eslint": { + "config": [".eslintrc.js", "**/.eslintrc.{js,json}"] + }, + "babel": { + "config": [ + "packages/kbn-babel-preset/common_preset.js", + "packages/kbn-babel-preset/webpack_preset.js", + "packages/kbn-babel-preset/node_preset.js", + "packages/kbn-babel-preset/istanbul_preset.js" + ] + }, + "nyc": { + "config": [ + "src/dev/code_coverage/nyc_config/nyc.jest.config.js", + "x-pack/platform/plugins/shared/osquery/.nycrc" + ] + }, + "webpack": { + "config": [ + "packages/kbn-optimizer/src/worker/webpack.config.ts", + "src/platform/plugins/shared/console/packaging/webpack.config.js", + "src/platform/packages/private/kbn-ui-shared-deps-npm/webpack.config.js", + "src/platform/packages/private/kbn-ui-shared-deps-src/webpack.config.js", + "src/platform/packages/shared/kbn-monaco/webpack.config.js", + "src/platform/kbn-ui/side-navigation/packaging/webpack.config.js" + ] + } + }, + "src/platform/packages/shared/kbn-safer-lodash-set": { + "entry": ["set.js", "setWith.js", "fp/*.js"] + } + } +} diff --git a/benchmarks/run.ts b/benchmarks/run.ts index 5f1d6be..4ed8650 100644 --- a/benchmarks/run.ts +++ b/benchmarks/run.ts @@ -29,10 +29,12 @@ type Fixture = { rawTsFiles: number; entry?: string[]; project?: string[]; + baseIgnore?: string[]; ignore?: string[]; setup?: 'grafana' | 'kibana'; extraFiles?: string; conformanceImporter?: string; + knipBenchmarkConfig?: string; }; type Options = { @@ -113,9 +115,14 @@ const kibanaSourceRoots = [ 'x-pack', 'packages', 'examples', - 'oas_docs', + 'plugins', +]; +const kibanaSourceRootPatterns = kibanaSourceRoots.map(root => `${root}/**/*.{ts,tsx,js,jsx}`); +const kibanaIgnorePatterns = [ + '**/node_modules/**', + '**/target/**', + '**/build/**', ]; -const kibanaSourceRootPatterns = kibanaSourceRoots.map(root => `${root}/**/*.{ts,tsx,mts,cts}`); const vscodeProjectPatterns = [ 'src/**/*.{ts,tsx,mts,cts}', 'build/**/*.{ts,tsx,mts,cts}', @@ -266,13 +273,14 @@ const fixtures: Fixture[] = [ commit: 'd706f62a04af1112db6b4dfef3c94955bdb98250', markerTarget: '@benchmark_kibana//:package_json', sourceFiles: 110440, - benchmarkedFiles: 85928, - rawTsFiles: 87408, + benchmarkedFiles: 90931, + rawTsFiles: 87177, entry: kibanaSourceRootPatterns, project: kibanaSourceRootPatterns, - ignore: ['**/*.gen.ts'], + baseIgnore: kibanaIgnorePatterns, setup: 'kibana', conformanceImporter: 'src/core/server/index.ts', + knipBenchmarkConfig: 'benchmarks/kibana_pr270237_knip.jsonc', }, { name: 'renovate', @@ -333,7 +341,7 @@ try { prepareFixture(fixture, fixtureRoot); const configPath = writeFixtureConfig(configRoot, fixture); const knipConfigPath = knipBin - ? writeKnipCompatibleConfig(configRoot, fixture, false, `${fixture.name}.knip-benchmark.json`) + ? writeKnipBenchmarkConfig(configRoot, fixture) : undefined; const tools = createTools(fixtureRoot, configPath, codescytheBin, knipBin, knipConfigPath); const rows = options.once ? runToolsOnce(tools) : measureTools(tools, options); @@ -484,7 +492,7 @@ function writeFixtureConfig( writeJson(configPath, { entry: fixture.entry ?? project, project, - ignore: [...ignorePatterns, ...(fixture.ignore ?? [])], + ignore: [...baseIgnorePatterns(fixture), ...(fixture.ignore ?? [])], testFilePatterns: [], includeEntryExports: true, ignoreExportsUsedInFile: false, @@ -528,7 +536,7 @@ function writeFuzzFixConfig(directory: string, fixture: Fixture): string { fixture.conformanceImporter, `${fuzzDirectory}/**/*.{ts,tsx,mts,cts}`, ], - ignore: [...ignorePatterns, ...(fixture.ignore ?? [])], + ignore: [...baseIgnorePatterns(fixture), ...(fixture.ignore ?? [])], testFilePatterns: [], includeEntryExports: true, ignoreExportsUsedInFile: false, @@ -536,6 +544,20 @@ function writeFuzzFixConfig(directory: string, fixture: Fixture): string { return configPath; } +function baseIgnorePatterns(fixture: Fixture): string[] { + return fixture.baseIgnore ?? ignorePatterns; +} + +function writeKnipBenchmarkConfig(directory: string, fixture: Fixture): string { + if (!fixture.knipBenchmarkConfig) { + return writeKnipCompatibleConfig(directory, fixture, false, `${fixture.name}.knip-benchmark.json`); + } + const configPath = path.join(directory, `${fixture.name}.knip-benchmark.jsonc`); + const sourcePath = resolveExistingPath(fixture.knipBenchmarkConfig, `${fixture.label} Knip benchmark config`); + writeFileSync(configPath, readFileSync(sourcePath, 'utf8')); + return configPath; +} + function prepareFixture(fixture: Fixture, fixtureRoot: string) { if (fixture.setup === 'grafana') { const tsconfigDir = path.join( @@ -1501,6 +1523,9 @@ function printSummary( console.log(`Corpus: ${formatCorpus(fixture)}`); console.log(`Config: entry ${formatPatterns(fixture.entry ?? fixture.project ?? sourcePatterns)}`); console.log(`Config: project ${formatPatterns(fixture.project ?? sourcePatterns)}`); + if (knipBin && fixture.knipBenchmarkConfig) { + console.log(`Knip config: ${fixture.knipBenchmarkConfig}`); + } if (parsed.once) { console.log('Runs: 1 functional smoke run'); } else { diff --git a/crates/codescythe/analyze/parse.rs b/crates/codescythe/analyze/parse.rs index 9804933..1c89923 100644 --- a/crates/codescythe/analyze/parse.rs +++ b/crates/codescythe/analyze/parse.rs @@ -3,8 +3,7 @@ use super::*; fn parse_file(cwd: &Path, path: &Path) -> Result { let source = fs::read_to_string(path) .with_context(|| format!("failed to read source file {}", path.display()))?; - let source_type = SourceType::from_path(path) - .with_context(|| format!("unsupported source extension for {}", path.display()))?; + let source_type = source_type_for_path(path)?; let allocator = Allocator::default(); let ParserReturn { program, errors, .. @@ -28,6 +27,16 @@ fn parse_file(cwd: &Path, path: &Path) -> Result { Ok(file) } +fn source_type_for_path(path: &Path) -> Result { + let source_type = SourceType::from_path(path) + .with_context(|| format!("unsupported source extension for {}", path.display()))?; + Ok(if source_type.is_javascript() { + source_type.with_jsx(true) + } else { + source_type + }) +} + pub(super) struct FileCache { pub(super) cwd: PathBuf, pub(super) paths: Vec, diff --git a/crates/codescythe/analyze/tests.rs b/crates/codescythe/analyze/tests.rs index 77dae67..d7a6730 100644 --- a/crates/codescythe/analyze/tests.rs +++ b/crates/codescythe/analyze/tests.rs @@ -150,6 +150,35 @@ fn discovers_nested_gitignore_files_by_default() { assert!(!analysis.issues.files.contains_key("src/feature/ignored.ts")); } +#[test] +fn parses_js_files_with_jsx_syntax() { + let tempdir = tempfile::tempdir().unwrap(); + let cwd = tempdir.path(); + + write_file( + cwd, + "codescythe.json", + r#"{ + "entry": "src/main.js", + "project": "src/**/*.js" + }"#, + ); + write_file( + cwd, + "src/main.js", + "import { Button } from './button';\nexport const app =