From c01885cdb17610d4b0e34f597590c802d5827413 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Thu, 18 Jun 2026 13:47:30 -0700 Subject: [PATCH 01/10] first pass % tests --- .ci.yaml | 9 + .ci/targets/coverage_checks.yaml | 7 + script/configs/coverage_exceptions.yaml | 2 + .../tool/lib/src/coverage_check_command.dart | 170 ++++++++++++++++++ script/tool/lib/src/main.dart | 2 + 5 files changed, 190 insertions(+) create mode 100644 .ci/targets/coverage_checks.yaml create mode 100644 script/configs/coverage_exceptions.yaml create mode 100644 script/tool/lib/src/coverage_check_command.dart diff --git a/.ci.yaml b/.ci.yaml index f988074e78ec..051ca33afe3d 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -133,6 +133,15 @@ targets: {"dependency": "open_jdk", "version": "version:21"} ] + - name: Linux coverage_checks + recipe: packages/packages + timeout: 60 + properties: + add_recipes_cq: "true" + target_file: coverage_checks.yaml + channel: master + version_file: flutter_master.version + - name: Linux dart_unit_test_shard_1 master recipe: packages/packages timeout: 60 diff --git a/.ci/targets/coverage_checks.yaml b/.ci/targets/coverage_checks.yaml new file mode 100644 index 000000000000..06233089f313 --- /dev/null +++ b/.ci/targets/coverage_checks.yaml @@ -0,0 +1,7 @@ +tasks: + - name: prepare tool + script: .ci/scripts/prepare_tool.sh + infra_step: true + - name: code Coverage Check + script: .ci/scripts/tool_runner.sh + args: ["coverage-check", "--base-branch=origin/main"] diff --git a/script/configs/coverage_exceptions.yaml b/script/configs/coverage_exceptions.yaml new file mode 100644 index 000000000000..89e412b20f08 --- /dev/null +++ b/script/configs/coverage_exceptions.yaml @@ -0,0 +1,2 @@ +coverage_exceptions: + - example_package_that_fails_coverage # TODO(camsim99): determine if this is needed diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart new file mode 100644 index 000000000000..f12a2c35c639 --- /dev/null +++ b/script/tool/lib/src/coverage_check_command.dart @@ -0,0 +1,170 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; + +import 'package:file/file.dart'; +import 'package:yaml/yaml.dart'; + +import 'common/package_looping_command.dart'; +import 'common/repository_package.dart'; + +/// A command to run coverage checks on changed packages. +class CoverageCheckCommand extends PackageLoopingCommand { + /// Creates an instance of the coverage check command. + CoverageCheckCommand(super.packagesDir, {super.processRunner, super.platform, super.gitDir}); + + @override + final String name = 'coverage-check'; + + @override + final String description = + 'Checks that code coverage does not decrease and stays above 60% ' + 'for modified packages.'; + + @override + PackageLoopingType get packageLoopingType => PackageLoopingType.includeAllSubpackages; + + final Set _exceptions = {}; + + @override + Future initializeRun() async { + final File exceptionsFile = packagesDir.parent + .childDirectory('script') + .childDirectory('configs') + .childFile('coverage_exceptions.yaml'); + if (exceptionsFile.existsSync()) { + final exceptionsConfig = loadYaml(exceptionsFile.readAsStringSync()) as YamlMap; + final packageList = exceptionsConfig['coverage_exceptions'] as YamlList?; + if (packageList != null) { + _exceptions.addAll(packageList.map((dynamic item) => item as String)); + } + } + } + + @override + Future runForPackage(RepositoryPackage package) async { + // Only run for first-party packages (in the 'packages/' directory). + if (!package.directory.path.contains( + '${packagesDir.fileSystem.path.separator}packages${packagesDir.fileSystem.path.separator}', + ) && + !package.directory.path.endsWith('${packagesDir.fileSystem.path.separator}packages')) { + return PackageResult.skip('Not a first-party package.'); + } + + if (!package.testDirectory.existsSync()) { + return PackageResult.skip('No test/ directory.'); + } + + final String packageName = package.directory.basename; + + // Run tests on current branch. + final double? currentCoverage = await _runCoverageAndParse(package); + if (currentCoverage == null) { + return PackageResult.fail(['Failed to run tests or parse coverage on HEAD']); + } + + // Checkout baseSha and run tests. + final io.ProcessResult stashResult = await processRunner.run('git', [ + 'stash', + ], workingDir: packagesDir.parent); + final io.ProcessResult checkoutBaseResult = await processRunner.run('git', [ + 'checkout', + baseSha, + ], workingDir: packagesDir.parent); + + if (checkoutBaseResult.exitCode != 0) { + return PackageResult.fail(['Failed to checkout base SHA ($baseSha).']); + } + + final double? baseCoverage = await _runCoverageAndParse(package); + + // Revert checkout + await processRunner.run('git', ['checkout', '-'], workingDir: packagesDir.parent); + if (stashResult.stdout.toString().contains('Saved working directory')) { + await processRunner.run('git', ['stash', 'pop'], workingDir: packagesDir.parent); + } + + if (baseCoverage == null) { + print( + 'Warning: Failed to run tests or parse coverage on base branch for $packageName. Assuming 0% base coverage.', + ); + } + + final double effectiveBaseCoverage = baseCoverage ?? 0.0; + + final errors = []; + + if (currentCoverage < effectiveBaseCoverage) { + errors.add( + 'Code coverage decreased from ${effectiveBaseCoverage.toStringAsFixed(1)}% ' + 'to ${currentCoverage.toStringAsFixed(1)}%.', + ); + } + + if (currentCoverage < 60.0) { + if (_exceptions.contains(packageName)) { + print( + 'Warning: Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, ' + 'which is below the 60.0% threshold. Allowed by exceptions list.', + ); + } else { + errors.add( + 'Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, ' + 'which is below the 60.0% threshold.', + ); + } + } + + if (errors.isNotEmpty) { + return PackageResult.fail(errors); + } + + return PackageResult.success(); + } + + Future _runCoverageAndParse(RepositoryPackage package) async { + final bool isFlutter = package.requiresFlutter(); + final executable = isFlutter ? 'flutter' : 'dart'; + + final args = ['test', '--coverage']; + + final io.ProcessResult result = await processRunner.run( + executable, + args, + workingDir: package.directory, + ); + + if (result.exitCode != 0) { + print('Test failed for ${package.directory.basename}:\n${result.stdout}\n${result.stderr}'); + return null; + } + + final File lcovFile = package.directory.childDirectory('coverage').childFile('lcov.info'); + if (!lcovFile.existsSync()) { + return null; + } + + return _calculateCoverage(lcovFile); + } + + double _calculateCoverage(File lcovFile) { + var linesHit = 0; + var linesFound = 0; + + final List lines = lcovFile.readAsLinesSync(); + for (final line in lines) { + if (line.startsWith('LH:')) { + linesHit += int.parse(line.substring(3)); + } else if (line.startsWith('LF:')) { + linesFound += int.parse(line.substring(3)); + } + } + + if (linesFound == 0) { + return 100.0; + } + return (linesHit / linesFound) * 100.0; + } +} diff --git a/script/tool/lib/src/main.dart b/script/tool/lib/src/main.dart index 7e56e19c6d11..89c7bf9525b0 100644 --- a/script/tool/lib/src/main.dart +++ b/script/tool/lib/src/main.dart @@ -12,6 +12,7 @@ import 'analyze_command.dart'; import 'branches_for_batch_release_command.dart'; import 'build_examples_command.dart'; import 'common/core.dart'; +import 'coverage_check_command.dart'; import 'create_all_packages_app_command.dart'; import 'custom_test_command.dart'; import 'dart_test_command.dart'; @@ -57,6 +58,7 @@ void main(List args) { ..addCommand(BranchesForBatchReleaseCommand(packagesDir)) ..addCommand(BuildExamplesCommand(packagesDir)) ..addCommand(CreateAllPackagesAppCommand(packagesDir)) + ..addCommand(CoverageCheckCommand(packagesDir)) ..addCommand(CustomTestCommand(packagesDir)) ..addCommand(DartTestCommand(packagesDir)) ..addCommand(DriveExamplesCommand(packagesDir)) From 405bb673f3b0d32a701a63c61eba37dc698fb046 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Mon, 22 Jun 2026 11:34:47 -0700 Subject: [PATCH 02/10] remove git stuff and make the test opt in --- script/configs/coverage_exceptions.yaml | 2 - script/configs/custom_coverage_minimums.yaml | 7 ++ .../tool/lib/src/coverage_check_command.dart | 97 ++++++++----------- 3 files changed, 46 insertions(+), 60 deletions(-) delete mode 100644 script/configs/coverage_exceptions.yaml create mode 100644 script/configs/custom_coverage_minimums.yaml diff --git a/script/configs/coverage_exceptions.yaml b/script/configs/coverage_exceptions.yaml deleted file mode 100644 index 89e412b20f08..000000000000 --- a/script/configs/coverage_exceptions.yaml +++ /dev/null @@ -1,2 +0,0 @@ -coverage_exceptions: - - example_package_that_fails_coverage # TODO(camsim99): determine if this is needed diff --git a/script/configs/custom_coverage_minimums.yaml b/script/configs/custom_coverage_minimums.yaml new file mode 100644 index 000000000000..4e7e55ce90c9 --- /dev/null +++ b/script/configs/custom_coverage_minimums.yaml @@ -0,0 +1,7 @@ +# A list of packages and their minimum required code coverage. +# ONLY packages listed in this file will be checked for code coverage. +# If a package drops below its listed threshold, the CI check will fail. +custom_coverage_minimums: + camera_android: 30.0 + camera_android_camerax: 18.3 + camera_avfoundation: 31.6 diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart index f12a2c35c639..048f042228e2 100644 --- a/script/tool/lib/src/coverage_check_command.dart +++ b/script/tool/lib/src/coverage_check_command.dart @@ -20,25 +20,27 @@ class CoverageCheckCommand extends PackageLoopingCommand { @override final String description = - 'Checks that code coverage does not decrease and stays above 60% ' - 'for modified packages.'; + 'Checks that code coverage meets the specified minimums ' + 'for opted-in packages.'; @override PackageLoopingType get packageLoopingType => PackageLoopingType.includeAllSubpackages; - final Set _exceptions = {}; + final Map _customMinimums = {}; @override Future initializeRun() async { - final File exceptionsFile = packagesDir.parent + final File minimumsFile = packagesDir.parent .childDirectory('script') .childDirectory('configs') - .childFile('coverage_exceptions.yaml'); - if (exceptionsFile.existsSync()) { - final exceptionsConfig = loadYaml(exceptionsFile.readAsStringSync()) as YamlMap; - final packageList = exceptionsConfig['coverage_exceptions'] as YamlList?; - if (packageList != null) { - _exceptions.addAll(packageList.map((dynamic item) => item as String)); + .childFile('custom_coverage_minimums.yaml'); + if (minimumsFile.existsSync()) { + final YamlMap minimumsConfig = loadYaml(minimumsFile.readAsStringSync()) as YamlMap; + final YamlMap? packageMap = minimumsConfig['custom_coverage_minimums'] as YamlMap?; + if (packageMap != null) { + for (final MapEntry entry in packageMap.entries) { + _customMinimums[entry.key as String] = (entry.value as num).toDouble(); + } } } } @@ -59,62 +61,30 @@ class CoverageCheckCommand extends PackageLoopingCommand { final String packageName = package.directory.basename; + if (!_customMinimums.containsKey(packageName)) { + return PackageResult.skip('Package not opted into coverage checks.'); + } + // Run tests on current branch. final double? currentCoverage = await _runCoverageAndParse(package); if (currentCoverage == null) { return PackageResult.fail(['Failed to run tests or parse coverage on HEAD']); } - // Checkout baseSha and run tests. - final io.ProcessResult stashResult = await processRunner.run('git', [ - 'stash', - ], workingDir: packagesDir.parent); - final io.ProcessResult checkoutBaseResult = await processRunner.run('git', [ - 'checkout', - baseSha, - ], workingDir: packagesDir.parent); - - if (checkoutBaseResult.exitCode != 0) { - return PackageResult.fail(['Failed to checkout base SHA ($baseSha).']); - } - - final double? baseCoverage = await _runCoverageAndParse(package); + final double requiredCoverage = _customMinimums[packageName]!; - // Revert checkout - await processRunner.run('git', ['checkout', '-'], workingDir: packagesDir.parent); - if (stashResult.stdout.toString().contains('Saved working directory')) { - await processRunner.run('git', ['stash', 'pop'], workingDir: packagesDir.parent); - } + final List errors = []; - if (baseCoverage == null) { - print( - 'Warning: Failed to run tests or parse coverage on base branch for $packageName. Assuming 0% base coverage.', - ); - } - - final double effectiveBaseCoverage = baseCoverage ?? 0.0; - - final errors = []; - - if (currentCoverage < effectiveBaseCoverage) { + if (currentCoverage < requiredCoverage) { errors.add( - 'Code coverage decreased from ${effectiveBaseCoverage.toStringAsFixed(1)}% ' - 'to ${currentCoverage.toStringAsFixed(1)}%.', + 'Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, ' + 'which is below the required ${requiredCoverage.toStringAsFixed(1)}%.', ); } - if (currentCoverage < 60.0) { - if (_exceptions.contains(packageName)) { - print( - 'Warning: Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, ' - 'which is below the 60.0% threshold. Allowed by exceptions list.', - ); - } else { - errors.add( - 'Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, ' - 'which is below the 60.0% threshold.', - ); - } + final Directory coverageDir = package.directory.childDirectory('coverage'); + if (coverageDir.existsSync()) { + coverageDir.deleteSync(recursive: true); } if (errors.isNotEmpty) { @@ -152,13 +122,24 @@ class CoverageCheckCommand extends PackageLoopingCommand { double _calculateCoverage(File lcovFile) { var linesHit = 0; var linesFound = 0; + var skipCurrentFile = false; final List lines = lcovFile.readAsLinesSync(); for (final line in lines) { - if (line.startsWith('LH:')) { - linesHit += int.parse(line.substring(3)); - } else if (line.startsWith('LF:')) { - linesFound += int.parse(line.substring(3)); + if (line.startsWith('SF:')) { + final String fileName = line.substring(3); + skipCurrentFile = + fileName.endsWith('.g.dart') || + fileName.endsWith('.pb.dart') || + fileName.endsWith('.pigeon.dart') || + fileName.endsWith('.mocks.dart') || + fileName.endsWith('.freezed.dart'); + } else if (!skipCurrentFile) { + if (line.startsWith('LH:')) { + linesHit += int.parse(line.substring(3)); + } else if (line.startsWith('LF:')) { + linesFound += int.parse(line.substring(3)); + } } } From 6a60b8022f622938f6efd0fc5925f91fc8aba572 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Mon, 22 Jun 2026 12:03:04 -0700 Subject: [PATCH 03/10] quick self review --- script/configs/custom_coverage_minimums.yaml | 3 +-- script/tool/lib/src/coverage_check_command.dart | 14 ++++++-------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/script/configs/custom_coverage_minimums.yaml b/script/configs/custom_coverage_minimums.yaml index 4e7e55ce90c9..fc4dcebd7128 100644 --- a/script/configs/custom_coverage_minimums.yaml +++ b/script/configs/custom_coverage_minimums.yaml @@ -1,7 +1,6 @@ # A list of packages and their minimum required code coverage. -# ONLY packages listed in this file will be checked for code coverage. +# Only packages listed in this file will be checked for code coverage. # If a package drops below its listed threshold, the CI check will fail. custom_coverage_minimums: camera_android: 30.0 camera_android_camerax: 18.3 - camera_avfoundation: 31.6 diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart index 048f042228e2..656da2fe9d0d 100644 --- a/script/tool/lib/src/coverage_check_command.dart +++ b/script/tool/lib/src/coverage_check_command.dart @@ -35,8 +35,8 @@ class CoverageCheckCommand extends PackageLoopingCommand { .childDirectory('configs') .childFile('custom_coverage_minimums.yaml'); if (minimumsFile.existsSync()) { - final YamlMap minimumsConfig = loadYaml(minimumsFile.readAsStringSync()) as YamlMap; - final YamlMap? packageMap = minimumsConfig['custom_coverage_minimums'] as YamlMap?; + final minimumsConfig = loadYaml(minimumsFile.readAsStringSync()) as YamlMap; + final packageMap = minimumsConfig['custom_coverage_minimums'] as YamlMap?; if (packageMap != null) { for (final MapEntry entry in packageMap.entries) { _customMinimums[entry.key as String] = (entry.value as num).toDouble(); @@ -47,7 +47,7 @@ class CoverageCheckCommand extends PackageLoopingCommand { @override Future runForPackage(RepositoryPackage package) async { - // Only run for first-party packages (in the 'packages/' directory). + // Only run for first-party packages. if (!package.directory.path.contains( '${packagesDir.fileSystem.path.separator}packages${packagesDir.fileSystem.path.separator}', ) && @@ -73,7 +73,7 @@ class CoverageCheckCommand extends PackageLoopingCommand { final double requiredCoverage = _customMinimums[packageName]!; - final List errors = []; + final errors = []; if (currentCoverage < requiredCoverage) { errors.add( @@ -95,13 +95,10 @@ class CoverageCheckCommand extends PackageLoopingCommand { } Future _runCoverageAndParse(RepositoryPackage package) async { - final bool isFlutter = package.requiresFlutter(); - final executable = isFlutter ? 'flutter' : 'dart'; - final args = ['test', '--coverage']; final io.ProcessResult result = await processRunner.run( - executable, + 'flutter', args, workingDir: package.directory, ); @@ -128,6 +125,7 @@ class CoverageCheckCommand extends PackageLoopingCommand { for (final line in lines) { if (line.startsWith('SF:')) { final String fileName = line.substring(3); + // Skip checking coverage of generated files. skipCurrentFile = fileName.endsWith('.g.dart') || fileName.endsWith('.pb.dart') || From c573a96d05b81c622d7c1370fca653ca1d02972d Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Mon, 22 Jun 2026 12:09:52 -0700 Subject: [PATCH 04/10] add tests --- .../test/coverage_check_command_test.dart | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 script/tool/test/coverage_check_command_test.dart diff --git a/script/tool/test/coverage_check_command_test.dart b/script/tool/test/coverage_check_command_test.dart new file mode 100644 index 000000000000..eea08be13f99 --- /dev/null +++ b/script/tool/test/coverage_check_command_test.dart @@ -0,0 +1,199 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/command_runner.dart'; +import 'package:file/file.dart'; +import 'package:flutter_plugin_tools/src/common/core.dart'; +import 'package:flutter_plugin_tools/src/common/plugin_utils.dart'; +import 'package:flutter_plugin_tools/src/coverage_check_command.dart'; +import 'package:git/git.dart'; +import 'package:platform/platform.dart'; +import 'package:test/test.dart'; + +import 'mocks.dart'; +import 'util.dart'; + +void main() { + group('CoverageCheckCommand', () { + late Platform mockPlatform; + late Directory packagesDir; + late CommandRunner runner; + late RecordingProcessRunner processRunner; + + setUp(() { + mockPlatform = MockPlatform(); + late GitDir gitDir; + (:packagesDir, :processRunner, gitProcessRunner: _, :gitDir) = configureBaseCommandMocks( + platform: mockPlatform, + ); + final command = CoverageCheckCommand( + packagesDir, + processRunner: processRunner, + platform: mockPlatform, + gitDir: gitDir, + ); + + runner = CommandRunner('coverage_test', 'Test for CoverageCheckCommand'); + runner.addCommand(command); + + final File minimumsFile = packagesDir.parent + .childDirectory('script') + .childDirectory('configs') + .childFile('custom_coverage_minimums.yaml'); + minimumsFile.createSync(recursive: true); + minimumsFile.writeAsStringSync(''' +custom_coverage_minimums: + plugin1: 50.0 + plugin2: 100.0 +'''); + }); + + test('skips packages not in custom minimums', () async { + createFakePlugin('unlisted_plugin', packagesDir, extraFiles: ['test/empty_test.dart']); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect( + output, + containsAllInOrder([ + contains('SKIPPING: Package not opted into coverage checks.'), + ]), + ); + expect(processRunner.recordedCalls, isEmpty); + }); + + test('passes when coverage meets minimum', () async { + final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/plugin1.dart +DA:1,1 +DA:2,1 +LF:2 +LH:2 +end_of_record +'''); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall('flutter', const ['test', '--coverage'], plugin.path), + ]), + ); + expect(output, contains(contains('Ran for 1 package(s)'))); + expect(coverageDir.existsSync(), isFalse); + }); + + test('fails when coverage is below minimum', () async { + final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/plugin1.dart +DA:1,1 +DA:2,0 +DA:3,0 +DA:4,0 +LF:4 +LH:1 +end_of_record +'''); + + Error? commandError; + final List output = await runCapturingPrint( + runner, + ['coverage-check'], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Code coverage for plugin1 is 25.0%, which is below the required 50.0%.'), + ]), + ); + }); + + test('ignores generated files when calculating coverage', () async { + final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/plugin1.dart +DA:1,1 +DA:2,1 +LF:2 +LH:2 +end_of_record +SF:lib/plugin1.g.dart +DA:1,0 +DA:2,0 +DA:3,0 +DA:4,0 +LF:4 +LH:0 +end_of_record +'''); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect(output, contains(contains('Ran for 1 package(s)'))); + }); + + test('fails when test command fails', () async { + createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + processRunner.mockProcessesForExecutable['flutter'] = [ + FakeProcessInfo(MockProcess(exitCode: 1), ['test', '--coverage']), + ]; + + Error? commandError; + final List output = await runCapturingPrint( + runner, + ['coverage-check'], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Failed to run tests or parse coverage on HEAD'), + ]), + ); + }); + + test('fails when lcov.info is missing', () async { + createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + Error? commandError; + final List output = await runCapturingPrint( + runner, + ['coverage-check'], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Failed to run tests or parse coverage on HEAD'), + ]), + ); + }); + }); +} From 144f4f3cdbb1c4b278d7cb7d4229f69239232c77 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Mon, 22 Jun 2026 12:18:08 -0700 Subject: [PATCH 05/10] address some holes in testing --- .../tool/lib/src/coverage_check_command.dart | 1 + .../test/coverage_check_command_test.dart | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart index 656da2fe9d0d..288c14672915 100644 --- a/script/tool/lib/src/coverage_check_command.dart +++ b/script/tool/lib/src/coverage_check_command.dart @@ -110,6 +110,7 @@ class CoverageCheckCommand extends PackageLoopingCommand { final File lcovFile = package.directory.childDirectory('coverage').childFile('lcov.info'); if (!lcovFile.existsSync()) { + print('Coverage file not found at ${lcovFile.path}.'); return null; } diff --git a/script/tool/test/coverage_check_command_test.dart b/script/tool/test/coverage_check_command_test.dart index eea08be13f99..af32adc90c03 100644 --- a/script/tool/test/coverage_check_command_test.dart +++ b/script/tool/test/coverage_check_command_test.dart @@ -150,6 +150,36 @@ end_of_record expect(output, contains(contains('Ran for 1 package(s)'))); }); + test('calculates coverage correctly across multiple files', () async { + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/file1.dart +DA:1,1 +DA:2,0 +LF:2 +LH:1 +end_of_record +SF:lib/file2.dart +DA:1,1 +DA:2,1 +DA:3,1 +LF:3 +LH:3 +end_of_record +'''); // Total LF: 5, LH: 4 => 80.0% coverage. 80.0 >= 50.0 so it should pass. + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect(output, contains(contains('Ran for 1 package(s)'))); + }); + test('fails when test command fails', () async { createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); @@ -191,6 +221,9 @@ end_of_record expect( output, containsAllInOrder([ + contains( + 'Coverage file not found at ${packagesDir.childDirectory('plugin1').childDirectory('coverage').childFile('lcov.info').path}.', + ), contains('Failed to run tests or parse coverage on HEAD'), ]), ); From e77c789f3f0864648dd6c95415649606cd718c75 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Mon, 22 Jun 2026 12:18:42 -0700 Subject: [PATCH 06/10] mark bringup true --- .ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci.yaml b/.ci.yaml index 051ca33afe3d..c3a8f2d98cc0 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -135,6 +135,7 @@ targets: - name: Linux coverage_checks recipe: packages/packages + bringup: true timeout: 60 properties: add_recipes_cq: "true" From 146ab9b9650924c595df5975c9b85f618737432a Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Mon, 22 Jun 2026 17:04:37 -0700 Subject: [PATCH 07/10] defend against malformed yaml --- script/tool/lib/src/coverage_check_command.dart | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart index 288c14672915..571827ebd0c8 100644 --- a/script/tool/lib/src/coverage_check_command.dart +++ b/script/tool/lib/src/coverage_check_command.dart @@ -35,11 +35,17 @@ class CoverageCheckCommand extends PackageLoopingCommand { .childDirectory('configs') .childFile('custom_coverage_minimums.yaml'); if (minimumsFile.existsSync()) { - final minimumsConfig = loadYaml(minimumsFile.readAsStringSync()) as YamlMap; - final packageMap = minimumsConfig['custom_coverage_minimums'] as YamlMap?; - if (packageMap != null) { - for (final MapEntry entry in packageMap.entries) { - _customMinimums[entry.key as String] = (entry.value as num).toDouble(); + final Object? yaml = loadYaml(minimumsFile.readAsStringSync()); + if (yaml is YamlMap) { + final Object? packageMap = yaml['custom_coverage_minimums']; + if (packageMap is YamlMap) { + for (final MapEntry entry in packageMap.entries) { + final Object? key = entry.key; + final Object? value = entry.value; + if (key is String && value is num) { + _customMinimums[key] = value.toDouble(); + } + } } } } From 56d8762b89c8b782eef2aaa1f1b7c6ba50821fb9 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Tue, 23 Jun 2026 11:12:44 -0700 Subject: [PATCH 08/10] self review --- .../tool/lib/src/coverage_check_command.dart | 49 +++++++++---------- .../test/coverage_check_command_test.dart | 5 +- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart index 571827ebd0c8..31ba7b4bf2fd 100644 --- a/script/tool/lib/src/coverage_check_command.dart +++ b/script/tool/lib/src/coverage_check_command.dart @@ -54,10 +54,8 @@ class CoverageCheckCommand extends PackageLoopingCommand { @override Future runForPackage(RepositoryPackage package) async { // Only run for first-party packages. - if (!package.directory.path.contains( - '${packagesDir.fileSystem.path.separator}packages${packagesDir.fileSystem.path.separator}', - ) && - !package.directory.path.endsWith('${packagesDir.fileSystem.path.separator}packages')) { + final List pathComponents = package.directory.fileSystem.path.split(package.path); + if (pathComponents.contains('third_party')) { return PackageResult.skip('Not a first-party package.'); } @@ -71,30 +69,23 @@ class CoverageCheckCommand extends PackageLoopingCommand { return PackageResult.skip('Package not opted into coverage checks.'); } - // Run tests on current branch. + // Collect code coverage for the package. final double? currentCoverage = await _runCoverageAndParse(package); if (currentCoverage == null) { - return PackageResult.fail(['Failed to run tests or parse coverage on HEAD']); + return PackageResult.fail(['Failed to run tests or parse coverage']); } - final double requiredCoverage = _customMinimums[packageName]!; - final errors = []; - - if (currentCoverage < requiredCoverage) { - errors.add( - 'Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, ' - 'which is below the required ${requiredCoverage.toStringAsFixed(1)}%.', - ); - } - + // Delete generated lcov.info. final Directory coverageDir = package.directory.childDirectory('coverage'); if (coverageDir.existsSync()) { coverageDir.deleteSync(recursive: true); } - if (errors.isNotEmpty) { - return PackageResult.fail(errors); + if (currentCoverage < requiredCoverage) { + return PackageResult.fail([ + 'Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, which is below the required ${requiredCoverage.toStringAsFixed(1)}%.', + ]); } return PackageResult.success(); @@ -123,6 +114,8 @@ class CoverageCheckCommand extends PackageLoopingCommand { return _calculateCoverage(lcovFile); } + /// Calculates code coverage for non-generated code files by finding + /// the percentage of covered lines of code over the total lines of code. double _calculateCoverage(File lcovFile) { var linesHit = 0; var linesFound = 0; @@ -132,19 +125,15 @@ class CoverageCheckCommand extends PackageLoopingCommand { for (final line in lines) { if (line.startsWith('SF:')) { final String fileName = line.substring(3); - // Skip checking coverage of generated files. - skipCurrentFile = - fileName.endsWith('.g.dart') || - fileName.endsWith('.pb.dart') || - fileName.endsWith('.pigeon.dart') || - fileName.endsWith('.mocks.dart') || - fileName.endsWith('.freezed.dart'); - } else if (!skipCurrentFile) { + skipCurrentFile = _isGeneratedFile(fileName); + } + if (!skipCurrentFile) { if (line.startsWith('LH:')) { linesHit += int.parse(line.substring(3)); } else if (line.startsWith('LF:')) { linesFound += int.parse(line.substring(3)); } + } } @@ -154,3 +143,11 @@ class CoverageCheckCommand extends PackageLoopingCommand { return (linesHit / linesFound) * 100.0; } } + +bool _isGeneratedFile(String fileName) { + return fileName.endsWith('.g.dart') || + fileName.endsWith('.pb.dart') || + fileName.endsWith('.pigeon.dart') || + fileName.endsWith('.mocks.dart') || + fileName.endsWith('.freezed.dart'); +} diff --git a/script/tool/test/coverage_check_command_test.dart b/script/tool/test/coverage_check_command_test.dart index af32adc90c03..372f8a3ffa19 100644 --- a/script/tool/test/coverage_check_command_test.dart +++ b/script/tool/test/coverage_check_command_test.dart @@ -5,7 +5,6 @@ import 'package:args/command_runner.dart'; import 'package:file/file.dart'; import 'package:flutter_plugin_tools/src/common/core.dart'; -import 'package:flutter_plugin_tools/src/common/plugin_utils.dart'; import 'package:flutter_plugin_tools/src/coverage_check_command.dart'; import 'package:git/git.dart'; import 'package:platform/platform.dart'; @@ -200,7 +199,7 @@ end_of_record expect( output, containsAllInOrder([ - contains('Failed to run tests or parse coverage on HEAD'), + contains('Failed to run tests or parse coverage'), ]), ); }); @@ -224,7 +223,7 @@ end_of_record contains( 'Coverage file not found at ${packagesDir.childDirectory('plugin1').childDirectory('coverage').childFile('lcov.info').path}.', ), - contains('Failed to run tests or parse coverage on HEAD'), + contains('Failed to run tests or parse coverage'), ]), ); }); From fcdfb2c40cd38b8a597311ca1d53404615c16ec2 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Tue, 23 Jun 2026 16:14:07 -0700 Subject: [PATCH 09/10] self review --- .ci/targets/coverage_checks.yaml | 2 +- script/configs/custom_coverage_minimums.yaml | 6 +- .../tool/lib/src/coverage_check_command.dart | 16 ++--- .../test/coverage_check_command_test.dart | 66 ++++++++++++++----- 4 files changed, 61 insertions(+), 29 deletions(-) diff --git a/.ci/targets/coverage_checks.yaml b/.ci/targets/coverage_checks.yaml index 06233089f313..7489e09b60c8 100644 --- a/.ci/targets/coverage_checks.yaml +++ b/.ci/targets/coverage_checks.yaml @@ -2,6 +2,6 @@ tasks: - name: prepare tool script: .ci/scripts/prepare_tool.sh infra_step: true - - name: code Coverage Check + - name: code coverage check script: .ci/scripts/tool_runner.sh args: ["coverage-check", "--base-branch=origin/main"] diff --git a/script/configs/custom_coverage_minimums.yaml b/script/configs/custom_coverage_minimums.yaml index fc4dcebd7128..7d27def5504c 100644 --- a/script/configs/custom_coverage_minimums.yaml +++ b/script/configs/custom_coverage_minimums.yaml @@ -1,6 +1,8 @@ # A list of packages and their minimum required code coverage. -# Only packages listed in this file will be checked for code coverage. -# If a package drops below its listed threshold, the CI check will fail. +# Only packages listed in this file will be checked for code coverage +# when running the script/tool/lib/src/coverage_check_command.dart +# command. If a package drops below its listed threshold, the check +# will fail. custom_coverage_minimums: camera_android: 30.0 camera_android_camerax: 18.3 diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart index 31ba7b4bf2fd..82468851b4c2 100644 --- a/script/tool/lib/src/coverage_check_command.dart +++ b/script/tool/lib/src/coverage_check_command.dart @@ -10,9 +10,9 @@ import 'package:yaml/yaml.dart'; import 'common/package_looping_command.dart'; import 'common/repository_package.dart'; -/// A command to run coverage checks on changed packages. +/// A command to run code coverage checks on changed packages. class CoverageCheckCommand extends PackageLoopingCommand { - /// Creates an instance of the coverage check command. + /// Creates a coverage check command instance. CoverageCheckCommand(super.packagesDir, {super.processRunner, super.platform, super.gitDir}); @override @@ -92,11 +92,12 @@ class CoverageCheckCommand extends PackageLoopingCommand { } Future _runCoverageAndParse(RepositoryPackage package) async { - final args = ['test', '--coverage']; - final io.ProcessResult result = await processRunner.run( 'flutter', - args, + [ + 'test', + '--coverage', + ], workingDir: package.directory, ); @@ -133,7 +134,6 @@ class CoverageCheckCommand extends PackageLoopingCommand { } else if (line.startsWith('LF:')) { linesFound += int.parse(line.substring(3)); } - } } @@ -147,7 +147,5 @@ class CoverageCheckCommand extends PackageLoopingCommand { bool _isGeneratedFile(String fileName) { return fileName.endsWith('.g.dart') || fileName.endsWith('.pb.dart') || - fileName.endsWith('.pigeon.dart') || - fileName.endsWith('.mocks.dart') || - fileName.endsWith('.freezed.dart'); + fileName.endsWith('.mocks.dart'); } diff --git a/script/tool/test/coverage_check_command_test.dart b/script/tool/test/coverage_check_command_test.dart index 372f8a3ffa19..e62b3ef9e9d3 100644 --- a/script/tool/test/coverage_check_command_test.dart +++ b/script/tool/test/coverage_check_command_test.dart @@ -35,7 +35,7 @@ void main() { runner = CommandRunner('coverage_test', 'Test for CoverageCheckCommand'); runner.addCommand(command); - + final File minimumsFile = packagesDir.parent .childDirectory('script') .childDirectory('configs') @@ -49,10 +49,14 @@ custom_coverage_minimums: }); test('skips packages not in custom minimums', () async { - createFakePlugin('unlisted_plugin', packagesDir, extraFiles: ['test/empty_test.dart']); - + createFakePlugin( + 'unlisted_plugin', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + final List output = await runCapturingPrint(runner, ['coverage-check']); - + expect( output, containsAllInOrder([ @@ -62,9 +66,31 @@ custom_coverage_minimums: expect(processRunner.recordedCalls, isEmpty); }); + test('skips third-party packages', () async { + createFakePlugin( + 'plugin1', + packagesDir.parent.childDirectory('third_party').childDirectory('packages'), + extraFiles: ['test/empty_test.dart'], + ); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect( + output, + containsAllInOrder([ + contains('SKIPPING: Not a first-party package.'), + ]), + ); + expect(processRunner.recordedCalls, isEmpty); + }); + test('passes when coverage meets minimum', () async { - final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); - + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + final Directory coverageDir = plugin.directory.childDirectory('coverage'); coverageDir.createSync(); coverageDir.childFile('lcov.info').writeAsStringSync(''' @@ -77,7 +103,7 @@ end_of_record '''); final List output = await runCapturingPrint(runner, ['coverage-check']); - + expect( processRunner.recordedCalls, orderedEquals([ @@ -89,8 +115,12 @@ end_of_record }); test('fails when coverage is below minimum', () async { - final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); - + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + final Directory coverageDir = plugin.directory.childDirectory('coverage'); coverageDir.createSync(); coverageDir.childFile('lcov.info').writeAsStringSync(''' @@ -123,8 +153,12 @@ end_of_record }); test('ignores generated files when calculating coverage', () async { - final RepositoryPackage plugin = createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); - + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + final Directory coverageDir = plugin.directory.childDirectory('coverage'); coverageDir.createSync(); coverageDir.childFile('lcov.info').writeAsStringSync(''' @@ -145,7 +179,7 @@ end_of_record '''); final List output = await runCapturingPrint(runner, ['coverage-check']); - + expect(output, contains(contains('Ran for 1 package(s)'))); }); @@ -181,7 +215,7 @@ end_of_record test('fails when test command fails', () async { createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); - + processRunner.mockProcessesForExecutable['flutter'] = [ FakeProcessInfo(MockProcess(exitCode: 1), ['test', '--coverage']), ]; @@ -198,15 +232,13 @@ end_of_record expect(commandError, isA()); expect( output, - containsAllInOrder([ - contains('Failed to run tests or parse coverage'), - ]), + containsAllInOrder([contains('Failed to run tests or parse coverage')]), ); }); test('fails when lcov.info is missing', () async { createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); - + Error? commandError; final List output = await runCapturingPrint( runner, From 6cc606c32f4c70516a0b76c109f8b33774058615 Mon Sep 17 00:00:00 2001 From: Camille Simon Date: Tue, 23 Jun 2026 16:20:59 -0700 Subject: [PATCH 10/10] remove unrequired prop --- .ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci.yaml b/.ci.yaml index c3a8f2d98cc0..64f30df592ca 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -138,7 +138,6 @@ targets: bringup: true timeout: 60 properties: - add_recipes_cq: "true" target_file: coverage_checks.yaml channel: master version_file: flutter_master.version