diff --git a/.ci.yaml b/.ci.yaml index f988074e78ec..64f30df592ca 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -133,6 +133,15 @@ targets: {"dependency": "open_jdk", "version": "version:21"} ] + - name: Linux coverage_checks + recipe: packages/packages + bringup: true + timeout: 60 + properties: + target_file: coverage_checks.yaml + channel: master + version_file: flutter_master.version + - name: Linux dart_unit_test_shard_1 master recipe: packages/packages timeout: 60 diff --git a/.ci/targets/coverage_checks.yaml b/.ci/targets/coverage_checks.yaml new file mode 100644 index 000000000000..7489e09b60c8 --- /dev/null +++ b/.ci/targets/coverage_checks.yaml @@ -0,0 +1,7 @@ +tasks: + - name: prepare tool + script: .ci/scripts/prepare_tool.sh + infra_step: true + - name: code coverage check + script: .ci/scripts/tool_runner.sh + args: ["coverage-check", "--base-branch=origin/main"] diff --git a/script/configs/custom_coverage_minimums.yaml b/script/configs/custom_coverage_minimums.yaml new file mode 100644 index 000000000000..7d27def5504c --- /dev/null +++ b/script/configs/custom_coverage_minimums.yaml @@ -0,0 +1,8 @@ +# A list of packages and their minimum required code coverage. +# Only packages listed in this file will be checked for code coverage +# when running the script/tool/lib/src/coverage_check_command.dart +# command. If a package drops below its listed threshold, the check +# will fail. +custom_coverage_minimums: + camera_android: 30.0 + camera_android_camerax: 18.3 diff --git a/script/tool/lib/src/coverage_check_command.dart b/script/tool/lib/src/coverage_check_command.dart new file mode 100644 index 000000000000..82468851b4c2 --- /dev/null +++ b/script/tool/lib/src/coverage_check_command.dart @@ -0,0 +1,151 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; + +import 'package:file/file.dart'; +import 'package:yaml/yaml.dart'; + +import 'common/package_looping_command.dart'; +import 'common/repository_package.dart'; + +/// A command to run code coverage checks on changed packages. +class CoverageCheckCommand extends PackageLoopingCommand { + /// Creates a coverage check command instance. + CoverageCheckCommand(super.packagesDir, {super.processRunner, super.platform, super.gitDir}); + + @override + final String name = 'coverage-check'; + + @override + final String description = + 'Checks that code coverage meets the specified minimums ' + 'for opted-in packages.'; + + @override + PackageLoopingType get packageLoopingType => PackageLoopingType.includeAllSubpackages; + + final Map _customMinimums = {}; + + @override + Future initializeRun() async { + final File minimumsFile = packagesDir.parent + .childDirectory('script') + .childDirectory('configs') + .childFile('custom_coverage_minimums.yaml'); + if (minimumsFile.existsSync()) { + final Object? yaml = loadYaml(minimumsFile.readAsStringSync()); + if (yaml is YamlMap) { + final Object? packageMap = yaml['custom_coverage_minimums']; + if (packageMap is YamlMap) { + for (final MapEntry entry in packageMap.entries) { + final Object? key = entry.key; + final Object? value = entry.value; + if (key is String && value is num) { + _customMinimums[key] = value.toDouble(); + } + } + } + } + } + } + + @override + Future runForPackage(RepositoryPackage package) async { + // Only run for first-party packages. + final List pathComponents = package.directory.fileSystem.path.split(package.path); + if (pathComponents.contains('third_party')) { + return PackageResult.skip('Not a first-party package.'); + } + + if (!package.testDirectory.existsSync()) { + return PackageResult.skip('No test/ directory.'); + } + + final String packageName = package.directory.basename; + + if (!_customMinimums.containsKey(packageName)) { + return PackageResult.skip('Package not opted into coverage checks.'); + } + + // Collect code coverage for the package. + final double? currentCoverage = await _runCoverageAndParse(package); + if (currentCoverage == null) { + return PackageResult.fail(['Failed to run tests or parse coverage']); + } + final double requiredCoverage = _customMinimums[packageName]!; + + // Delete generated lcov.info. + final Directory coverageDir = package.directory.childDirectory('coverage'); + if (coverageDir.existsSync()) { + coverageDir.deleteSync(recursive: true); + } + + if (currentCoverage < requiredCoverage) { + return PackageResult.fail([ + 'Code coverage for $packageName is ${currentCoverage.toStringAsFixed(1)}%, which is below the required ${requiredCoverage.toStringAsFixed(1)}%.', + ]); + } + + return PackageResult.success(); + } + + Future _runCoverageAndParse(RepositoryPackage package) async { + final io.ProcessResult result = await processRunner.run( + 'flutter', + [ + 'test', + '--coverage', + ], + workingDir: package.directory, + ); + + if (result.exitCode != 0) { + print('Test failed for ${package.directory.basename}:\n${result.stdout}\n${result.stderr}'); + return null; + } + + final File lcovFile = package.directory.childDirectory('coverage').childFile('lcov.info'); + if (!lcovFile.existsSync()) { + print('Coverage file not found at ${lcovFile.path}.'); + return null; + } + + return _calculateCoverage(lcovFile); + } + + /// Calculates code coverage for non-generated code files by finding + /// the percentage of covered lines of code over the total lines of code. + double _calculateCoverage(File lcovFile) { + var linesHit = 0; + var linesFound = 0; + var skipCurrentFile = false; + + final List lines = lcovFile.readAsLinesSync(); + for (final line in lines) { + if (line.startsWith('SF:')) { + final String fileName = line.substring(3); + skipCurrentFile = _isGeneratedFile(fileName); + } + if (!skipCurrentFile) { + if (line.startsWith('LH:')) { + linesHit += int.parse(line.substring(3)); + } else if (line.startsWith('LF:')) { + linesFound += int.parse(line.substring(3)); + } + } + } + + if (linesFound == 0) { + return 100.0; + } + return (linesHit / linesFound) * 100.0; + } +} + +bool _isGeneratedFile(String fileName) { + return fileName.endsWith('.g.dart') || + fileName.endsWith('.pb.dart') || + fileName.endsWith('.mocks.dart'); +} diff --git a/script/tool/lib/src/main.dart b/script/tool/lib/src/main.dart index 7e56e19c6d11..89c7bf9525b0 100644 --- a/script/tool/lib/src/main.dart +++ b/script/tool/lib/src/main.dart @@ -12,6 +12,7 @@ import 'analyze_command.dart'; import 'branches_for_batch_release_command.dart'; import 'build_examples_command.dart'; import 'common/core.dart'; +import 'coverage_check_command.dart'; import 'create_all_packages_app_command.dart'; import 'custom_test_command.dart'; import 'dart_test_command.dart'; @@ -57,6 +58,7 @@ void main(List args) { ..addCommand(BranchesForBatchReleaseCommand(packagesDir)) ..addCommand(BuildExamplesCommand(packagesDir)) ..addCommand(CreateAllPackagesAppCommand(packagesDir)) + ..addCommand(CoverageCheckCommand(packagesDir)) ..addCommand(CustomTestCommand(packagesDir)) ..addCommand(DartTestCommand(packagesDir)) ..addCommand(DriveExamplesCommand(packagesDir)) diff --git a/script/tool/test/coverage_check_command_test.dart b/script/tool/test/coverage_check_command_test.dart new file mode 100644 index 000000000000..e62b3ef9e9d3 --- /dev/null +++ b/script/tool/test/coverage_check_command_test.dart @@ -0,0 +1,263 @@ +// Copyright 2013 The Flutter Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/command_runner.dart'; +import 'package:file/file.dart'; +import 'package:flutter_plugin_tools/src/common/core.dart'; +import 'package:flutter_plugin_tools/src/coverage_check_command.dart'; +import 'package:git/git.dart'; +import 'package:platform/platform.dart'; +import 'package:test/test.dart'; + +import 'mocks.dart'; +import 'util.dart'; + +void main() { + group('CoverageCheckCommand', () { + late Platform mockPlatform; + late Directory packagesDir; + late CommandRunner runner; + late RecordingProcessRunner processRunner; + + setUp(() { + mockPlatform = MockPlatform(); + late GitDir gitDir; + (:packagesDir, :processRunner, gitProcessRunner: _, :gitDir) = configureBaseCommandMocks( + platform: mockPlatform, + ); + final command = CoverageCheckCommand( + packagesDir, + processRunner: processRunner, + platform: mockPlatform, + gitDir: gitDir, + ); + + runner = CommandRunner('coverage_test', 'Test for CoverageCheckCommand'); + runner.addCommand(command); + + final File minimumsFile = packagesDir.parent + .childDirectory('script') + .childDirectory('configs') + .childFile('custom_coverage_minimums.yaml'); + minimumsFile.createSync(recursive: true); + minimumsFile.writeAsStringSync(''' +custom_coverage_minimums: + plugin1: 50.0 + plugin2: 100.0 +'''); + }); + + test('skips packages not in custom minimums', () async { + createFakePlugin( + 'unlisted_plugin', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect( + output, + containsAllInOrder([ + contains('SKIPPING: Package not opted into coverage checks.'), + ]), + ); + expect(processRunner.recordedCalls, isEmpty); + }); + + test('skips third-party packages', () async { + createFakePlugin( + 'plugin1', + packagesDir.parent.childDirectory('third_party').childDirectory('packages'), + extraFiles: ['test/empty_test.dart'], + ); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect( + output, + containsAllInOrder([ + contains('SKIPPING: Not a first-party package.'), + ]), + ); + expect(processRunner.recordedCalls, isEmpty); + }); + + test('passes when coverage meets minimum', () async { + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/plugin1.dart +DA:1,1 +DA:2,1 +LF:2 +LH:2 +end_of_record +'''); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall('flutter', const ['test', '--coverage'], plugin.path), + ]), + ); + expect(output, contains(contains('Ran for 1 package(s)'))); + expect(coverageDir.existsSync(), isFalse); + }); + + test('fails when coverage is below minimum', () async { + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/plugin1.dart +DA:1,1 +DA:2,0 +DA:3,0 +DA:4,0 +LF:4 +LH:1 +end_of_record +'''); + + Error? commandError; + final List output = await runCapturingPrint( + runner, + ['coverage-check'], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains('Code coverage for plugin1 is 25.0%, which is below the required 50.0%.'), + ]), + ); + }); + + test('ignores generated files when calculating coverage', () async { + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/plugin1.dart +DA:1,1 +DA:2,1 +LF:2 +LH:2 +end_of_record +SF:lib/plugin1.g.dart +DA:1,0 +DA:2,0 +DA:3,0 +DA:4,0 +LF:4 +LH:0 +end_of_record +'''); + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect(output, contains(contains('Ran for 1 package(s)'))); + }); + + test('calculates coverage correctly across multiple files', () async { + final RepositoryPackage plugin = createFakePlugin( + 'plugin1', + packagesDir, + extraFiles: ['test/empty_test.dart'], + ); + + final Directory coverageDir = plugin.directory.childDirectory('coverage'); + coverageDir.createSync(); + coverageDir.childFile('lcov.info').writeAsStringSync(''' +SF:lib/file1.dart +DA:1,1 +DA:2,0 +LF:2 +LH:1 +end_of_record +SF:lib/file2.dart +DA:1,1 +DA:2,1 +DA:3,1 +LF:3 +LH:3 +end_of_record +'''); // Total LF: 5, LH: 4 => 80.0% coverage. 80.0 >= 50.0 so it should pass. + + final List output = await runCapturingPrint(runner, ['coverage-check']); + + expect(output, contains(contains('Ran for 1 package(s)'))); + }); + + test('fails when test command fails', () async { + createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + processRunner.mockProcessesForExecutable['flutter'] = [ + FakeProcessInfo(MockProcess(exitCode: 1), ['test', '--coverage']), + ]; + + Error? commandError; + final List output = await runCapturingPrint( + runner, + ['coverage-check'], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([contains('Failed to run tests or parse coverage')]), + ); + }); + + test('fails when lcov.info is missing', () async { + createFakePlugin('plugin1', packagesDir, extraFiles: ['test/empty_test.dart']); + + Error? commandError; + final List output = await runCapturingPrint( + runner, + ['coverage-check'], + errorHandler: (Error e) { + commandError = e; + }, + ); + + expect(commandError, isA()); + expect( + output, + containsAllInOrder([ + contains( + 'Coverage file not found at ${packagesDir.childDirectory('plugin1').childDirectory('coverage').childFile('lcov.info').path}.', + ), + contains('Failed to run tests or parse coverage'), + ]), + ); + }); + }); +}