diff --git a/scripts/ci/prek/check_provider_yaml_files.py b/scripts/ci/prek/check_provider_yaml_files.py index 3f3ec5183a5d0..5e8d8da7a9817 100755 --- a/scripts/ci/prek/check_provider_yaml_files.py +++ b/scripts/ci/prek/check_provider_yaml_files.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # /// script -# requires-python = ">=3.10,<3.11" +# requires-python = ">=3.10" # dependencies = [ # "rich>=13.6.0", # ] @@ -27,14 +27,12 @@ import sys from common_prek_utils import ( - KNOWN_SECOND_LEVEL_PATHS, + get_provider_base_dir_from_path, initialize_breeze_prek, run_command_via_breeze_run, validate_cmd_result, ) -initialize_breeze_prek(__name__, __file__) - def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: """ @@ -46,11 +44,12 @@ def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: conn-fields check runs even when only the hook changes. All paths are relative to the ``providers/`` directory, as supplied by - prek. The first path segment is the provider package name - (e.g. ``samba/src/airflow/...`` → ``samba/provider.yaml``), except for - namespace packages in ``KNOWN_SECOND_LEVEL_PATHS`` (e.g. ``apache``, - ``common``), which nest an extra level (e.g. - ``apache/beam/src/airflow/...`` → ``apache/beam/provider.yaml``). + prek. Rather than guessing how many path segments make up the provider + package name, this walks up the real directory tree (via + ``get_provider_base_dir_from_path``) until it finds the actual + ``provider.yaml`` file, so nested/namespace provider packages + (e.g. ``apache/beam``, ``ibm/mq``) resolve correctly without maintaining + a list of known namespace prefixes. """ result: set[str] = set() for f in raw_files: @@ -58,23 +57,20 @@ def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: if p.name == "provider.yaml": result.add(f) else: - # Map any Python file to the provider.yaml of its package root. - # Path structure: / - # e.g. samba/src/airflow/providers/samba/hooks/samba.py - parts = p.parts - if parts: - if parts[0] in KNOWN_SECOND_LEVEL_PATHS and len(parts) > 1: - result.add(f"{parts[0]}/{parts[1]}/provider.yaml") - else: - result.add(f"{parts[0]}/provider.yaml") + provider_dir = get_provider_base_dir_from_path(pathlib.Path(f)) + if provider_dir is not None: + result.add((provider_dir / "provider.yaml").as_posix()) return sorted(result) -files_to_test = _resolve_provider_yaml_files(sys.argv[1:]) -cmd_result = run_command_via_breeze_run( - ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], - backend="sqlite", - warn_image_upgrade_needed=True, - extra_env={"PYTHONWARNINGS": "default"}, -) -validate_cmd_result(cmd_result, include_ci_env_check=True) +if __name__ == "__main__": + initialize_breeze_prek(__name__, __file__) + + files_to_test = _resolve_provider_yaml_files(sys.argv[1:]) + cmd_result = run_command_via_breeze_run( + ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], + backend="sqlite", + warn_image_upgrade_needed=True, + extra_env={"PYTHONWARNINGS": "default"}, + ) + validate_cmd_result(cmd_result, include_ci_env_check=True) diff --git a/scripts/tests/ci/prek/test_check_provider_yaml_files.py b/scripts/tests/ci/prek/test_check_provider_yaml_files.py new file mode 100644 index 0000000000000..590ad73efb2ef --- /dev/null +++ b/scripts/tests/ci/prek/test_check_provider_yaml_files.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +from ci.prek.check_provider_yaml_files import _resolve_provider_yaml_files + + +def _touch(path): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + +class TestResolveProviderYamlFiles: + @pytest.mark.parametrize( + "file_names, raw_files, expected", + ( + ( + ["samba/provider.yaml"], + ["samba/provider.yaml"], + ["samba/provider.yaml"], + ), + ( + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + ], + ["samba/src/airflow/providers/samba/hooks/samba.py"], + ["samba/provider.yaml"], + ), + ( + [ + "ibm/mq/provider.yaml", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + ["ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py"], + ["ibm/mq/provider.yaml"], + ), + ( + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + "ibm/mq/provider.yaml", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + ["ibm/mq/provider.yaml", "samba/provider.yaml"], + ), + (["unrelated/hooks/hook.py"], ["unrelated/hooks/hook.py"], []), + ), + ids=[ + "provider_yaml_path_preserved", + "top_level_provider_hook_file_resolves", + "nested_namespace_provider_hook_file_resolves_without_known_list", + "mixed_input_dedups_and_sorts", + "no_provider_found", + ], + ) + def test__resolve_provider_yaml_files(self, tmp_path, monkeypatch, file_names, raw_files, expected): + monkeypatch.chdir(tmp_path) + for file_name in file_names: + _touch(tmp_path / file_name) + + result = _resolve_provider_yaml_files(raw_files) + + assert result == expected