From cee89cc50ec910418763d6a950d7a089608295f2 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 22 Jul 2026 23:01:17 +0800 Subject: [PATCH] Resolve nested provider paths for conn-fields check from the real directory tree The conn-fields prek hook guessed how many path segments made up a provider package name from a hardcoded list of known namespace prefixes, so it kept crashing on newly nested providers (e.g. IBM MQ) until someone manually patched the list. Walking the actual directory tree to find each provider's provider.yaml makes it work for any current or future nested provider automatically. --- scripts/ci/prek/check_provider_yaml_files.py | 48 +++++------ .../ci/prek/test_check_provider_yaml_files.py | 84 +++++++++++++++++++ 2 files changed, 106 insertions(+), 26 deletions(-) create mode 100644 scripts/tests/ci/prek/test_check_provider_yaml_files.py diff --git a/scripts/ci/prek/check_provider_yaml_files.py b/scripts/ci/prek/check_provider_yaml_files.py index 3f3ec5183a5d0..5e8d8da7a9817 100755 --- a/scripts/ci/prek/check_provider_yaml_files.py +++ b/scripts/ci/prek/check_provider_yaml_files.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # /// script -# requires-python = ">=3.10,<3.11" +# requires-python = ">=3.10" # dependencies = [ # "rich>=13.6.0", # ] @@ -27,14 +27,12 @@ import sys from common_prek_utils import ( - KNOWN_SECOND_LEVEL_PATHS, + get_provider_base_dir_from_path, initialize_breeze_prek, run_command_via_breeze_run, validate_cmd_result, ) -initialize_breeze_prek(__name__, __file__) - def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: """ @@ -46,11 +44,12 @@ def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: conn-fields check runs even when only the hook changes. All paths are relative to the ``providers/`` directory, as supplied by - prek. The first path segment is the provider package name - (e.g. ``samba/src/airflow/...`` → ``samba/provider.yaml``), except for - namespace packages in ``KNOWN_SECOND_LEVEL_PATHS`` (e.g. ``apache``, - ``common``), which nest an extra level (e.g. - ``apache/beam/src/airflow/...`` → ``apache/beam/provider.yaml``). + prek. Rather than guessing how many path segments make up the provider + package name, this walks up the real directory tree (via + ``get_provider_base_dir_from_path``) until it finds the actual + ``provider.yaml`` file, so nested/namespace provider packages + (e.g. ``apache/beam``, ``ibm/mq``) resolve correctly without maintaining + a list of known namespace prefixes. """ result: set[str] = set() for f in raw_files: @@ -58,23 +57,20 @@ def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: if p.name == "provider.yaml": result.add(f) else: - # Map any Python file to the provider.yaml of its package root. - # Path structure: / - # e.g. samba/src/airflow/providers/samba/hooks/samba.py - parts = p.parts - if parts: - if parts[0] in KNOWN_SECOND_LEVEL_PATHS and len(parts) > 1: - result.add(f"{parts[0]}/{parts[1]}/provider.yaml") - else: - result.add(f"{parts[0]}/provider.yaml") + provider_dir = get_provider_base_dir_from_path(pathlib.Path(f)) + if provider_dir is not None: + result.add((provider_dir / "provider.yaml").as_posix()) return sorted(result) -files_to_test = _resolve_provider_yaml_files(sys.argv[1:]) -cmd_result = run_command_via_breeze_run( - ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], - backend="sqlite", - warn_image_upgrade_needed=True, - extra_env={"PYTHONWARNINGS": "default"}, -) -validate_cmd_result(cmd_result, include_ci_env_check=True) +if __name__ == "__main__": + initialize_breeze_prek(__name__, __file__) + + files_to_test = _resolve_provider_yaml_files(sys.argv[1:]) + cmd_result = run_command_via_breeze_run( + ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], + backend="sqlite", + warn_image_upgrade_needed=True, + extra_env={"PYTHONWARNINGS": "default"}, + ) + validate_cmd_result(cmd_result, include_ci_env_check=True) diff --git a/scripts/tests/ci/prek/test_check_provider_yaml_files.py b/scripts/tests/ci/prek/test_check_provider_yaml_files.py new file mode 100644 index 0000000000000..590ad73efb2ef --- /dev/null +++ b/scripts/tests/ci/prek/test_check_provider_yaml_files.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +from ci.prek.check_provider_yaml_files import _resolve_provider_yaml_files + + +def _touch(path): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + +class TestResolveProviderYamlFiles: + @pytest.mark.parametrize( + "file_names, raw_files, expected", + ( + ( + ["samba/provider.yaml"], + ["samba/provider.yaml"], + ["samba/provider.yaml"], + ), + ( + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + ], + ["samba/src/airflow/providers/samba/hooks/samba.py"], + ["samba/provider.yaml"], + ), + ( + [ + "ibm/mq/provider.yaml", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + ["ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py"], + ["ibm/mq/provider.yaml"], + ), + ( + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + "ibm/mq/provider.yaml", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + ["ibm/mq/provider.yaml", "samba/provider.yaml"], + ), + (["unrelated/hooks/hook.py"], ["unrelated/hooks/hook.py"], []), + ), + ids=[ + "provider_yaml_path_preserved", + "top_level_provider_hook_file_resolves", + "nested_namespace_provider_hook_file_resolves_without_known_list", + "mixed_input_dedups_and_sorts", + "no_provider_found", + ], + ) + def test__resolve_provider_yaml_files(self, tmp_path, monkeypatch, file_names, raw_files, expected): + monkeypatch.chdir(tmp_path) + for file_name in file_names: + _touch(tmp_path / file_name) + + result = _resolve_provider_yaml_files(raw_files) + + assert result == expected