Improve pyproject.toml synchronization hook with fallback logic (#61597)

potiuk · web-flow · commit e7ecdbda7cf5 · 2026-02-07T16:11:28.000+01:00
Add fallback mechanism for provider minimum version detection when
metadata is unavailable. The hook now attempts to read local provider
pyproject.toml files as a fallback before giving up.

Changes:
- Add get_local_provider_version() to read from local pyproject.toml
- Extract fallback logic into _fallback_provider_version() helper
- Add extra_information parameter to insert_documentation() for clearer
  update messages showing what was synchronized

This ensures the hook works correctly even for providers without
published metadata, improving reliability during development.
diff --git a/scripts/ci/prek/common_prek_utils.py b/scripts/ci/prek/common_prek_utils.py
@@ -129,7 +129,12 @@ def pre_process_files(files: list[str]) -> list[str]:
 
 
 def insert_documentation(
-    file_path: Path, content: list[str], header: str, footer: str, add_comment: bool = False
+    file_path: Path,
+    content: list[str],
+    header: str,
+    footer: str,
+    add_comment: bool = False,
+    extra_information: str | None = None,
 ) -> bool:
     found = False
     old_content = file_path.read_text()
@@ -155,7 +160,7 @@ def insert_documentation(
         sys.exit(1)
     if new_content != old_content:
         file_path.write_text(new_content)
-        console.print(f"Updated {file_path}")
+        console.print(f"Updated {file_path} with {extra_information or 'generated documentation'}")
         return True
     return False
 
diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py b/scripts/ci/prek/update_airflow_pyproject_toml.py
@@ -108,7 +108,49 @@ def provider_path(provider_id: str) -> str:
 all_providers_dependencies = json.loads(PROVIDER_DEPENDENCIES_FILE_PATH.read_text())
 
 
+def _read_toml(path: Path) -> dict[str, Any]:
+    try:
+        import tomllib
+    except ImportError:
+        import tomli as tomllib
+    return tomllib.loads(path.read_text())
+
+
+def get_local_provider_version(provider_id: str) -> Version | None:
+    provider_pyproject = PROVIDERS_DIR / provider_path(provider_id) / "pyproject.toml"
+    if not provider_pyproject.exists():
+        return None
+    try:
+        provider_toml = _read_toml(provider_pyproject)
+    except Exception:
+        return None
+    version_str = provider_toml.get("project", {}).get("version")
+    if not version_str:
+        return None
+    return parse_version(version_str)
+
+
+def _fallback_provider_version(
+    provider_id: str, min_version_override: Version | None
+) -> tuple[Version | None, str]:
+    if min_version_override:
+        console.print(
+            f"[yellow]Provider id {provider_id} min version fallback:[/] "
+            f"MIN_VERSION_OVERRIDE -> {min_version_override}"
+        )
+        return min_version_override, f" # Set from MIN_VERSION_OVERRIDE in {Path(__file__).name}"
+    local_version = get_local_provider_version(provider_id)
+    if local_version:
+        console.print(
+            f"[yellow]Provider id {provider_id} min version fallback:[/] "
+            f"local provider pyproject.toml -> {local_version}"
+        )
+        return local_version, " # Set from local provider pyproject.toml"
+    return None, ""
+
+
 def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
+    console.print(f"[bright_blue]Finding min version for provider id:[/] {provider_id}")
     metadata = all_providers_metadata.get(provider_id)
     # We should periodically update the starting date to avoid pip install resolution issues
     # TODO: when min Python version is 3.11 change back the code to fromisoformat
@@ -119,10 +161,11 @@ def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
     last_version_newer_than_cutoff: Version | None = None
     date_released: datetime | None = None
     min_version_override = MIN_VERSION_OVERRIDE.get(provider_id)
+    override_comment = ""
     if not metadata:
-        if not min_version_override:
-            return None, ""
-        last_version_newer_than_cutoff = min_version_override
+        last_version_newer_than_cutoff, override_comment = _fallback_provider_version(
+            provider_id, min_version_override
+        )
     else:
         versions: list[Version] = sorted([parse_version(version) for version in metadata], reverse=True)
         for version in versions:
@@ -133,11 +176,19 @@ def find_min_provider_version(provider_id: str) -> tuple[Version | None, str]:
             if date_released < cut_off_date:
                 break
             last_version_newer_than_cutoff = version
-    console.print(
-        f"[bright_blue]Provider id {provider_id} min version found:[/] "
-        f"{last_version_newer_than_cutoff} (date {date_released}"
-    )
-    override_comment = ""
+        if not last_version_newer_than_cutoff:
+            last_version_newer_than_cutoff, override_comment = _fallback_provider_version(
+                provider_id, min_version_override
+            )
+    if date_released:
+        console.print(
+            f"[bright_blue]Provider id {provider_id} min version found:[/] "
+            f"{last_version_newer_than_cutoff} (date {date_released})"
+        )
+    else:
+        console.print(
+            f"[yellow]Provider id {provider_id} has no released versions newer than cutoff date {cut_off_date}![/]"
+        )
     if last_version_newer_than_cutoff:
         if min_version_override and min_version_override > last_version_newer_than_cutoff:
             console.print(
@@ -216,13 +267,17 @@ def get_python_exclusion(provider_dependencies: dict[str, Any]) -> str:
         all_optional_dependencies,
         START_OPTIONAL_DEPENDENCIES,
         END_OPTIONAL_DEPENDENCIES,
+        False,
+        "optional dependencies",
     )
     all_mypy_paths = []
     for provider_id in all_providers:
         provider_mypy_path = f"$MYPY_CONFIG_FILE_DIR/providers/{provider_path(provider_id)}"
         all_mypy_paths.append(f'    "{provider_mypy_path}/src",\n')
         all_mypy_paths.append(f'    "{provider_mypy_path}/tests",\n')
-    insert_documentation(AIRFLOW_PYPROJECT_TOML_FILE, all_mypy_paths, START_MYPY_PATHS, END_MYPY_PATHS)
+    insert_documentation(
+        AIRFLOW_PYPROJECT_TOML_FILE, all_mypy_paths, START_MYPY_PATHS, END_MYPY_PATHS, False, "mypy paths"
+    )
     all_workspace_items = []
     for provider_id in all_providers:
         all_workspace_items.append(f"{provider_distribution_name(provider_id)} = {{ workspace = true }}\n")
@@ -231,6 +286,8 @@ def get_python_exclusion(provider_dependencies: dict[str, Any]) -> str:
         all_workspace_items,
         START_WORKSPACE_ITEMS,
         END_WORKSPACE_ITEMS,
+        False,
+        "workspace items",
     )
     all_workspace_members = []
     for provider_id in all_providers:
@@ -240,4 +297,6 @@ def get_python_exclusion(provider_dependencies: dict[str, Any]) -> str:
         all_workspace_members,
         START_PROVIDER_WORKSPACE_MEMBERS,
         END_PROVIDER_WORKSPACE_MEMBERS,
+        False,
+        "provider workspace members",
     )
