Switch cloudant provider from cloudant library to ibmcloudant library (…

…#41555)

.github/workflows/check-providers.yml

@@ -108,6 +108,11 @@ jobs:
         run: >
           breeze release-management generate-issue-content-providers
           --only-available-in-dist --disable-progress
+      - name: >
+          Remove incompatible Python ${{ matrix.python-version }} provider packages
+        run: |
+          echo "Removing Python 3.8-incompatible provider: cloudant"
+          rm -vf dist/apache_airflow_providers_cloudant*
       - name: "Generate source constraints from CI image"
         shell: bash
         run: >

airflow/providers/cloudant/CHANGELOG.rst

@@ -27,6 +27,14 @@
 Changelog
 ---------
 
+4.0.0
+.....
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+* ``Switch cloudant provider from cloudant library to ibmcloudant library (#41555)``
+
 3.6.0
 .....
 

airflow/providers/cloudant/cloudant_fake.py

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+
+class CloudantV1:
+    """Phony class to pass mypy when real class is not imported."""
+
+    def __init__(self, authenticator):
+        pass
+
+    def set_service_url(self, service_url: str):
+        pass
+
+
+class CouchDbSessionAuthenticator:
+    """Phony class to pass mypy when real class is not imported."""
+
+    def __init__(self, username: str, password: str):
+        pass

airflow/providers/cloudant/hooks/cloudant.py

@@ -19,13 +19,20 @@
 
 from __future__ import annotations
 
-from typing import Any
+import sys
+from typing import TYPE_CHECKING, Any
 
-from cloudant import cloudant  # type: ignore[attr-defined]
+if sys.version_info < (3, 10):
+    from airflow.providers.cloudant.cloudant_fake import CloudantV1, CouchDbSessionAuthenticator
+else:
+    from ibmcloudant import CloudantV1, CouchDbSessionAuthenticator
 
 from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 
+if TYPE_CHECKING:
+    from airflow.models import Connection
+
 
 class CloudantHook(BaseHook):
     """
@@ -45,35 +52,44 @@ class CloudantHook(BaseHook):
     def get_ui_field_behaviour(cls) -> dict[str, Any]:
         """Return custom field behaviour."""
         return {
-            "hidden_fields": ["port", "extra"],
-            "relabeling": {"host": "Account", "login": "Username (or API Key)", "schema": "Database"},
+            "hidden_fields": ["schema", "port", "extra"],
+            "relabeling": {"host": "Account", "login": "Username (or API Key)"},
         }
 
     def __init__(self, cloudant_conn_id: str = default_conn_name) -> None:
         super().__init__()
         self.cloudant_conn_id = cloudant_conn_id
 
-    def get_conn(self) -> cloudant:
+    def get_conn(self) -> CloudantV1:
         """
-        Open a connection to the cloudant service and close it automatically if used as context manager.
+        Create an authenticated service object for connection to the Cloudant service.
 
         .. note::
             In the connection form:
-            - 'host' equals the 'Account' (optional)
+            - 'host' equals the 'Account' (required)
             - 'login' equals the 'Username (or API Key)' (required)
             - 'password' equals the 'Password' (required)
 
-        :return: an authorized cloudant session context manager object.
+        :return: a CloudantV1 service object backed by a session-based user/password authenticator.
         """
         conn = self.get_connection(self.cloudant_conn_id)
 
         self._validate_connection(conn)
 
-        cloudant_session = cloudant(user=conn.login, passwd=conn.password, account=conn.host)
+        authenticator = CouchDbSessionAuthenticator(username=conn.login, password=conn.password)
+        service = CloudantV1(authenticator=authenticator)
+        service.set_service_url(f"https://{conn.host}.cloudant.com")
 
-        return cloudant_session
+        return service
 
-    def _validate_connection(self, conn: cloudant) -> None:
-        for conn_param in ["login", "password"]:
+    @staticmethod
+    def _validate_connection(conn: Connection) -> None:
+        missing_params = []
+        for conn_param in ["host", "login", "password"]:
             if not getattr(conn, conn_param):
-                raise AirflowException(f"missing connection parameter {conn_param}")
+                missing_params.append(conn_param)
+
+        if missing_params:
+            raise AirflowException(
+                f"Missing connection parameter{'s' if len(missing_params) > 1 else ''}: {', '.join(missing_params)}"
+            )

airflow/providers/cloudant/provider.yaml

@@ -25,6 +25,7 @@ state: ready
 source-date-epoch: 1723969866
 # note that those versions are maintained by release manager - do not update them manually
 versions:
+  - 4.0.0
   - 3.6.0
   - 3.5.2
   - 3.5.1
@@ -46,7 +47,15 @@ versions:
 
 dependencies:
   - apache-airflow>=2.8.0
-  - cloudant>=2.13.0
+  - ibmcloudant>=0.9.1
+
+excluded-python-versions:
+  # ibmcloudant transitively brings in urllib3 2.x, but the snowflake provider has a dependency that pins
+  # urllib3 to 1.x on Python 3.8 and 3.9; thus we exclude those Python versions from taking the update
+  # to ibmcloudant.
+  # See #21004, #41555, and https://github.com/snowflakedb/snowflake-connector-python/issues/2016
+  - "3.8"
+  - "3.9"
 
 integrations:
   - integration-name: IBM Cloudant

dev/breeze/src/airflow_breeze/global_constants.py

@@ -509,19 +509,19 @@ def get_airflow_extras():
     {
         "python-version": "3.8",
         "airflow-version": "2.8.4",
-        "remove-providers": "fab",
+        "remove-providers": "cloudant fab",
         "run-tests": "true",
     },
     {
         "python-version": "3.8",
         "airflow-version": "2.9.3",
-        "remove-providers": "",
+        "remove-providers": "cloudant",
         "run-tests": "true",
     },
     {
         "python-version": "3.8",
         "airflow-version": "2.10.0",
-        "remove-providers": "",
+        "remove-providers": "cloudant",
         "run-tests": "true",
     },
 ]

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -81,7 +81,6 @@
 USE_PUBLIC_RUNNERS_LABEL = "use public runners"
 USE_SELF_HOSTED_RUNNERS_LABEL = "use self-hosted runners"
 
-
 ALL_CI_SELECTIVE_TEST_TYPES = (
     "API Always BranchExternalPython BranchPythonVenv "
     "CLI Core ExternalPython Operators Other PlainAsserts "
@@ -1278,7 +1277,10 @@ def excluded_providers_as_string(self) -> str:
             if "excluded-python-versions" in provider_info:
                 for python_version in provider_info["excluded-python-versions"]:
                     providers_to_exclude[python_version].append(provider)
-        return json.dumps(providers_to_exclude)
+        sorted_providers_to_exclude = dict(
+            sorted(providers_to_exclude.items(), key=lambda item: int(item[0].split(".")[1]))
+        )  # ^ sort by Python minor version
+        return json.dumps(sorted_providers_to_exclude)
 
     @cached_property
     def testable_integrations(self) -> list[str]:

dev/breeze/tests/test_selective_checks.py

@@ -853,7 +853,9 @@ def test_excluded_providers():
     )
     assert_outputs_are_printed(
         {
-            "excluded-providers-as-string": json.dumps({"3.12": ["apache.beam"]}),
+            "excluded-providers-as-string": json.dumps(
+                {"3.8": ["cloudant"], "3.9": ["cloudant"], "3.12": ["apache.beam"]}
+            ),
         },
         str(stderr),
     )

docs/spelling_wordlist.txt

@@ -250,8 +250,8 @@ classpaths
 cleartext
 cli
 clientId
-Cloudant
 cloudant
+CloudantV
 cloudbuild
 CloudBuildClient
 cloudml
@@ -775,6 +775,7 @@ hyperparameter
 hyperparameters
 IaC
 iam
+ibmcloudant
 idempotence
 idempotency
 IdP

generated/provider_dependencies.json

@@ -352,12 +352,15 @@
   "cloudant": {
     "deps": [
       "apache-airflow>=2.8.0",
-      "cloudant>=2.13.0"
+      "ibmcloudant>=0.9.1"
     ],
     "devel-deps": [],
     "plugins": [],
     "cross-providers-deps": [],
-    "excluded-python-versions": [],
+    "excluded-python-versions": [
+      "3.8",
+      "3.9"
+    ],
     "state": "ready"
   },
   "cncf.kubernetes": {

tests/always/test_project_structure.py

@@ -85,6 +85,7 @@ def test_providers_modules_should_have_tests(self):
             "tests/providers/apache/hive/plugins/test_hive.py",
             "tests/providers/celery/executors/test_celery_executor_utils.py",
             "tests/providers/celery/executors/test_default_celery.py",
+            "tests/providers/cloudant/test_cloudant_fake.py",
             "tests/providers/cncf/kubernetes/backcompat/test_backwards_compat_converters.py",
             "tests/providers/cncf/kubernetes/executors/test_kubernetes_executor_types.py",
             "tests/providers/cncf/kubernetes/executors/test_kubernetes_executor_utils.py",

tests/providers/cloudant/hooks/test_cloudant.py

@@ -17,15 +17,24 @@
 # under the License.
 from __future__ import annotations
 
+import sys
 from unittest.mock import patch
 
 import pytest
 
 from airflow.exceptions import AirflowException
 from airflow.models import Connection
-from airflow.providers.cloudant.hooks.cloudant import CloudantHook
 
-pytestmark = pytest.mark.db_test
+pytestmark = [pytest.mark.db_test]
+
+if sys.version_info < (3, 10):
+    pytestmark.append(
+        pytest.mark.skip(
+            f"Skipping {__name__} as the cloudant provider is not supported on Python 3.8 and 3.9, see #41555."
+        )
+    )
+else:
+    from airflow.providers.cloudant.hooks.cloudant import CloudantHook
 
 
 class TestCloudantHook:
@@ -34,20 +43,39 @@ def setup_method(self):
 
     @patch(
         "airflow.providers.cloudant.hooks.cloudant.CloudantHook.get_connection",
-        return_value=Connection(login="user", password="password", host="account"),
+        return_value=Connection(login="the_user", password="the_password", host="the_account"),
     )
-    @patch("airflow.providers.cloudant.hooks.cloudant.cloudant")
-    def test_get_conn(self, mock_cloudant, mock_get_connection):
+    @patch("airflow.providers.cloudant.hooks.cloudant.CouchDbSessionAuthenticator")
+    @patch("airflow.providers.cloudant.hooks.cloudant.CloudantV1")
+    def test_get_conn_passes_expected_params_and_returns_cloudant_object(
+        self, mock_cloudant_v1, mock_session_authenticator, mock_get_connection
+    ):
         cloudant_session = self.cloudant_hook.get_conn()
 
         conn = mock_get_connection.return_value
-        mock_cloudant.assert_called_once_with(user=conn.login, passwd=conn.password, account=conn.host)
-        assert cloudant_session == mock_cloudant.return_value
 
-    @patch(
-        "airflow.providers.cloudant.hooks.cloudant.CloudantHook.get_connection",
-        return_value=Connection(login="user"),
+        mock_session_authenticator.assert_called_once_with(username=conn.login, password=conn.password)
+        mock_cloudant_v1.assert_called_once_with(authenticator=mock_session_authenticator.return_value)
+
+        cloudant_service = mock_cloudant_v1.return_value
+        cloudant_service.set_service_url.assert_called_once_with(f"https://{conn.host}.cloudant.com")
+
+        assert cloudant_session == cloudant_service
+
+    @pytest.mark.parametrize(
+        "conn",
+        [
+            Connection(),
+            Connection(host="acct"),
+            Connection(login="user"),
+            Connection(password="pwd"),
+            Connection(host="acct", login="user"),
+            Connection(host="acct", password="pwd"),
+            Connection(login="user", password="pwd"),
+        ],
     )
-    def test_get_conn_invalid_connection(self, mock_get_connection):
+    @patch("airflow.providers.cloudant.hooks.cloudant.CloudantHook.get_connection")
+    def test_get_conn_invalid_connection(self, mock_get_connection, conn):
+        mock_get_connection.return_value = conn
         with pytest.raises(AirflowException):
             self.cloudant_hook.get_conn()