From 95c21aea3ad50c1f08a0aeadd7c0757c46f42dc3 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Wed, 23 Aug 2023 10:56:29 +0800 Subject: [PATCH 1/2] fix(providers/azure): remove json.dumps when querying cosmos json.dumps leads to "azure.cosmos.exceptions.CosmosHttpResponseError: (BadRequest) Message: {"Errors":["Invalid query. Specified parameterized query JSON is malformed."]}" --- airflow/providers/microsoft/azure/hooks/cosmos.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/airflow/providers/microsoft/azure/hooks/cosmos.py b/airflow/providers/microsoft/azure/hooks/cosmos.py index 3a73eeada7d7d..91f66bf351e07 100644 --- a/airflow/providers/microsoft/azure/hooks/cosmos.py +++ b/airflow/providers/microsoft/azure/hooks/cosmos.py @@ -25,7 +25,6 @@ """ from __future__ import annotations -import json import uuid from typing import Any @@ -150,7 +149,7 @@ def does_collection_exist(self, collection_name: str, database_name: str) -> boo .get_database_client(self.__get_database_name(database_name)) .query_containers( "SELECT * FROM r WHERE r.id=@id", - parameters=[json.dumps({"name": "@id", "value": collection_name})], + parameters=[{"name": "@id", "value": collection_name}], # type: ignore[list-item] ) ) if not existing_container: @@ -175,7 +174,7 @@ def create_collection( .get_database_client(self.__get_database_name(database_name)) .query_containers( "SELECT * FROM r WHERE r.id=@id", - parameters=[json.dumps({"name": "@id", "value": collection_name})], + parameters=[{"name": "@id", "value": collection_name}], # type: ignore[list-item] ) ) @@ -193,7 +192,7 @@ def does_database_exist(self, database_name: str) -> bool: existing_database = list( self.get_conn().query_databases( "SELECT * FROM r WHERE r.id=@id", - parameters=[json.dumps({"name": "@id", "value": database_name})], + parameters=[{"name": "@id", "value": database_name}], # type: ignore[list-item] ) ) if not existing_database: @@ -211,7 +210,7 @@ def create_database(self, database_name: str) -> None: existing_database = list( self.get_conn().query_databases( "SELECT * FROM r WHERE r.id=@id", - parameters=[json.dumps({"name": "@id", "value": database_name})], + parameters=[{"name": "@id", "value": database_name}], # type: ignore[list-item] ) ) From fe2567ad8df5fc9b075e6800736dd2ef7e8a5bd4 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Sat, 26 Aug 2023 00:27:11 +0800 Subject: [PATCH 2/2] docs(providers/microsoft): add comment for the type ignoring --- airflow/providers/microsoft/azure/hooks/cosmos.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airflow/providers/microsoft/azure/hooks/cosmos.py b/airflow/providers/microsoft/azure/hooks/cosmos.py index 91f66bf351e07..4b42217e68ff9 100644 --- a/airflow/providers/microsoft/azure/hooks/cosmos.py +++ b/airflow/providers/microsoft/azure/hooks/cosmos.py @@ -144,6 +144,8 @@ def does_collection_exist(self, collection_name: str, database_name: str) -> boo if collection_name is None: raise AirflowBadRequest("Collection name cannot be None.") + # The ignores below is due to typing bug in azure-cosmos 9.2.0 + # https://github.com/Azure/azure-sdk-for-python/issues/31811 existing_container = list( self.get_conn() .get_database_client(self.__get_database_name(database_name)) @@ -169,6 +171,8 @@ def create_collection( # We need to check to see if this container already exists so we don't try # to create it twice + # The ignores below is due to typing bug in azure-cosmos 9.2.0 + # https://github.com/Azure/azure-sdk-for-python/issues/31811 existing_container = list( self.get_conn() .get_database_client(self.__get_database_name(database_name)) @@ -189,6 +193,8 @@ def does_database_exist(self, database_name: str) -> bool: if database_name is None: raise AirflowBadRequest("Database name cannot be None.") + # The ignores below is due to typing bug in azure-cosmos 9.2.0 + # https://github.com/Azure/azure-sdk-for-python/issues/31811 existing_database = list( self.get_conn().query_databases( "SELECT * FROM r WHERE r.id=@id", @@ -207,6 +213,8 @@ def create_database(self, database_name: str) -> None: # We need to check to see if this database already exists so we don't try # to create it twice + # The ignores below is due to typing bug in azure-cosmos 9.2.0 + # https://github.com/Azure/azure-sdk-for-python/issues/31811 existing_database = list( self.get_conn().query_databases( "SELECT * FROM r WHERE r.id=@id",