fix(elasticsearch): disable verification

CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Changed
 
 - Google Big Query: A simple status check that validates the private key's format has been implemented
+- Elasticsearch: Host verification has been disabled to tolerate strict network configurations
 
 ## [3.23.25] 2024-01-17
 

tests/elasticsearch/test_elasticsearch.py

@@ -39,13 +39,10 @@ def pytest_generate_tests(metafunc):
 
 
 def test_connector(mocker):
-    class ElasticsearchMock:
-        def search(self, index, body):
-            return {'hits': {'hits': [{'_source': {'yo': 'la'}}]}}
 
     module = 'toucan_connectors.elasticsearch.elasticsearch_connector'
     mock_es = mocker.patch(f'{module}.Elasticsearch')
-    mock_es.return_value = ElasticsearchMock()
+    mock_es.return_value.search.return_value = {'hits': {'hits': [{'_source': {'yo': 'la'}}]}}
 
     con = ElasticsearchConnector(
         name='test',

toucan_connectors/elasticsearch/elasticsearch_connector.py

@@ -145,6 +145,14 @@ def _retrieve_data(self, data_source: ElasticsearchDataSource) -> pd.DataFrame:
             connection_params.append(h)
 
         esclient = Elasticsearch(connection_params)
+        # We need to set this flag as some customers force auth and refuse the connection if no auth
+        # header is present. Elasticsearch-py accepts 401/403s
+        # (https://github.com/elastic/elasticsearch-py/blob/v7.17.6/elasticsearch/transport.py#L586),
+        # but not connection errors. In consequence, we set the flag to True, which means that we
+        # couldn't figure out wether we are talking to Elasticsearch or not due to an auth error:
+        # https://github.com/elastic/elasticsearch-py/blob/v7.17.6/elasticsearch/transport.py#L216.
+        # If we are indded not talking to Elasticsearch, the query will fail later on.
+        esclient.transport._verified_elasticsearch = True
         response = getattr(esclient, data_source.search_method)(
             index=data_source.index, body=data_source.body
         )