From 7a3636cfbcc78cf393a0d2ebff49854b2d6b35b1 Mon Sep 17 00:00:00 2001 From: William Gorge Date: Tue, 23 Jan 2024 15:16:37 +0100 Subject: [PATCH 1/5] fix(elasticsearch): disable verification --- toucan_connectors/elasticsearch/elasticsearch_connector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/toucan_connectors/elasticsearch/elasticsearch_connector.py b/toucan_connectors/elasticsearch/elasticsearch_connector.py index 288b9521b..81316ab56 100644 --- a/toucan_connectors/elasticsearch/elasticsearch_connector.py +++ b/toucan_connectors/elasticsearch/elasticsearch_connector.py @@ -145,6 +145,7 @@ def _retrieve_data(self, data_source: ElasticsearchDataSource) -> pd.DataFrame: connection_params.append(h) esclient = Elasticsearch(connection_params) + esclient.transport._verified_elasticsearch = True response = getattr(esclient, data_source.search_method)( index=data_source.index, body=data_source.body ) From c72bfcdc5b5de7f3833f695f70ab5597e12938e6 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 23 Jan 2024 16:31:07 +0100 Subject: [PATCH 2/5] add doc comment Signed-off-by: Luka Peschke --- toucan_connectors/elasticsearch/elasticsearch_connector.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/toucan_connectors/elasticsearch/elasticsearch_connector.py b/toucan_connectors/elasticsearch/elasticsearch_connector.py index 81316ab56..4dea15044 100644 --- a/toucan_connectors/elasticsearch/elasticsearch_connector.py +++ b/toucan_connectors/elasticsearch/elasticsearch_connector.py @@ -145,6 +145,13 @@ def _retrieve_data(self, data_source: ElasticsearchDataSource) -> pd.DataFrame: connection_params.append(h) esclient = Elasticsearch(connection_params) + # We need to set this flag as some customers force auth and refuse the connection if no auth + # header is present. Elasticsearch-py accepts 401/403s + # (https://github.com/elastic/elasticsearch-py/blob/v7.17.6/elasticsearch/transport.py#L586), + # but not connection errors. In consequence, we set the flag to True, which means that we + # couldn't figure out wether we are talking to Elasticsearch or not due to an auth error: + # https://github.com/elastic/elasticsearch-py/blob/v7.17.6/elasticsearch/transport.py#L216. + # If we are indded not talking to Elasticsearch, the query will fail later on. esclient.transport._verified_elasticsearch = True response = getattr(esclient, data_source.search_method)( index=data_source.index, body=data_source.body From 20ada48e59ab67a993302c8ab5e2d1da3e2dde1b Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 23 Jan 2024 16:35:31 +0100 Subject: [PATCH 3/5] fix tests Signed-off-by: Luka Peschke --- tests/elasticsearch/test_elasticsearch.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/elasticsearch/test_elasticsearch.py b/tests/elasticsearch/test_elasticsearch.py index a7beb5148..1fe9fa8c0 100644 --- a/tests/elasticsearch/test_elasticsearch.py +++ b/tests/elasticsearch/test_elasticsearch.py @@ -39,13 +39,10 @@ def pytest_generate_tests(metafunc): def test_connector(mocker): - class ElasticsearchMock: - def search(self, index, body): - return {'hits': {'hits': [{'_source': {'yo': 'la'}}]}} module = 'toucan_connectors.elasticsearch.elasticsearch_connector' mock_es = mocker.patch(f'{module}.Elasticsearch') - mock_es.return_value = ElasticsearchMock() + mock_es.return_value.search.return_value = {'hits': {'hits': [{'_source': {'yo': 'la'}}]}} con = ElasticsearchConnector( name='test', From b44ce10e21efa0f549c297fbdbbfbb78e5b1ca9c Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 23 Jan 2024 16:37:00 +0100 Subject: [PATCH 4/5] update changelog Signed-off-by: Luka Peschke --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdbd72c78..583bdd197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Changed - Google Big Query: A simple status check that validates the private key's format has been implemented +- Elasticsearch: Host verification has been disabled to tolerate strict network configurations ## [3.23.25] 2024-01-17 From 83e20988852f583a918876213f8afe7e8440050e Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 23 Jan 2024 16:37:53 +0100 Subject: [PATCH 5/5] trigger CI Signed-off-by: Luka Peschke