feat(gsheets2): retrieve unformatted values instead of formatted ones (…

…#214)

* fix(gsheets2): added keyword to status

* cleaned

* Reformated

* doc(google-sheets): add link to API param valueRenderOption

* Fixed test_get_decimal_separator

* Updated code connector & test with hidden properties

* v0.41.4

Co-authored-by: Raphaël <raphael.vignes@toucantoco.com>
Co-authored-by: David Nowinsky <david@toucantoco.com>
Co-authored-by: raphaelvignes <71276735+raphaelvignes@users.noreply.github.com>

doc/connectors/google_sheets_2.md

@@ -34,6 +34,7 @@ the url: https://docs.google.com/spreadsheets/d/<spreadsheet_id_is_here>/edit?pr
 * `sheet`: str. By default, the extractor returns the first sheet.
 * `header_row`: int, default to 0. Row of the header of the spreadsheet
 
+Values are retrieved with the parameter [valueRenderOption](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/get#body.QUERY_PARAMETERS.value_render_option) set to `UNFORMATTED_VALUE` to escape the rendering based on the locale defined in google sheets.
 
 ```coffee
 DATA_SOURCES: [
@@ -45,4 +46,4 @@ DATA_SOURCES: [
 ,
   ...
 ]
-```
+```

setup.py

@@ -66,7 +66,7 @@ def get_static_file_paths():
 
 setup(
     name='toucan_connectors',
-    version='0.41.3',
+    version='0.41.4',
     description='Toucan Toco Connectors',
     long_description=(HERE / 'README.md').read_text(encoding='utf-8'),
     long_description_content_type='text/markdown',

tests/google_sheets_2/test_google_sheets_2.py

@@ -180,7 +180,7 @@ def test_spreadsheet_without_sheet(mocker, con, ds_without_sheet, fake_kwargs):
     """
 
     def mock_api_responses(uri: str, _token):
-        if uri.endswith('/Foo'):
+        if '/Foo' in uri:
             return FAKE_SHEET
         else:
             return FAKE_SHEET_LIST_RESPONSE
@@ -197,7 +197,7 @@ def mock_api_responses(uri: str, _token):
     )
     assert (
         fetch_mock.call_args_list[1][0][0]
-        == 'https://sheets.googleapis.com/v4/spreadsheets/1SMnhnmBm-Tup3SfhS03McCf6S4pS2xqjI6CAXSSBpHU/values/Foo'
+        == 'https://sheets.googleapis.com/v4/spreadsheets/1SMnhnmBm-Tup3SfhS03McCf6S4pS2xqjI6CAXSSBpHU/values/Foo?valueRenderOption=UNFORMATTED_VALUE'
     )
 
     assert df.shape == (2, 2)
@@ -235,3 +235,13 @@ def test_get_status_api_down(mocker, con, fake_kwargs):
     mocker.patch.object(GoogleSheets2Connector, '_run_fetch', side_effect=HttpError)
 
     assert con.get_status(**fake_kwargs).status is False
+
+
+def test_get_decimal_separator(mocker, con, ds, fake_kwargs):
+    """
+    It should returns number data in float type
+    """
+    fake_results = {'metadata': '...', 'values': [['Number'], [1.3], [1.2]]}
+    mocker.patch.object(GoogleSheets2Connector, '_run_fetch', return_value=fake_results)
+    df = con.get_df(ds, **fake_kwargs)
+    assert df.to_dict() == {'Number': {1: 1.3, 2: 1.2}}

toucan_connectors/google_sheets_2/google_sheets_2_connector.py

@@ -48,7 +48,7 @@ def get_form(cls, connector: 'GoogleSheets2Connector', current_config, **kwargs)
         with suppress(Exception):
             partial_endpoint = current_config['spreadsheet_id']
             final_url = f'{connector._baseroute}{partial_endpoint}'
-            secrets = kwargs.get('secrets')(connector.auth_flow_id)
+            secrets = kwargs.get('secrets')(auth_flow_id=connector.auth_flow_id)
             data = connector._run_fetch(final_url, secrets['access_token'])
             available_sheets = [str(x['properties']['title']) for x in data['sheets']]
             constraints['sheet'] = strlist_to_enum('sheet', available_sheets)
@@ -106,9 +106,9 @@ def _retrieve_data(self, data_source: GoogleSheets2DataSource, **kwargs) -> pd.D
             data_source.sheet = available_sheets[0]
 
         # https://developers.google.com/sheets/api/samples/reading
-        read_sheet_endpoint = f'{data_source.spreadsheet_id}/values/{data_source.sheet}'
+        read_sheet_endpoint = f'{data_source.spreadsheet_id}/values/{data_source.sheet}?valueRenderOption=UNFORMATTED_VALUE'
         full_url = f'{self._baseroute}{read_sheet_endpoint}'
-
+        # Rajouter le param FORMATTED_VALUE pour le séparateur de décimal dans la Baseroute
         data = self._run_fetch(full_url, access_token)['values']
         df = pd.DataFrame(data)
 
@@ -133,7 +133,7 @@ def get_status(self, **kwargs) -> ConnectorStatus:
         If successful, returns a message with the email of the connected user account.
         """
         try:
-            secrets = kwargs.get('secrets')(self.auth_flow_id)
+            secrets = kwargs.get('secrets')(auth_flow_id=self.auth_flow_id)
             access_token = secrets['access_token']
         except Exception:
             return ConnectorStatus(status=False, error='Credentials are missing')