Error uploading dataframes

[jgoizueta]

The last cell of the dataset example fails with this error:

TypeError                                 Traceback (most recent call last)
~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/pandas/core/indexes/base.py in get_value(self, series, key)
   4735             try:
-> 4736                 return libindex.get_value_box(s, key)
   4737             except IndexError:

pandas/_libs/index.pyx in pandas._libs.index.get_value_box()

pandas/_libs/index.pyx in pandas._libs.index.get_value_at()

pandas/_libs/util.pxd in pandas._libs.util.get_value_at()

pandas/_libs/util.pxd in pandas._libs.util.validate_indexer()

TypeError: 'NoneType' object cannot be interpreted as an integer

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
~/.local/share/virtualenvs/cartoframes-voNRwPo2/src/carto/carto/auth.py in send(self, relative_path, http_method, **requests_args)
    146 
--> 147             response = super(APIKeyAuthClient, self).send(relative_path, http_method, **requests_args)
    148         except Exception as e:

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/pyrestcli/auth.py in send(self, relative_path, http_method, **requests_args)
     35 
---> 36         return self.session.request(http_method, url, **requests_args)
     37 

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    532         send_kwargs.update(settings)
--> 533         resp = self.send(prep, **send_kwargs)
    534 

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/requests/sessions.py in send(self, request, **kwargs)
    645         # Send the request
--> 646         r = adapter.send(request, **kwargs)
    647 

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    468 
--> 469                     for i in request.body:
    470                         low_conn.send(hex(len(i))[2:].encode('utf-8'))

~/.local/share/virtualenvs/cartoframes-voNRwPo2/src/carto/carto/sql.py in _compress_chunks(self, chunk_generator, compression_level)
    334                                       zlib_mode)
--> 335         for chunk in chunk_generator:
    336             compressed_chunk = compressor.compress(chunk)

~/dev/cartoframes/cartoframes/data/registry/dataframe_dataset.py in _rows(df, cols, with_lnglat, geom_col, enc_type)
    140         for col in cols:
--> 141             val = row[col]
    142             if _is_null(val):

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/pandas/core/series.py in __getitem__(self, key)
   1067         try:
-> 1068             result = self.index.get_value(self, key)
   1069 

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/pandas/core/indexes/base.py in get_value(self, series, key)
   4743                 else:
-> 4744                     raise e1
   4745             except Exception:  # pragma: no cover

~/.local/share/virtualenvs/cartoframes-voNRwPo2/lib/python3.7/site-packages/pandas/core/indexes/base.py in get_value(self, series, key)
   4729         try:
-> 4730             return self._engine.get_value(s, k, tz=getattr(series.dtype, "tz", None))
   4731         except KeyError as e1:

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_value()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_value()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: None

During handling of the above exception, another exception occurred:

CartoException                            Traceback (most recent call last)
~/.local/share/virtualenvs/cartoframes-voNRwPo2/src/carto/carto/sql.py in copyfrom(self, query, iterable_data, compress, compression_level)
    378                                         headers=headers,
--> 379                                         stream=True)
    380             response_json = self.client.get_response_data(response)

~/.local/share/virtualenvs/cartoframes-voNRwPo2/src/carto/carto/auth.py in send(self, relative_path, http_method, **requests_args)
    148         except Exception as e:
--> 149             raise CartoException(e)
    150 

CartoException: None

During handling of the above exception, another exception occurred:

CartoException                            Traceback (most recent call last)
<ipython-input-7-4a4be79e84ca> in <module>
      4 )
      5 
----> 6 dataset.upload(table_name='citybikes', credentials=credentials)

~/dev/cartoframes/cartoframes/data/dataset.py in upload(self, with_lnglat, if_exists, table_name, schema, credentials)
    295             self._strategy.schema = schema
    296 
--> 297         self._strategy.upload(if_exists, with_lnglat)
    298         self._is_saved_in_carto = True
    299 

~/dev/cartoframes/cartoframes/data/registry/dataframe_dataset.py in upload(self, if_exists, with_lnglat)
     61             raise self._already_exists_error()
     62 
---> 63         self._copyfrom(normalized_column_names, with_lnglat)
     64 
     65     def delete(self):

~/dev/cartoframes/cartoframes/data/registry/dataframe_dataset.py in _copyfrom(self, normalized_column_names, with_lnglat)
     92             enc_type)
     93 
---> 94         self._context.upload(query, data)
     95 
     96     def _create_table(self, normalized_column_names, with_lnglat=None):

~/dev/cartoframes/cartoframes/context/api_context.py in upload(self, query, data)
     41 
     42     def upload(self, query, data):
---> 43         return self.copy_client.copyfrom(query.strip(), data)
     44 
     45     def execute_query(self, query, parse_json=True, do_post=True, format=None, **request_args):

~/.local/share/virtualenvs/cartoframes-voNRwPo2/src/carto/carto/sql.py in copyfrom(self, query, iterable_data, compress, compression_level)
    382             raise e
    383         except Exception as e:
--> 384             raise CartoException(e)
    385 
    386         return response_json

CartoException: None

I've found the same problem trying to upload other dataframes created with csv files.

[simon-contreras-deel]

Ei Goizueta! I have just run the notebook and everything goes fine (develop branch). Can you check it?

[jgoizueta]

The notebook example is working, but dataframes without geometry keep failing.
(In the notebook the dataframe gets a geometry column after Map(Layer(dataset)))

It works for me if I change this:

diff --git a/cartoframes/data/dataset/registry/dataframe_dataset.py b/cartoframes/data/dataset/registry/dataframe_dataset.py
index 88c765a2..64d5ad27 100644
--- a/cartoframes/data/dataset/registry/dataframe_dataset.py
+++ b/cartoframes/data/dataset/registry/dataframe_dataset.py
@@ -138,6 +138,8 @@ def _rows(df, cols, with_lnglat, geom_col, enc_type):
         lng_val = None
         lat_val = None
         for col in cols:
+            if col is None:
+                continue
             val = row[col]
             if _is_null(val):
                 val = ''

[jgoizueta]

Regarding the fix, it was failing because in

cartoframes/cartoframes/data/dataset/registry/dataframe_dataset.py

Lines 77 to 78 in f086eeb

	columns_origin = [geom_col]
	for norm, orig in normalized_column_names:

a None is inserted in the list of columns (because no geom column is found)

[jgoizueta]

... so perhaps this would be a more sensible fix:

diff --git a/cartoframes/data/dataset/registry/dataframe_dataset.py b/cartoframes/data/dataset/registry/dataframe_dataset.py
index 88c765a2..a6298630 100644
--- a/cartoframes/data/dataset/registry/dataframe_dataset.py
+++ b/cartoframes/data/dataset/registry/dataframe_dataset.py
@@ -74,7 +74,9 @@ class DataFrameDataset(BaseDataset):
         enc_type = _detect_encoding_type(self._df, geom_col)

         columns_normalized = []
-        columns_origin = [geom_col]
+        columns_origin = []
+        if geom_col:
+            columns_origin.append(geom_col)
         for norm, orig in normalized_column_names:
             columns_normalized.append(norm)
             columns_origin.append(orig)

[simon-contreras-deel]

Good catch! Thanks, I am going to fix it!

[jgoizueta]

🤔 for some reason my latest patch doesn't quite fix it:

carto.exceptions.CartoException: ['missing data for column "the_geom"']