Delete docs/developer_center directory

.github/workflows/cartoframes-ci.yml

@@ -0,0 +1,36 @@
+name: Run CARTOFrames tests
+
+on:
+  push:
+  pull_request:
+    branches:
+      - master
+      - develop
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    name: Run tests on Python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+          python -m pip install --upgrade pip
+          pip install tox
+          pip install tox-gh-actions
+
+    - name: Test with tox
+      run: |
+        tox

CHANGELOG.md

@@ -5,19 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Pending]
+
+### Changed
+- Allow to set a value for null geometries in the `read_carto` and `Geocoding.geocode` methods (#1667)
+
 ## [1.0.4] - 2020-07-06
 
-## Added
+### Added
 - Add list_tables function (#1649)
 - Add catalog public filter to providers, countries and categories (#1658)
 - Add set_default_do_credentials function for DO authentication (#1655)
 
-## Changed
+### Changed
 - Open publication link in another window (#1647)
 - Show a warning when uploading a GeoDataFrame without geometry (#1650)
 - Improve GeoDataFrame CRS check, docs and examples (#1656)
 
-## Fixed
+### Fixed
 - Fix empty geometries issue (#1652)
 - Fix Layout publication API key issue (#1654)
 - Fix ColumnInfo comparison when replacing a table (#1660)

cartoframes/auth/credentials.py

@@ -109,15 +109,25 @@ def session(self, session):
         """Set session"""
         self._session = session
 
+    @property
+    def me_data(self):
+        me_data = {}
+
+        try:
+            me_data = self.get_api_key_auth_client().send(ME_SERVICE, 'get').json()
+        except Exception:
+            pass
+
+        return me_data
+
     @property
     def user_id(self):
         """Credentials user ID"""
         if not self._user_id:
             log.debug('Getting `user_id` for {}'.format(self._username))
-            api_key_auth_client = self.get_api_key_auth_client()
 
             try:
-                user_me = api_key_auth_client.send(ME_SERVICE, 'get').json()
+                user_me = self.me_data()
                 user_data = user_me.get('user_data')
                 if user_data:
                     self._user_id = user_data.get('id')

cartoframes/auth/defaults.py

@@ -13,7 +13,7 @@ def set_default_credentials(
 
     Args:
         credentials (:py:class:`Credentials <cartoframes.credentials.Credentials>`, optional):
-          A :py:class:`Credentials <cartoframes.credentials.Credentials>`
+          A :py:class:`Credentials <cartoframes.auth.Credentials>`
           instance can be used in place of a `username | base_url`/`api_key` combination.
         base_url (str, optional): Base URL of CARTO user account. Cloud-based accounts
           should use the form ``https://{username}.carto.com`` (e.g.,

cartoframes/data/observatory/catalog/catalog.py

@@ -5,8 +5,10 @@
 from .dataset import Dataset
 from .geography import Geography
 from .subscriptions import Subscriptions
-from .repository.constants import COUNTRY_FILTER, CATEGORY_FILTER, GEOGRAPHY_FILTER, PROVIDER_FILTER, PUBLIC_FILTER
+from .repository.constants import (COUNTRY_FILTER, CATEGORY_FILTER, GEOGRAPHY_FILTER, GLOBAL_COUNTRY_FILTER,
+                                   PROVIDER_FILTER, PUBLIC_FILTER)
 
+from ....utils.logger import log
 from ....utils.utils import get_credentials
 
 
@@ -142,6 +144,7 @@ def categories(self):
             CatalogError: if there's a problem when connecting to the catalog or no datasets are found.
 
         """
+        self._global_message()
         return Category.get_all(self.filters)
 
     @property
@@ -155,6 +158,7 @@ def providers(self):
             CatalogError: if there's a problem when connecting to the catalog or no datasets are found.
 
         """
+        self._global_message()
         return Provider.get_all(self.filters)
 
     @property
@@ -168,6 +172,7 @@ def datasets(self):
             CatalogError: if there's a problem when connecting to the catalog or no datasets are found.
 
         """
+        self._global_message()
         return Dataset.get_all(self.filters)
 
     @property
@@ -181,6 +186,7 @@ def geographies(self):
             CatalogError: if there's a problem when connecting to the catalog or no datasets are found.
 
         """
+        self._global_message()
         return Geography.get_all(self.filters)
 
     def country(self, country_id):
@@ -295,3 +301,8 @@ def datasets_filter(self, filter_dataset):
 
         """
         return Dataset.get_datasets_spatial_filtered(filter_dataset)
+
+    def _global_message(self):
+        if self.filters and self.filters.get(COUNTRY_FILTER) != GLOBAL_COUNTRY_FILTER:
+            log.info('You can find more entities with the Global country filter. To apply that filter run:'
+                     "\n\tCatalog().country('glo')")

cartoframes/data/observatory/catalog/repository/constants.py

@@ -6,3 +6,4 @@
 PUBLIC_FILTER = 'public'
 VARIABLE_FILTER = 'variable'
 VARIABLE_GROUP_FILTER = 'variable_group'
+GLOBAL_COUNTRY_FILTER = 'glo'

cartoframes/data/services/geocoding.py

@@ -57,7 +57,8 @@ def geocode(self, source, street,
                 city=None, state=None, country=None,
                 status=geocoding_constants.DEFAULT_STATUS,
                 table_name=None, if_exists='fail',
-                dry_run=False, cached=None):
+                dry_run=False, cached=None,
+                null_geom_value=None):
         """Geocode method.
 
         Args:
@@ -92,6 +93,8 @@ def geocode(self, source, street,
                 table. This parameter should be used along with ``table_name``.
             dry_run (bool, optional): no actual geocoding will be performed (useful to
                 check the needed quota)
+            null_geom_value (Object, optional): value for the `the_geom` column when it's null.
+                Defaults to None
 
         Returns:
             A named-tuple ``(data, metadata)`` containing  either a ``data`` geopandas.GeoDataFrame
@@ -184,7 +187,7 @@ def geocode(self, source, street,
         if dry_run:
             return self.result(data=None, metadata=metadata)
 
-        gdf = read_carto(input_table_name, self._credentials)
+        gdf = read_carto(input_table_name, self._credentials, null_geom_value=null_geom_value)
 
         if self._source_manager.is_dataframe() and CARTO_INDEX_KEY in gdf:
             del gdf[CARTO_INDEX_KEY]

cartoframes/data/services/isolines.py

@@ -202,8 +202,7 @@ def _iso_areas(self,
         # Execute and download the query to generate the isolines
         gdf = read_carto(sql, self._credentials)
 
-        # Sorting by `data_range` column and recalculating `cartodb_id`
-        gdf.sort_values(by=[DATA_RANGE_KEY], ascending=ascending, inplace=True)
+        # Recalculating `cartodb_id`
         gdf.reset_index(drop=True, inplace=True)
         if CARTO_INDEX_KEY in gdf.columns:
             gdf[CARTO_INDEX_KEY] = gdf.index + 1

cartoframes/data/services/utils/geocoding_constants.py

@@ -1,6 +1,5 @@
 __all__ = [
     'HASH_COLUMN',
-    'BATCH_SIZE',
     'DEFAULT_STATUS',
     'QUOTA_SERVICE',
     'STATUS_FIELDS',
@@ -12,8 +11,6 @@
 
 HASH_COLUMN = 'carto_geocode_hash'
 
-BATCH_SIZE = 200
-
 DEFAULT_STATUS = {'gc_status_rel': 'relevance'}
 
 QUOTA_SERVICE = 'hires_geocoder'

cartoframes/data/services/utils/geocoding_utils.py

@@ -144,16 +144,14 @@ def geocode_query(table, schema, street, city, state, country, status):
             {street},
             {city},
             {state},
-            {country},
-            {batch_size}
+            {country}
         )
     """.format(
         query=query,
         street=column_name(street),
         city=column_name(city),
         state=column_name(state),
-        country=column_name(country),
-        batch_size=geocoding_constants.BATCH_SIZE
+        country=column_name(country)
     )
 
     status_assignment, status_columns = status_assignment_columns(status)

cartoframes/io/carto.py

@@ -1,11 +1,12 @@
 """Functions to interact with the CARTO platform"""
+import math
 
 from pandas import DataFrame
 from geopandas import GeoDataFrame
 
 from carto.exceptions import CartoException
 
-from .managers.context_manager import ContextManager
+from .managers.context_manager import ContextManager, _compute_copy_data, get_dataframe_columns_info
 from ..utils.geom_utils import check_crs, has_geometry, set_geometry
 from ..utils.logger import log
 from ..utils.utils import is_valid_str, is_sql_query
@@ -15,9 +16,14 @@
 GEOM_COLUMN_NAME = 'the_geom'
 IF_EXISTS_OPTIONS = ['fail', 'replace', 'append']
 
+MAX_UPLOAD_SIZE_BYTES = 2000000000  # 2GB
+SAMPLE_ROWS_NUMBER = 100
+CSV_TO_CARTO_RATIO = 1.4
+
 
 @send_metrics('data_downloaded')
-def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None, index_col=None, decode_geom=True):
+def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None, index_col=None, decode_geom=True,
+               null_geom_value=None):
     """Read a table or a SQL query from the CARTO account.
 
     Args:
@@ -32,6 +38,8 @@ def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None,
             `current_schema()` using the credentials.
         index_col (str, optional): name of the column to be loaded as index. It can be used also to set the index name.
         decode_geom (bool, optional): convert the "the_geom" column into a valid geometry column.
+        null_geom_value (Object, optional): value for the `the_geom` column when it's null.
+            Defaults to None
 
     Returns:
         geopandas.GeoDataFrame
@@ -59,12 +67,16 @@ def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None,
         # Decode geometry column
         set_geometry(gdf, GEOM_COLUMN_NAME, inplace=True)
 
+        if null_geom_value is not None:
+            gdf[GEOM_COLUMN_NAME].fillna(null_geom_value, inplace=True)
+
     return gdf
 
 
 @send_metrics('data_uploaded')
 def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col=None, index=False, index_label=None,
-             cartodbfy=True, log_enabled=True):
+             cartodbfy=True, log_enabled=True, retry_times=3, max_upload_size=MAX_UPLOAD_SIZE_BYTES,
+             skip_quota_warning=False):
     """Upload a DataFrame to CARTO. The geometry's CRS must be WGS 84 (EPSG:4326) so you can use it on CARTO.
 
     Args:
@@ -79,6 +91,11 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col
             uses the name of the index from the dataframe.
         cartodbfy (bool, optional): convert the table to CARTO format. Default True. More info
             `here <https://carto.com/developers/sql-api/guides/creating-tables/#create-tables>`.
+        skip_quota_warning (bool, optional): skip the quota exceeded check and force the upload.
+            (The upload will still fail if the size of the dataset exceeds the remaining DB quota).
+            Default is False.
+        retry_times (int, optional):
+            Number of time to retry the upload in case it fails. Default is 3.
 
     Returns:
         string: the table name normalized.
@@ -102,6 +119,19 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col
 
     context_manager = ContextManager(credentials)
 
+    if not skip_quota_warning:
+        me_data = context_manager.credentials.me_data
+        if me_data is not None and me_data.get('user_data'):
+            n = min(SAMPLE_ROWS_NUMBER, len(dataframe))
+            estimated_byte_size = len(dataframe.sample(n=n).to_csv(header=False)) * len(dataframe) \
+                / n / CSV_TO_CARTO_RATIO
+            remaining_byte_quota = me_data.get('user_data').get('remaining_byte_quota')
+
+            if remaining_byte_quota is not None and estimated_byte_size > remaining_byte_quota:
+                raise CartoException('DB Quota will be exceeded. '
+                                     'The remaining quota is {} bytes and the dataset size is {} bytes.'.format(
+                                        remaining_byte_quota, estimated_byte_size))
+
     gdf = GeoDataFrame(dataframe, copy=True)
 
     if index:
@@ -118,13 +148,23 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col
         gdf.set_geometry(dataframe.geometry.name, inplace=True)
 
     if has_geometry(gdf):
+        if GEOM_COLUMN_NAME in gdf and dataframe.geometry.name != GEOM_COLUMN_NAME:
+            gdf.drop(columns=[GEOM_COLUMN_NAME], inplace=True)
+
         # Prepare geometry column for the upload
         gdf.rename_geometry(GEOM_COLUMN_NAME, inplace=True)
 
     elif isinstance(dataframe, GeoDataFrame):
         log.warning('Geometry column not found in the GeoDataFrame.')
 
-    table_name = context_manager.copy_from(gdf, table_name, if_exists, cartodbfy)
+    chunk_count = math.ceil(estimate_csv_size(gdf) / max_upload_size)
+    chunk_row_size = int(math.ceil(len(gdf) / chunk_count))
+    chunked_gdf = [gdf[i:i + chunk_row_size] for i in range(0, gdf.shape[0], chunk_row_size)]
+
+    for i, chunk in enumerate(chunked_gdf):
+        if i > 0:
+            if_exists = 'append'
+        table_name = context_manager.copy_from(chunk, table_name, if_exists, cartodbfy, retry_times)
 
     if log_enabled:
         log.info('Success! Data uploaded to table "{}" correctly'.format(table_name))
@@ -358,3 +398,10 @@ def update_privacy_table(table_name, privacy, credentials=None, log_enabled=True
 
     if log_enabled:
         log.info('Success! Table "{}" privacy updated correctly'.format(table_name))
+
+
+def estimate_csv_size(gdf):
+    n = min(SAMPLE_ROWS_NUMBER, len(gdf))
+    columns = get_dataframe_columns_info(gdf)
+    return sum([len(x) for x in
+                _compute_copy_data(gdf.sample(n=n), columns)]) * len(gdf) / n