diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..cb2fc91c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,5 @@ +{ + "name": "Python 3", + "image": "mcr.microsoft.com/devcontainers/python", + "postCreateCommand": "python -m pip install nox pre-commit && pre-commit install" +} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 337cf138..c02c9bb0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,9 +14,7 @@ jobs: - uses: actions/setup-python@v2 - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine build + run: pip install setuptools wheel twine build - name: Build and publish env: diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 80202134..ff0ba7a2 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -39,7 +39,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] include: - os: macos-latest # macos test python-version: "3.10" diff --git a/pyproject.toml b/pyproject.toml index c665f6b4..255666ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] requires-python = ">=3.8" dependencies = [ @@ -37,6 +38,8 @@ dependencies = [ "planet>=2", "pyarrow", "localtileserver>=0.7.0", # first pure rio version + "pygaul>=0.3.1", # use the class implementation + "pygadm>=0.5.0", # use the class implementation # miscellaneous "python-box", "tqdm", @@ -83,7 +86,7 @@ test = [ "pytest-regressions", ] doc = [ - "sphinx", + "sphinx<7", "jupyter-sphinx", "pydata-sphinx-theme", "sphinx-notfound-page", @@ -174,4 +177,4 @@ ignore-path-errors = ["docs/source/index.rst;D000"] using = "PEP631:test;dev;doc" [tool.codespell] -skip = 'CHANGELOG.md,sepal_ui/message/**/*.json' +skip = 'CHANGELOG.md,sepal_ui/message/**/*.json,sepal_ui/data/gaul_iso.json' diff --git a/sepal_ui/aoi/aoi_model.py b/sepal_ui/aoi/aoi_model.py index ada5ae18..2a230797 100644 --- a/sepal_ui/aoi/aoi_model.py +++ b/sepal_ui/aoi/aoi_model.py @@ -7,6 +7,8 @@ import ee import geopandas as gpd import pandas as pd +import pygadm +import pygaul import traitlets as t from ipyleaflet import GeoJSON from typing_extensions import Self @@ -27,28 +29,8 @@ class AoiModel(Model): # ### dataset const ### # ########################################################################### - FILE: List[Path] = [ - Path(__file__).parents[1] / "data" / "gadm_database.parquet", - Path(__file__).parents[1] / "data" / "gaul_database.parquet", - ] - "Paths to the GADM(0) and GAUL(1) database" - - CODE: List[str] = ["GID_{}", "ADM{}_CODE"] - "GADM(0) and GAUL(1) administrative codes key format" - - NAME: List[str] = ["NAME_{}", "ADM{}_NAME"] - "GADM(0) and GAUL(1) naming key format" - - ISO: List[str] = ["GID_0", "ISO 3166-1 alpha-3"] - "GADM(0) and GAUL(1) iso codes key" - - GADM_BASE_URL: str = ( - "https://geodata.ucdavis.edu/gadm/gadm4.1/json/gadm41_{}_{}.json" - ) - "The base url to download gadm maps" - - GAUL_ASSET: str = "FAO/GAUL/2015/level{}" - "The GAUL asset name" + MAPPING: Path = Path(__file__).parents[1] / "data" / "gaul_iso.json" + "GAUL -> ISO-3 mapping of country code" ASSET_SUFFIX: str = "aoi_" "The suffix to identify the asset in GEE" @@ -391,7 +373,7 @@ def _from_geo_json(self, geo_json: dict) -> Self: return self def _from_admin(self, admin: str) -> Self: - """Set the object according to given an administrative number in the GADM norm. + """Set the object according to the given an administrative code in the GADM/GAUL codes. Args: admin: the admin code corresponding to FAO GAUl (if gee) or GADM @@ -399,48 +381,23 @@ def _from_admin(self, admin: str) -> Self: if not admin: raise Exception(ms.aoi_sel.exception.no_admlyr) - # get the admin level corresponding to the given admin code - df = pd.read_parquet(self.FILE[self.gee]).astype(str) - - # extract the first element that include this administrative code and set the level accordingly - is_in = df.filter([self.CODE[self.gee].format(i) for i in range(3)]).isin( - [admin] - ) - - if not is_in.any().any(): - raise Exception(ms.aoi_sel.exception.invalid_code) - - index = 3 if self.gee else -1 - level = is_in[~((~is_in).all(axis=1))].idxmax(1).iloc[0][index] - + # get the data from either the pygaul or the pygadm libs + # pygaul needs extra work as ISO codes are not included in the GEE dataset if self.gee: - - # get the feature_collection - self.feature_collection = ee.FeatureCollection( - self.GAUL_ASSET.format(level) - ).filter(ee.Filter.eq(f"ADM{level}_CODE", int(admin))) - - # transform it into gdf + self.feature_collection = pygaul.AdmItems(admin=admin) features = self.feature_collection.getInfo()["features"] self.gdf = gpd.GeoDataFrame.from_features(features).set_crs(epsg=4326) + gaul_country = str(self.gdf.ADM0_CODE.unique()[0]) + iso = json.loads(self.MAPPING.read_text())[gaul_country] + self.gdf["ISO"] = iso else: - # save the country iso_code - iso_3 = admin[:3] - - # read the data from server - level_gdf = gpd.read_file(self.GADM_BASE_URL.format(iso_3, level)) - level_gdf.rename(columns={"COUNTRY": "NAME_0"}, inplace=True) - self.gdf = level_gdf[level_gdf[self.CODE[self.gee].format(level)] == admin] - - # set the name using the layer - r = df[df[self.CODE[self.gee].format(level)] == admin].iloc[0] - names = [ - su.normalize_str(r[self.NAME[self.gee].format(i)]) - if i - else r[self.ISO[self.gee]] - for i in range(int(level) + 1) - ] + self.gdf = pygadm.AdmItems(admin=admin) + + # generate the name from the columns + r = self.gdf.iloc[0] + names = [su.normalize_str(r[c]) for c in self.gdf.columns if "NAME" in c] + names[0] = r.ISO if self.gee else r.GID_0[:3] self.name = "_".join(names) return self diff --git a/sepal_ui/aoi/aoi_view.py b/sepal_ui/aoi/aoi_view.py index 57c749e0..5d9392c2 100644 --- a/sepal_ui/aoi/aoi_view.py +++ b/sepal_ui/aoi/aoi_view.py @@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Union import ipyvuetify as v -import pandas as pd +import pygadm +import pygaul import traitlets as t from deprecated.sphinx import versionadded from typing_extensions import Self @@ -111,7 +112,7 @@ def __init__( ) -> None: """An admin level selector. - It is binded to ee (GAUL 2015) or not (GADM 2021). allows to select administrative codes taking into account the administrative parent code and displaying humanly readable administrative names. + It is binded to ee (GAUL 2015) or not (GADM). Allows to select administrative codes taking into account the administrative parent code and displaying humanly readable administrative names. Args: level: The administrative level of the field @@ -144,30 +145,19 @@ def get_items(self, filter_: str = "") -> Self: r"""Update the item list based on the given filter. Args: - filter\_ (str): The code of the parent v_model to filter the current results + filter\_: The code of the parent v_model to filter the current results """ - # extract the level list - df = ( - pd.read_parquet(AoiModel.FILE[self.gee]) - .astype(str) - .drop_duplicates(subset=AoiModel.CODE[self.gee].format(self.level)) - .sort_values(AoiModel.NAME[self.gee].format(self.level)) - ) - - # filter it - if filter_: - df = df[df[AoiModel.CODE[self.gee].format(self.level - 1)] == filter_] + AdmNames = pygaul.AdmNames if self.gee else pygadm.AdmNames + df = AdmNames(admin=filter_, content_level=self.level) + df = df.sort_values(by=[df.columns[0]]) # formatted as a item list for a select component - self.items = [ - { - "text": su.normalize_str( - r[AoiModel.NAME[self.gee].format(self.level)], folder=False - ), - "value": r[AoiModel.CODE[self.gee].format(self.level)], - } - for _, r in df.iterrows() - ] + # first column will be the name, second the code + items = [] + for _, r in df.iterrows(): + text = su.normalize_str(r.iloc[0], folder=False) + items.append({"text": text, "value": str(r.iloc[1])}) + self.items = items return self @@ -415,12 +405,12 @@ def _activate(self, change: dict) -> None: else: self.aoi_dc.hide() + # init the name to the current value + now = dt.now().strftime("%Y-%m-%d_%H-%M-%S") + self.w_draw.v_model = None if change["new"] is None else f"Manual_aoi_{now}" + # activate the correct widget w = next((w for k, w in self.components.items() if k == change["new"]), None) w is None or w.show() - # init the name to the current value - now = dt.now().strftime("%Y-%m-%d_%H-%M-%S") - self.w_draw.v_model = None if change["new"] is None else f"Manual_aoi_{now}" - return diff --git a/sepal_ui/data/gadm_database.parquet b/sepal_ui/data/gadm_database.parquet deleted file mode 100644 index 34bdaf0c..00000000 Binary files a/sepal_ui/data/gadm_database.parquet and /dev/null differ diff --git a/sepal_ui/data/gaul_database.parquet b/sepal_ui/data/gaul_database.parquet deleted file mode 100644 index c982650f..00000000 Binary files a/sepal_ui/data/gaul_database.parquet and /dev/null differ diff --git a/sepal_ui/data/gaul_iso.json b/sepal_ui/data/gaul_iso.json new file mode 100644 index 00000000..7a48b199 --- /dev/null +++ b/sepal_ui/data/gaul_iso.json @@ -0,0 +1,278 @@ +{ + "257": "TZA", + "3": "ALB", + "4": "DZA", + "6": "-", + "7": "AND", + "269": "YEM", + "270": "ZMB", + "271": "ZWE", + "29": "BEN", + "34": "BIH", + "35": "BWA", + "37": "BRA", + "41": "BGR", + "42": "BFA", + "43": "BDI", + "45": "CMR", + "47": "CPV", + "49": "CAF", + "50": "TCD", + "58": "COM", + "59": "COG", + "40765": "EGY", + "62": "HRV", + "64": "CYP", + "66": "CIV", + "68": "COD", + "70": "DJI", + "74": "-", + "76": "GNQ", + "77": "ERI", + "79": "ETH", + "85": "FRA", + "2647": "MNE", + "2648": "SRB", + "89": "GAB", + "90": "GMB", + "94": "GHA", + "97": "GRC", + "105": "GNB", + "106": "GIN", + "118": "IRQ", + "121": "ISR", + "122": "ITA", + "130": "JOR", + "133": "KEN", + "141": "LBN", + "144": "LBR", + "145": "LBY", + "152": "MWI", + "155": "MLI", + "156": "MLT", + "159": "MRT", + "166": "MCO", + "169": "MAR", + "170": "MOZ", + "172": "NAM", + "181": "NER", + "182": "NGA", + "199": "PRT", + "205": "RWA", + "213": "SMR", + "214": "STP", + "215": "SAU", + "217": "SEN", + "221": "SLE", + "226": "SOM", + "229": "ESP", + "238": "SYR", + "241": "MKD", + "243": "TGO", + "248": "TUN", + "249": "TUR", + "253": "UGA", + "74578": "-", + "151": "-", + "110": "VAT", + "40760": "-", + "40762": "-", + "61013": "-", + "102": "-", + "268": "ESH", + "91": "PSE", + "267": "-", + "95": "GIB", + "207": "SHN", + "147296": "-", + "56": "CCK", + "54": "CXR", + "16": "-", + "88": "ATF", + "25": "-", + "80": "-", + "96": "-", + "131": "-", + "161": "MYT", + "206": "REU", + "247": "-", + "167": "MNG", + "132": "KAZ", + "202": "KOR", + "222": "SGP", + "154": "MDV", + "175": "NPL", + "160": "MUS", + "189": "PLW", + "38": "IOT", + "193": "-", + "218": "-", + "216": "-", + "52": "-", + "2": "-", + "230": "-", + "15": "-", + "149": "MAC", + "33364": "HKG", + "147295": "CHN", + "115": "IND", + "126": "JPN", + "8": "AGO", + "31": "BTN", + "142": "LSO", + "171": "MMR", + "44": "KHM", + "153": "MYS", + "188": "PAK", + "240": "THA", + "264": "VNM", + "17": "AUS", + "116": "IDN", + "235": "SWZ", + "23": "BGD", + "138": "KGZ", + "150": "MDG", + "220": "SYC", + "239": "TJK", + "261": "UZB", + "1": "AFG", + "196": "PHL", + "242": "TLS", + "227": "ZAF", + "40": "BRN", + "40781": "-", + "67": "PRK", + "139": "LAO", + "120": "IMN", + "82": "FRO", + "234": "SJM", + "210": "SPM", + "98": "GRL", + "255": "ARE", + "113": "HUN", + "84": "FIN", + "114": "ISL", + "177": "NLD", + "27": "BEL", + "148": "LUX", + "146": "LIE", + "250": "TKM", + "137": "KWT", + "21": "BHR", + "201": "QAT", + "187": "OMN", + "46": "CAN", + "140": "LVA", + "186": "NOR", + "198": "POL", + "236": "SWE", + "13": "ARM", + "18": "AUT", + "26": "BLR", + "69": "-", + "78": "EST", + "92": "GEO", + "93": "DEU", + "119": "IRL", + "203": "ROU", + "254": "UKR", + "223": "SVK", + "224": "SVN", + "147": "LTU", + "231": "LKA", + "19": "AZE", + "237": "CHE", + "65": "CZE", + "204": "RUS", + "165": "MDA", + "259": "USA", + "117": "IRN", + "256": "GBR", + "104": "GGY", + "128": "JEY", + "136": "-", + "185": "MNP", + "101": "GUM", + "265": "-", + "225": "SLB", + "157": "MHL", + "163": "FSM", + "192": "PNG", + "83": "FJI", + "162": "MEX", + "103": "GTM", + "75": "SLV", + "183": "NIU", + "173": "NRU", + "212": "WSM", + "245": "TON", + "252": "TUV", + "244": "TKL", + "262": "VUT", + "135": "KIR", + "179": "NZL", + "22": "-", + "60": "COK", + "134": "-", + "127": "-", + "164": "-", + "178": "NCL", + "190": "-", + "5": "ASM", + "112": "-", + "129": "-", + "184": "NFK", + "87": "PYF", + "55": "-", + "266": "WLF", + "158": "MTQ", + "100": "GLP", + "176": "-", + "32": "-", + "14": "ABW", + "258": "VIR", + "73": "ECU", + "86": "GUF", + "99": "GRD", + "123": "JAM", + "208": "KNA", + "246": "TTO", + "209": "LCA", + "211": "VCT", + "71": "DMA", + "24": "BRB", + "20": "BHS", + "11": "ATG", + "28": "BLZ", + "63": "CUB", + "108": "HTI", + "107": "GUY", + "191": "PAN", + "57": "COL", + "111": "HND", + "233": "SUR", + "180": "NIC", + "263": "VEN", + "61": "CRI", + "72": "DOM", + "174": "-", + "200": "PRI", + "168": "MSR", + "251": "TCA", + "39": "VGB", + "48": "CYM", + "9": "AIA", + "30": "BMU", + "195": "PER", + "51": "CHL", + "33": "BOL", + "194": "PRY", + "12": "ARG", + "260": "URY", + "81": "-", + "228": "SGS", + "109": "HMD", + "36": "BVT", + "10": "ATA", + "197": "PCN" +} diff --git a/sepal_ui/frontend/css/custom.css b/sepal_ui/frontend/css/custom.css index af8a2596..aec612ab 100644 --- a/sepal_ui/frontend/css/custom.css +++ b/sepal_ui/frontend/css/custom.css @@ -104,3 +104,9 @@ nav.v-navigation-drawer { width: calc(100vw - 80px); height: calc(100vh - 20px); } + +/* hide the disclaimer message placed on top of the ui notebook + * the message will be displayed until the css is loaded */ +#loading-app { + display: none !important; +} diff --git a/sepal_ui/message/en/utils.json b/sepal_ui/message/en/utils.json index f5f691ec..54b0a9bc 100644 --- a/sepal_ui/message/en/utils.json +++ b/sepal_ui/message/en/utils.json @@ -2,6 +2,9 @@ "utils": { "check_input": { "error": "The value has not been initialized" + }, + "ee": { + "no_asset_root": "No asset folder is available with this credentials. Please check your account and try again." } } } diff --git a/sepal_ui/scripts/decorator.py b/sepal_ui/scripts/decorator.py index 81087c38..66bebbc0 100644 --- a/sepal_ui/scripts/decorator.py +++ b/sepal_ui/scripts/decorator.py @@ -54,6 +54,7 @@ def init_ee() -> None: # if the user is in local development the authentication should # already be available ee.Initialize(http_transport=httplib2.Http()) + assert len(ee.data.getAssetRoots()) > 0, ms.utils.ee.no_asset_root return diff --git a/sepal_ui/scripts/utils.py b/sepal_ui/scripts/utils.py index 7de31d7f..34925cb8 100644 --- a/sepal_ui/scripts/utils.py +++ b/sepal_ui/scripts/utils.py @@ -113,12 +113,8 @@ def get_file_size(filename: Union[str, Path]) -> str: """ file_size = Path(filename).stat().st_size - if file_size == 0: - return "0B" - size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - - i = int(math.floor(math.log(file_size, 1024))) + i = int(math.floor(math.log(file_size, 1024))) if file_size > 0 else 0 s = file_size / (1024**i) return "{:.1f} {}".format(s, size_name[i]) @@ -146,6 +142,7 @@ def init_ee() -> None: # if the user is in local development the authentication should # already be available ee.Initialize(http_transport=httplib2.Http()) + assert len(ee.data.getAssetRoots()) > 0, ms.utils.ee.no_asset_root return diff --git a/sepal_ui/sepalwidgets/inputs.py b/sepal_ui/sepalwidgets/inputs.py index 51638b69..2c0de078 100644 --- a/sepal_ui/sepalwidgets/inputs.py +++ b/sepal_ui/sepalwidgets/inputs.py @@ -52,7 +52,6 @@ reason="Empty v_model will be treated as empty string: :code:`v_model=''`.", ) class DatePicker(v.Layout, SepalWidget): - menu: Optional[v.Menu] = None "the menu widget to display the datepicker" @@ -178,13 +177,15 @@ def is_valid_date(date: str) -> bool: class FileInput(v.Flex, SepalWidget): - extensions: List[str] = [] "list: the extensions list" folder: Path = Path.home() "the current folder" + initial_folder: Path = Path.home() + "the starting point of the file input" + file: t.Unicode = t.Unicode("").tag(sync=True) "the current file" @@ -223,6 +224,7 @@ def __init__( v_model: str = "", clearable: bool = False, root: Union[str, Path] = "", + cache=False, **kwargs, ) -> None: """Custom input field to select a file in the sepal folders. @@ -237,8 +239,10 @@ def __init__( kwargs: any parameter from a v.Flex abject. If set, 'children' will be overwritten. """ self.extensions = extensions + self.initial_folder = folder self.folder = Path(folder) self.root = str(root) if isinstance(root, Path) else root + self.cache_dirs = {} self.selected_file = v.TextField( readonly=True, @@ -330,9 +334,8 @@ def reset(self, *args) -> Self: # do nothing if nothing is set to avoids extremely long waiting # time when multiple fileInput are reset at the same time as in the aoiView if self.v_model is not None: - # move to root - self._on_file_select({"new": Path.home()}) + self._on_file_select({"new": self.initial_folder}) # remove v_model self.v_model = "" @@ -369,7 +372,10 @@ def _on_file_select(self, change: dict) -> Self: if new_value.is_dir(): self.folder = new_value - self._change_folder() + + # don't change folder if the folder is the parent of the root + if not self.folder == Path(self.root).parent: + self._change_folder() elif new_value.is_file(): self.file = str(new_value) @@ -399,18 +405,21 @@ def _get_items(self) -> List[v.ListItem]: """ folder = self.folder - list_dir = [el for el in folder.glob("*/") if not el.name.startswith(".")] + list_dir = [el for el in folder.glob("*") if not el.name.startswith(".")] if self.extensions: list_dir = [ el for el in list_dir if el.is_dir() or el.suffix in self.extensions ] + if folder in self.cache_dirs: + if self.cache_dirs[folder]["files"] == list_dir: + return self.cache_dirs[folder]["items"] + folder_list = [] file_list = [] for el in list_dir: - if el.is_dir(): icon = self.ICON_STYLE[""]["icon"] color = self.ICON_STYLE[""]["color"][v.theme.dark] @@ -439,10 +448,7 @@ def _get_items(self) -> List[v.ListItem]: folder_list = humansorted(folder_list, key=lambda x: x.value) file_list = humansorted(file_list, key=lambda x: x.value) - folder_list.extend(file_list) - # add the parent item if root is set and is not reached yet - # if root is not set then we always display it parent_item = v.ListItem( value=str(folder.parent), children=[ @@ -459,16 +465,17 @@ def _get_items(self) -> List[v.ListItem]: ), ], ) - root_folder = Path(self.root) - if self.root == "": - folder_list.insert(0, parent_item) - elif root_folder in folder.parents: - folder_list.insert(0, parent_item) + + folder_list.extend(file_list) + folder_list.insert(0, parent_item) + + self.cache_dirs.setdefault(folder, {}) + self.cache_dirs[folder]["files"] = list_dir + self.cache_dirs[folder]["items"] = folder_list return folder_list def _on_reload(self, *args) -> None: - # force the update of the current folder self._change_folder() @@ -484,7 +491,6 @@ def close_menu(self, change: dict) -> None: class LoadTableField(v.Col, SepalWidget): - fileInput: Optional[FileInput] = None "The file input to select the .csv or .txt file" @@ -638,7 +644,6 @@ def _set_v_model(self, key: str, value: Any) -> None: class AssetSelect(v.Combobox, SepalWidget): - TYPES: dict = { "IMAGE": ms.widgets.asset_select.types[0], "TABLE": ms.widgets.asset_select.types[1], @@ -723,7 +728,6 @@ def _validate(self, change: dict) -> None: self.v_model = self.v_model.strip() if change["new"]: - # check that the asset can be accessed try: self.asset_info = ee.data.getAsset(self.v_model) @@ -735,7 +739,6 @@ def _validate(self, change: dict) -> None: ) except Exception: - self.error_messages = ms.widgets.asset_select.no_access self.valid = self.error_messages is None @@ -745,13 +748,11 @@ def _validate(self, change: dict) -> None: @sd.switch("loading", "disabled") def _get_items(self, *args) -> Self: - # init the item list items = [] # add the default values if needed if self.default_asset: - if isinstance(self.default_asset, str): self.default_asset = [self.default_asset] @@ -828,7 +829,6 @@ def _toggle_pwd(self, *args) -> None: class NumberField(v.TextField, SepalWidget): - max_: t.Int = t.Int(10).tag(sync=True) "Maximum selectable number." @@ -879,7 +879,6 @@ def decrement(self, *args) -> None: class VectorField(v.Col, SepalWidget): - original_gdf: Optional[gpd.GeoDataFrame] = None "The originally selected dataframe" diff --git a/sepal_ui/templates/map_app/ui.ipynb b/sepal_ui/templates/map_app/ui.ipynb index 112ccfa5..0a55e266 100644 --- a/sepal_ui/templates/map_app/ui.ipynb +++ b/sepal_ui/templates/map_app/ui.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "