From 2d3f3eb0f6a08801108e57fa61451fc840dd5a6d Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Wed, 15 Nov 2023 13:20:00 +0000 Subject: [PATCH] updated setup tutorial; updated subsetting pytest --- clients/python/tests/test_subsetting.py | 12 +-- .../tutorials/developer/ubuntu_arm_setup.md | 93 ++++--------------- 2 files changed, 23 insertions(+), 82 deletions(-) diff --git a/clients/python/tests/test_subsetting.py b/clients/python/tests/test_subsetting.py index 64903aa22..d47db90a9 100644 --- a/clients/python/tests/test_subsetting.py +++ b/clients/python/tests/test_subsetting.py @@ -25,8 +25,8 @@ def test_rasterize(self, init): gdf = icesat2.atl06p(parms, resources=[resource]) assert init assert len(gdf) == 953 - assert abs(gdf["h_mean"].describe()["mean"] - 1749.8456579488566) < 0.00001 - assert abs(gdf["y_atc"].describe()["mean"] - -5516.94775390625) < 0.00001 + assert abs(gdf["h_mean"].describe()["mean"] - 1749.8443895024502) < 0.01 + assert abs(gdf["y_atc"].describe()["mean"] - -5516.94775390625) < 0.01 def test_180_edge(self, init): resource = "ATL03_20221012073759_03291712_005_01.h5" @@ -46,8 +46,8 @@ def test_180_edge(self, init): gdf = icesat2.atl06p(parms, resources=[resource]) assert init assert len(gdf) == 33998 - assert abs(gdf["h_mean"].describe()["mean"] - 1963.3552883770715) < 0.00001 - assert abs(gdf["y_atc"].describe()["mean"] - 1.4383291006088257) < 0.00001 + assert abs(gdf["h_mean"].describe()["mean"] - 1963.3553175453283) < 0.01 + assert abs(gdf["y_atc"].describe()["mean"] - 1.438330888748169) < 0.01 def test_150_translation(self, init): resource = "ATL03_20221009072040_02831712_005_01.h5" @@ -67,5 +67,5 @@ def test_150_translation(self, init): gdf = icesat2.atl06p(parms, resources=[resource]) assert init assert len(gdf) == 26663 - assert abs(gdf["h_mean"].describe()["mean"] - 95.01489210986797) < 0.00001 - assert abs(gdf["y_atc"].describe()["mean"] - 77.65708923339844) < 0.00001 + assert abs(gdf["h_mean"].describe()["mean"] - 95.01306951414814) < 0.01 + assert abs(gdf["y_atc"].describe()["mean"] - 77.65707397460938) < 0.01 diff --git a/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md b/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md index 43d40dcbf..7c94f577e 100644 --- a/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md +++ b/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md @@ -65,56 +65,12 @@ sudo systemctl enable apport.service - create a key on the server, while logged into your account: `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"` - add the key as an ssh key to your github account -### 6. Update and Install OS Packages +### 6. Install and Configure Git -The following OS dependencies are needed to build the SlideRule server. ```bash -sudo apt install build-essential libreadline-dev liblua5.3-dev -sudo apt install git cmake -sudo apt install curl libcurl4-openssl-dev -sudo apt install zlib1g-dev -sudo apt install libgdal-dev +sudo apt install git ``` -Note: starting with release v2.1.0, SlideRule requires a >=3.6.0 version of GDAL. This can require building GDAL from source. See the SlideRule cluster Dockerfile for details on how that could be accomplished. - -### 7. Install RapidJson - -RapidJson is used by SlideRule for parsing JSON in the C++ server-side code. -```bash -git clone https://github.com/Tencent/rapidjson.git -cd rapidjson -mkdir build -cd build -cmake .. -make -sudo make install -``` - -### 8. Install Apache Arrow - -Apache Arrow is used by SlideRule for its GeoParquet support. -```bash -git clone https://github.com/apache/arrow.git -cd arrow -mkdir build -cd build -cmake .. -DARROW_PARQUET=ON -DARROW_WITH_ZLIB=ON -make -j8 -sudo make install -``` - -### 9. Install Pistache - -SlideRule supports running either a native HTTP server, or the Pistache HTTP server which requires the Pistache library to be installed. -```bash -$ sudo add-apt-repository ppa:pistache+team/unstable -$ sudo apt update -$ sudo apt install libpistache-dev -``` - -### 10. Configure Git - Create the local file `~/.gitconfig` in the user's home directory with the following contents: ```yml [user] @@ -151,18 +107,16 @@ Create the local file `~/.gitconfig` in the user's home directory with the follo helper = !/usr/bin/gh auth git-credential ``` -### 11. Clone Projects +### 7. Clone Projects ```bash mkdir meta cd meta git clone git@github.com:ICESat2-SlideRule/sliderule.git git clone git@github.com:ICESat2-SlideRule/sliderule-python.git -git clone git@github.com:ICESat2-SlideRule/sliderule-docs.git -git clone git@github.com:ICESat2-SlideRule/sliderule-build-and-deploy.git ``` -### 12. Installing and Configuring Docker +### 8. Installing and Configuring Docker ```bash sudo apt-get install ca-certificates curl gnupg lsb-release @@ -175,34 +129,19 @@ sudo usermod -aG docker newgrp docker ``` -### 13. Installing Docker-Compose - -```bash -wget https://github.com/docker/compose/releases/download/v2.11.2/docker-compose-linux-aarch64 -sudo mv docker-compose-linux-aarch64 /usr/local/bin/docker-compose -sudo chmod +x /usr/local/bin/docker-compose -``` +### 9. Install Dependencies got Local Build -### 14. Installing NGINX +The most reliable way to install all of the dependencies needed to build sliderule is to follow the steps outlined in the [Dockerfile](https://github.com/ICESat2-SlideRule/sliderule/blob/main/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv) for building the development environment. Some translation of the steps from the Dockerfile format is needed, but all of the dependencies and the configuration options needed for those dependencies are explicitly called out in that file. -Install NGINX which is used for running the containers locally. +Alternatively, the `sliderule-buildenv` Docker image can be built and used as your development environment. To do so, run the following commands from the root of the sliderule repository. ```bash -sudo apt install nginx -``` -Copy over the nginx configuration and restart nginx -```bash -sudo cp nginx.service /etc/nginx/sites-available/sliderule -sudo ln -s /etc/nginx/sites-available/sliderule /etc/nginx/sites-enabled/sliderule -sudo unlink /etc/nginx/sites-enabled/default +cd targets/slideruleearth-aws +make sliderule-buildenv-docker +make run-buildenv ``` +Then, inside the container, you'll find the sliderule repository at `/host/sliderule`. Change directory there and you will then be able to build the sliderule code however you'd like use the provided makefiles under the root and `targets` directory. -Create the self-signed certs -```bash -sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt -sudo systemctl restart nginx -``` - -### 15. Install and Configure Miniconda +### 10. Install and Configure Miniconda ```bash wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-aarch64.sh @@ -212,7 +151,9 @@ conda config --set changeps1 False conda config --set auto_activate_base false ``` -### 16. Install GitHub Command Line Client +Once you have miniconda setup, you can navigate to the `sliderule-python` repository and run `conda env create -f environment.yml` to create a `sliderule_env` environment with everything you will need for the SlideRule Python client. + +### 11. Install GitHub Command Line Client ```bash type -p curl >/dev/null || sudo apt install curl -y @@ -223,7 +164,7 @@ curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo && sudo apt install gh -y ``` -### 17. Install and Configure AWS Command Line Client +### 12. Install and Configure AWS Command Line Client ```bash sudo apt install awscli @@ -251,7 +192,7 @@ To login to the AWS Elastic Container Registry, run: aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account_number.dkr.ecr.$region.amazonaws.com ``` -### 18. Install Terraform & Packer +### 13. Install Terraform & Packer ```bash sudo apt install unzip