From 54c857fdcc86bb18409bb45916e3fa30b29db523 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Mon, 11 Dec 2023 20:13:05 +0000 Subject: [PATCH 01/15] tutorial on setting up amazon linux 2023 for development --- .../developer/amazon_linux_arm_setup.md | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md diff --git a/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md new file mode 100644 index 000000000..bc173cc2b --- /dev/null +++ b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md @@ -0,0 +1,261 @@ +# Setting Up Development Environment + +2023-12-11 + +## Overview + +These steps setup a development environment for SlideRule. The target platform is a Graviton3 processor running Amazon Linux 2023 in AWS. + +## Steps + +### 1. Logging in the first time + +```bash +ssh -i .ssh/.pem ec2-user@ +sudo dnf upgrade --refresh +``` + +### 2. Creating Users + +```bash +sudo useradd -m -s /usr/bin/bash +sudo passwd +sudo usermod -aG wheel +su - +``` + +### 3. Setup Bash + +Replace the appropriate section in the .bashrc file with the contents below +```bash +parse_git_branch() { + git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/' +} +PS1="\n\[\033[01;32m\]\u@\h\[\033[00m\]\[\033[33m\] [\$CONDA_DEFAULT_ENV]\$(parse_git_branch):\[\033[01;34m\]\w\[\033[00m\]\n\$ " +``` + +Setup core dumps by adding the following to the end of the .bashrc file +```bash +ulimit -c unlimited +``` + +### 4. Managing Keys for Remote Login + +- create a local key on your laptop or desktop for remote use: `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"` +- your new key is located in `.ssh/id_rsa.pub` +- to log into a remote server with this key, while on the remote server copy and paste the key into the file `~/.ssh/authorized_keys` + +### 5. Managing Keys for GitHub + +- create a key on the server, while logged into your account: `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"` +- add the key as an ssh key to your github account + +### 6. Install and Configure Git + +```bash +sudo dnf install git +``` + +Create the local file `~/.gitconfig` in the user's home directory with the following contents: +```yml +[user] + name = + email = + +[push] + default = simple + +[diff] + tool = vimdiff + +[merge] + tool = vimdiff + +[difftool] + prompt = false + +[mergetool] + keepBackup = false + +[difftool "vimdiff"] + cmd = vimdiff "$LOCAL" "$REMOTE" + +[alias] + dd = difftool --dir-diff + +[credential "https://github.com"] + helper = + helper = !/usr/bin/gh auth git-credential + +[credential "https://gist.github.com"] + helper = + helper = !/usr/bin/gh auth git-credential +``` + +### 7. Clone Projects + +```bash +mkdir meta +cd meta +git clone git@github.com:ICESat2-SlideRule/sliderule.git +git clone git@github.com:ICESat2-SlideRule/sliderule-python.git +``` + +### 8. Installing and Configuring Docker + +```bash +sudo dnf install docker +sudo usermod -aG docker +newgrp docker +``` + +### 9. Install Dependencies got Local Build + +```bash +sudo dnf groupinstall "Development Tools" +sudo dnf install \ + cmake \ + readline-devel \ + lua-devel \ + openssl-devel + libuuid-devel \ + libtiff-devel \ + sqlite-devel \ + curl-devel \ + python-devel \ + meson \ + llvm \ + clang \ + clang-tools-extra \ + cppcheck +``` + +#### install rapidjson dependency +git clone https://github.com/Tencent/rapidjson.git +cd rapidjson +mkdir build +cd build +cmake .. +make -j8 +sudo make install +sudo ldconfig + +#### install arrow dependency +git clone https://github.com/apache/arrow.git +cd arrow +mkdir build +cd build +cmake ../cpp -DARROW_PARQUET=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON +make -j8 +sudo make install +sudo ldconfig + +#### install proj9 gdal/pdal dependency +git clone https://github.com/OSGeo/PROJ.git +cd PROJ +mkdir build +cd build +cmake .. -DCMAKE_BUILD_TYPE=Release +make -j8 +sudo make install +sudo ldconfig + +#### install geotiff gdal/pdal dependency +git clone https://github.com/OSGeo/libgeotiff.git +cd libgeotiff +mkdir build +cd build +cmake ../libgeotiff -DCMAKE_BUILD_TYPE=Release +make -j8 +sudo make install +sudo ldconfig + +#### install geos gdal dependency +git clone https://github.com/libgeos/geos.git +cd geos +mkdir build +cd build +cmake .. -DCMAKE_BUILD_TYPE=Release +make -j8 +sudo make install +sudo ldconfig + +#### install gdal dependency +git clone https://github.com/OSGeo/gdal.git +cd gdal +mkdir build +cd build +cmake .. -DCMAKE_BUILD_TYPE=Release +make -j8 +sudo make install +sudo ldconfig + +#### install pistache dependency +git clone https://github.com/pistacheio/pistache.git +cd pistache +meson setup build +sudo meson install -C build +sudo ldconfig + + +### 10. Install and Configure Miniconda + +```bash +wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-aarch64.sh +chmod +x Miniconda3-py39_4.12.0-Linux-aarch64.sh +./Miniconda3-py39_4.12.0-Linux-aarch64.sh +conda config --set changeps1 False +conda config --set auto_activate_base false +``` + +Once you have miniconda setup, you can navigate to the `sliderule-python` repository and run `conda env create -f environment.yml` to create a `sliderule_env` environment with everything you will need for the SlideRule Python client. + +### 11. Install GitHub Command Line Client + +```bash +sudo dnf install 'dnf-command(config-manager)' +sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo +sudo dnf install gh +``` + +### 12. Install and Configure AWS Command Line Client + +```bash +sudo dnf install awscli +``` + +Make sure to setup an initial .aws/credentials file so that it has the sliderule profile access key and secret access key. The credentials file will look something like: +``` +[default] +aws_access_key_id = _ +aws_secret_access_key = _ +aws_session_token = _ + +[sliderule] +aws_access_key_id = _ +aws_secret_access_key = _ +``` + +To populate the default keys and session token, run: +```bash +aws --profile=sliderule sts get-session-token --serial-number arn:aws:iam::$account_number:mfa/$user_name --token-code=$code +``` + +To login to the AWS Elastic Container Registry, run: +```bash +aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account_number.dkr.ecr.$region.amazonaws.com +``` + +### 13. Install Terraform & Packer + +```bash +sudo apt install unzip + +wget https://releases.hashicorp.com/terraform/1.3.1/terraform_1.3.1_linux_arm64.zip +unzip terraform_1.3.1_linux_arm64.zip +sudo mv terraform /usr/local/bin/ + +wget https://releases.hashicorp.com/packer/1.8.3/packer_1.8.3_linux_arm64.zip +unzip packer_1.8.3_linux_arm64.zip +sudo mv packer /usr/local/bin/ +``` From 23a675df236bdc8744031192faade1caa5c9586c Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Mon, 11 Dec 2023 20:15:29 +0000 Subject: [PATCH 02/15] linked in new doc for setting up amazon linux --- docs/rtd/source/tutorials/developer.rst | 1 + docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md | 4 +--- docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/rtd/source/tutorials/developer.rst b/docs/rtd/source/tutorials/developer.rst index 8f03fbc07..561f9f45c 100644 --- a/docs/rtd/source/tutorials/developer.rst +++ b/docs/rtd/source/tutorials/developer.rst @@ -10,5 +10,6 @@ Developer developer/docker_cli_tips.md developer/pub_to_pypi.md developer/ubuntu_arm_setup.md + developer/amazon_linux_arm_setup.md developer/certbot_instructions.md developer/backdoor_access.md diff --git a/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md index bc173cc2b..d55d04c10 100644 --- a/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md +++ b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md @@ -1,4 +1,4 @@ -# Setting Up Development Environment +# Setting Up Amazon Linux Development Environment 2023-12-11 @@ -249,8 +249,6 @@ aws ecr get-login-password --region $region | docker login --username AWS --pass ### 13. Install Terraform & Packer ```bash -sudo apt install unzip - wget https://releases.hashicorp.com/terraform/1.3.1/terraform_1.3.1_linux_arm64.zip unzip terraform_1.3.1_linux_arm64.zip sudo mv terraform /usr/local/bin/ diff --git a/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md b/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md index 7c94f577e..15a58130b 100644 --- a/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md +++ b/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md @@ -1,4 +1,4 @@ -# Setting Up Development Environment +# Setting Up Ubuntu Development Environment 2022-10-07 From 615eefd86f5f1ff28dc023b876edcb703b3a4082 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Mon, 11 Dec 2023 20:47:17 +0000 Subject: [PATCH 03/15] updates for amazon linux 2023 tutorial --- .../developer/amazon_linux_arm_setup.md | 31 ++++++++++++++----- project-config.cmake | 2 +- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md index d55d04c10..b234c947b 100644 --- a/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md +++ b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md @@ -109,6 +109,14 @@ sudo usermod -aG docker newgrp docker ``` +Then install Docker Compose plugin +```bash +DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} +mkdir -p $DOCKER_CONFIG/cli-plugins +curl -SL https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-aarch64 -o $DOCKER_CONFIG/cli-plugins/docker-compose +chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose +``` + ### 9. Install Dependencies got Local Build ```bash @@ -130,7 +138,14 @@ sudo dnf install \ cppcheck ``` -#### install rapidjson dependency +In an editor (e.g. vi), create a file in the `/etc/ld.so.conf.d/` directory called `local.conf` and in it put the one line below, and run `sudo ldconfig` afterwards. This allows any applications that are linked to libraries installed in `/usr/local/lib64` to be found. +```bash +/usr/local/lib64 +``` + +Go through and install all of the dependencies for SlideRule/ +```bash +# install rapidjson dependency git clone https://github.com/Tencent/rapidjson.git cd rapidjson mkdir build @@ -140,7 +155,7 @@ make -j8 sudo make install sudo ldconfig -#### install arrow dependency +# install arrow dependency git clone https://github.com/apache/arrow.git cd arrow mkdir build @@ -150,7 +165,7 @@ make -j8 sudo make install sudo ldconfig -#### install proj9 gdal/pdal dependency +# install proj9 gdal/pdal dependency git clone https://github.com/OSGeo/PROJ.git cd PROJ mkdir build @@ -160,7 +175,7 @@ make -j8 sudo make install sudo ldconfig -#### install geotiff gdal/pdal dependency +# install geotiff gdal/pdal dependency git clone https://github.com/OSGeo/libgeotiff.git cd libgeotiff mkdir build @@ -170,7 +185,7 @@ make -j8 sudo make install sudo ldconfig -#### install geos gdal dependency +# install geos gdal dependency git clone https://github.com/libgeos/geos.git cd geos mkdir build @@ -180,7 +195,7 @@ make -j8 sudo make install sudo ldconfig -#### install gdal dependency +# install gdal dependency git clone https://github.com/OSGeo/gdal.git cd gdal mkdir build @@ -190,13 +205,13 @@ make -j8 sudo make install sudo ldconfig -#### install pistache dependency +# install pistache dependency git clone https://github.com/pistacheio/pistache.git cd pistache meson setup build sudo meson install -C build sudo ldconfig - +``` ### 10. Install and Configure Miniconda diff --git a/project-config.cmake b/project-config.cmake index 787a7d8f4..1d4f4278d 100644 --- a/project-config.cmake +++ b/project-config.cmake @@ -120,7 +120,7 @@ if(CMAKE_BUILD_PLATFORM MATCHES "Linux") # Prefer libraries installed in /usr/local INCLUDE_DIRECTORIES(/usr/local/include) - LINK_DIRECTORIES(/usr/local/lib) + LINK_DIRECTORIES(/usr/local/lib /usr/local/lib64) # Set Environment Variables set (INSTALLDIR /usr/local CACHE STRING "Installation directory for library and executables") From 0f8039dfdc3c97f5a8a6183660fdc9a89925a1ae Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Tue, 12 Dec 2023 14:37:43 +0000 Subject: [PATCH 04/15] updated curl lib and iam_role_auth to use IMDSv2 --- packages/netsvc/CurlLib.cpp | 212 +++++++++++++++------------ packages/netsvc/CurlLib.h | 13 +- packages/netsvc/OrchestratorLib.cpp | 23 +-- packages/netsvc/netsvc.cpp | 1 + scripts/extensions/iam_role_auth.lua | 90 +++++++----- 5 files changed, 187 insertions(+), 152 deletions(-) diff --git a/packages/netsvc/CurlLib.cpp b/packages/netsvc/CurlLib.cpp index 972cb6872..f2e524807 100644 --- a/packages/netsvc/CurlLib.cpp +++ b/packages/netsvc/CurlLib.cpp @@ -60,17 +60,25 @@ void CurlLib::deinit (void) } /*---------------------------------------------------------------------------- - * get + * request *----------------------------------------------------------------------------*/ -long CurlLib::get (const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname) +long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname, vector* headers) { long http_code = 0; CURL* curl = NULL; /* Initialize Request */ data_t rqst; - rqst.data = (char*)data; - rqst.size = StringLib::size(data); + if(data) + { + rqst.data = (char*)data; + rqst.size = StringLib::size(data); + } + else + { + rqst.data = NULL; + rqst.size = 0; + } /* Initialize Response */ List rsps_set(EXPECTED_RESPONSE_SEGMENTS); @@ -91,99 +99,39 @@ long CurlLib::get (const char* url, const char* data, const char** response, int curl_easy_setopt(curl, CURLOPT_COOKIEFILE, ".cookies"); curl_easy_setopt(curl, CURLOPT_COOKIEJAR, ".cookies"); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - if(rqst.size > 0) + + if(verb == EndpointObject::GET && rqst.size > 0) { curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET"); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, rqst.data); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)rqst.size); } - - /* - * If you want to connect to a site who isn't using a certificate that is - * signed by one of the certs in the CA bundle you have, you can skip the - * verification of the server's certificate. This makes the connection - * A LOT LESS SECURE. - * - * If you have a CA cert for the server stored someplace else than in the - * default bundle, then the CURLOPT_CAPATH option might come handy for - * you. - */ - if(!verify_peer) + else if(verb == EndpointObject::POST) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, CurlLib::readData); + curl_easy_setopt(curl, CURLOPT_READDATA, &rqst); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)rqst.size); } - - /* - * If the site you're connecting to uses a different host name that what - * they have mentioned in their server certificate's commonName (or - * subjectAltName) fields, libcurl will refuse to connect. You can skip - * this check, but this will make the connection less secure. - */ - if(!verify_hostname) + else if(verb == EndpointObject::PUT) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, CurlLib::readData); + curl_easy_setopt(curl, CURLOPT_READDATA, &rqst); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)rqst.size); } - /* Perform the request, res will get the return code */ - CURLcode res = curl_easy_perform(curl); - - /* Check for Success */ - if(res == CURLE_OK) - { - /* Get HTTP Code */ - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - - /* Get Response */ - CurlLib::combineResponse(&rsps_set, response, size); - } - else + /* Add Headers */ + struct curl_slist* hdr_slist = NULL; + if(headers && headers->size() > 0) { - /* Unable to Perform cURL Call */ - FString error_msg("%s", curl_easy_strerror(res)); - *response = error_msg.c_str(true); - http_code = EndpointObject::Service_Unavailable; + for(const char* hdr: *headers) + { + hdr_slist = curl_slist_append(hdr_slist, hdr); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hdr_slist); } - /* Always Cleanup */ - curl_easy_cleanup(curl); - } - - /* Return HTTP Code */ - return http_code; -} - -/*---------------------------------------------------------------------------- - * post - *----------------------------------------------------------------------------*/ -long CurlLib::post (const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname) -{ - long http_code = 0; - CURL* curl = NULL; - - /* Initialize Request */ - data_t rqst; - rqst.data = (char*)data; - rqst.size = StringLib::size(data); - - /* Initialize Response */ - List rsps_set; - if(response) *response = NULL; - if(size) *size = 0; - - /* Initialize cURL */ - curl = curl_easy_init(); - if(curl) - { - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, CONNECTION_TIMEOUT); // seconds - curl_easy_setopt(curl, CURLOPT_TIMEOUT, DATA_TIMEOUT); // seconds - curl_easy_setopt(curl, CURLOPT_READFUNCTION, CurlLib::readData); - curl_easy_setopt(curl, CURLOPT_READDATA, &rqst); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)rqst.size); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlLib::writeData); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rsps_set); - /* * If you want to connect to a site who isn't using a certificate that is * signed by one of the certs in the CA bundle you have, you can skip the @@ -225,12 +173,14 @@ long CurlLib::post (const char* url, const char* data, const char** response, in else { /* Unable to Perform cURL Call */ - mlog(ERROR, "Unable to perform cRUL call on %s: %s", url, curl_easy_strerror(res)); + FString error_msg("%s", curl_easy_strerror(res)); + if(response) *response = error_msg.c_str(true); http_code = EndpointObject::Service_Unavailable; } /* Always Cleanup */ curl_easy_cleanup(curl); + curl_slist_free_all(hdr_slist); } /* Return HTTP Code */ @@ -375,25 +325,101 @@ long CurlLib::postAsRecord (const char* url, const char* data, Publisher* outq, return http_code; } +/*---------------------------------------------------------------------------- + * getHeaders + *----------------------------------------------------------------------------*/ +int CurlLib::getHeaders (lua_State* L, int index, vector& header_list) +{ + int num_hdrs = 0; + + /* Must be table of strings */ + if((lua_gettop(L) >= index) && lua_istable(L, index)) + { + /* Iterate through each item in table */ + int num_strings = lua_rawlen(L, index); + for(int i = 0; i < num_strings; i++) + { + /* Get item */ + lua_rawgeti(L, index, i+1); + if(lua_isstring(L, -1)) + { + const char* header = StringLib::duplicate(LuaObject::getLuaString(L, -1)); + header_list.push_back(header); + num_hdrs++; + } + + /* Clean up stack */ + lua_pop(L, 1); + } + } + + return num_hdrs; +} + /*---------------------------------------------------------------------------- * luaGet *----------------------------------------------------------------------------*/ int CurlLib::luaGet (lua_State* L) { bool status = false; + vector header_list; try { /* Get Parameters */ const char* url = LuaObject::getLuaString(L, 1); - bool verify_peer = LuaObject::getLuaBoolean(L, 2, true, false); - bool verify_hostname = LuaObject::getLuaBoolean(L, 3, true, false); - const char* data = LuaObject::getLuaString(L, 4, true, ""); + const char* data = LuaObject::getLuaString(L, 2, true, NULL); + int num_hdrs = CurlLib::getHeaders(L, 3, header_list); (void)num_hdrs; + bool verify_peer = LuaObject::getLuaBoolean(L, 4, true, false); + bool verify_hostname = LuaObject::getLuaBoolean(L, 5, true, false); /* Perform Request */ const char* response = NULL; int size = 0; - long http_code = CurlLib::get(url, data, &response, &size, verify_peer, verify_hostname); + long http_code = CurlLib::request(EndpointObject::GET, url, data, &response, &size, verify_peer, verify_hostname, &header_list); + if(response) + { + status = (http_code >= 200 && http_code < 300); + lua_pushlstring(L, response, size); + delete [] response; + } + else + { + lua_pushnil(L); + } + } + catch(const RunTimeException& e) + { + mlog(e.level(), "Error performing netsvc GET: %s", e.what()); + lua_pushnil(L); + } + + /* Return Status */ + lua_pushboolean(L, status); + return 2; +} + +/*---------------------------------------------------------------------------- + * luaPut + *----------------------------------------------------------------------------*/ +int CurlLib::luaPut (lua_State* L) +{ + bool status = false; + vector header_list; + + try + { + /* Get Parameters */ + const char* url = LuaObject::getLuaString(L, 1); + const char* data = LuaObject::getLuaString(L, 2, true, NULL); + int num_hdrs = CurlLib::getHeaders(L, 3, header_list); (void)num_hdrs; + bool verify_peer = LuaObject::getLuaBoolean(L, 4, true, false); + bool verify_hostname = LuaObject::getLuaBoolean(L, 5, true, false); + + /* Perform Request */ + const char* response = NULL; + int size = 0; + long http_code = CurlLib::request(EndpointObject::PUT, url, data, &response, &size, verify_peer, verify_hostname, &header_list); if(response) { status = (http_code >= 200 && http_code < 300); @@ -422,17 +448,19 @@ int CurlLib::luaGet (lua_State* L) int CurlLib::luaPost (lua_State* L) { bool status = false; + vector header_list; try { /* Get Parameters */ - const char* url = LuaObject::getLuaString(L, 1); - const char* data = LuaObject::getLuaString(L, 2, true, "{}"); + const char* url = LuaObject::getLuaString(L, 1); + const char* data = LuaObject::getLuaString(L, 2, true, "{}"); + int num_hdrs = CurlLib::getHeaders(L, 3, header_list); (void)num_hdrs; /* Perform Request */ const char* response = NULL; int size = 0; - long http_code = CurlLib::post(url, data, &response, &size); + long http_code = CurlLib::request(EndpointObject::POST, url, data, &response, &size, false, false, &header_list); if(response) { status = (http_code >= 200 && http_code < 300); diff --git a/packages/netsvc/CurlLib.h b/packages/netsvc/CurlLib.h index 06ae75bfc..ac16961e6 100644 --- a/packages/netsvc/CurlLib.h +++ b/packages/netsvc/CurlLib.h @@ -36,7 +36,10 @@ * INCLUDES ******************************************************************************/ -#include "core.h" +#include "LuaObject.h" +#include "EndpointObject.h" +#include "OsApi.h" +#include "MsgQ.h" /****************************************************************************** * cURL LIBRARY CLASS @@ -53,11 +56,15 @@ class CurlLib static void init (void); static void deinit (void); - static long get (const char* url, const char* data, const char** response, int* size=NULL, bool verify_peer=false, bool verify_hostname=false); - static long post (const char* url, const char* data, const char** response, int* size=NULL, bool verify_peer=false, bool verify_hostname=false); + static long request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false, vector* headers=NULL); + static long get (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); + static long put (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); + static long post (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long postAsStream (const char* url, const char* data, Publisher* outq, bool with_terminator); static long postAsRecord (const char* url, const char* data, Publisher* outq, bool with_terminator, int timeout, bool* active=NULL); + static int getHeaders (lua_State* L, int index, vector& header_list); static int luaGet (lua_State* L); + static int luaPut (lua_State* L); static int luaPost (lua_State* L); private: diff --git a/packages/netsvc/OrchestratorLib.cpp b/packages/netsvc/OrchestratorLib.cpp index df4f6b9ed..516ae570f 100644 --- a/packages/netsvc/OrchestratorLib.cpp +++ b/packages/netsvc/OrchestratorLib.cpp @@ -71,27 +71,10 @@ void OrchestratorLib::deinit (void) *----------------------------------------------------------------------------*/ OrchestratorLib::rsps_t OrchestratorLib::request (EndpointObject::verb_t verb, const char* resource, const char* data) { + rsps_t rsps; FString path("%s%s", URL, resource); - if(verb == EndpointObject::GET) - { - rsps_t rsps; - rsps.code = CurlLib::get(path.c_str(), data, &rsps.response, &rsps.size); - return rsps; - } - - if(verb == EndpointObject::POST) - { - rsps_t rsps; - rsps.code = CurlLib::post(path.c_str(), data, &rsps.response, &rsps.size); - return rsps; - } - - rsps_t err_rsps = { - .code = static_cast(EndpointObject::Bad_Request), - .response = NULL, - .size = 0 - }; - return err_rsps; + rsps.code = CurlLib::request(verb, path.c_str(), data, &rsps.response, &rsps.size); + return rsps; } /*---------------------------------------------------------------------------- diff --git a/packages/netsvc/netsvc.cpp b/packages/netsvc/netsvc.cpp index 46f770f6b..3ec4d5512 100644 --- a/packages/netsvc/netsvc.cpp +++ b/packages/netsvc/netsvc.cpp @@ -53,6 +53,7 @@ int netsvc_open (lua_State* L) { static const struct luaL_Reg netsvc_functions[] = { {"get", CurlLib::luaGet}, + {"put", CurlLib::luaPut}, {"post", CurlLib::luaPost}, {"proxy", EndpointProxy::luaCreate}, {"orchurl", OrchestratorLib::luaUrl}, diff --git a/scripts/extensions/iam_role_auth.lua b/scripts/extensions/iam_role_auth.lua index d521b8657..44ce1c8db 100644 --- a/scripts/extensions/iam_role_auth.lua +++ b/scripts/extensions/iam_role_auth.lua @@ -5,53 +5,69 @@ local json = require("json") local parm = json.decode(arg[1] or "{}") +local aws_token_url = "http://169.254.169.254/latest/api/token" local aws_meta_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials" local identity = parm["identity"] or "iam-role" +-- make aws api request +local function aws_api_rqst(url) + local token, status1 = netsvc.put(aws_token_url,"",{"X-aws-ec2-metadata-token-ttl-seconds: 3600"}) + if not status1 then + sys.log(core.CRITICAL, "Unable to get AWS API token from "..aws_token_url) + end + local rsp, status2 = netsvc.get(url,"",{string.format("X-aws-ec2-metadata-token: %s",token)}) + if not status2 then + sys.log(core.CRITICAL, "Unable to make AWS API request to "..url) + end + return rsp, (status1 and status2) +end + -- get current EC2 role -local role, status = netsvc.get(aws_meta_url) -if status then - - -- maintain aws credentials - while sys.alive() do - sys.log(core.DEBUG, "Fetching IAM role credentials...") - - -- get new credentials - local response, status = netsvc.get(aws_meta_url.."/"..role) - - -- convert reponse to credential table - if status then - local rc, credential = pcall(json.decode, response) - - -- store credentials and wait for next time to update - if rc and credential and credential.Expiration then - sys.log(core.INFO, string.format("New IAM role %s credentials fetched, expiration: %s", role, credential.Expiration)) - - -- store credentials for use by server - rc = aws.csput(identity, credential) - - if rc then - -- calculate next fetch time - local now = time.gps() - local credential_gps = time.gmt2gps(credential.Expiration) - local credential_duration = credential_gps - now - local next_fetch_time = now + (credential_duration / 2) - - -- wait until next fetch time - while sys.alive() and time.gps() < next_fetch_time do - sys.wait(5) - end - else - sys.log(core.CRITICAL, string.format("Failed to process IAM role %s credentials, retrying in 60 seconds", role)) - sys.wait(60) +local role, status = aws_api_rqst(aws_meta_url) +if not status then + sys.log(core.CRITICAL, "Unable to fetch IAM security credentials!") + do return false end +end + +-- maintain aws credentials +while sys.alive() do + sys.log(core.DEBUG, "Fetching IAM role credentials...") + + -- get new credentials + local response, status = aws_api_rqst(aws_meta_url.."/"..role) + + -- convert reponse to credential table + if status then + local rc, credential = pcall(json.decode, response) + + -- store credentials and wait for next time to update + if rc and credential and credential.Expiration then + sys.log(core.INFO, string.format("New IAM role %s credentials fetched, expiration: %s", role, credential.Expiration)) + + -- store credentials for use by server + rc = aws.csput(identity, credential) + + if rc then + -- calculate next fetch time + local now = time.gps() + local credential_gps = time.gmt2gps(credential.Expiration) + local credential_duration = credential_gps - now + local next_fetch_time = now + (credential_duration / 2) + + -- wait until next fetch time + while sys.alive() and time.gps() < next_fetch_time do + sys.wait(5) end else - sys.log(core.CRITICAL, string.format("Failed to decode IAM role %s response, retrying in 60 seconds", role)) + sys.log(core.CRITICAL, string.format("Failed to process IAM role %s credentials, retrying in 60 seconds", role)) sys.wait(60) end else - sys.log(core.CRITICAL, string.format("Failed to retrieve IAM role %s credentials, retrying in 60 seconds", role)) + sys.log(core.CRITICAL, string.format("Failed to decode IAM role %s response, retrying in 60 seconds", role)) sys.wait(60) end + else + sys.log(core.CRITICAL, string.format("Failed to retrieve IAM role %s credentials, retrying in 60 seconds", role)) + sys.wait(60) end end From 98110bc8a4cad02447f42a4104caec67507869a2 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Wed, 13 Dec 2023 13:17:13 +0000 Subject: [PATCH 05/15] static analysis updates for al2023 --- packages/aws/CredentialStore.cpp | 2 +- packages/aws/CredentialStore.h | 2 +- packages/aws/S3CurlIODriver.cpp | 2 +- packages/ccsds/CcsdsParserAOSFrameModule.cpp | 2 +- packages/ccsds/CcsdsParserAOSFrameModule.h | 2 +- packages/ccsds/CcsdsParserZFrameModule.cpp | 22 +++---- packages/ccsds/CcsdsRecord.cpp | 2 +- packages/ccsds/CcsdsRecord.h | 2 +- packages/core/Asset.cpp | 2 +- packages/core/Asset.h | 2 +- packages/core/AssetIndex.h | 6 +- packages/core/Dictionary.h | 20 +++--- packages/core/HttpClient.cpp | 7 +- packages/core/LuaEngine.h | 2 +- packages/core/LuaObject.cpp | 4 +- packages/core/MathLib.cpp | 5 ++ packages/core/TimeLib.cpp | 2 - packages/geo/GdalRaster.cpp | 7 +- packages/h5/H5Coro.cpp | 2 +- packages/h5/H5DArray.cpp | 3 +- packages/legacy/CcsdsPacketProcessor.cpp | 4 -- packages/legacy/CfsInterface.cpp | 2 +- packages/legacy/CfsInterface.h | 2 +- packages/legacy/CosmosInterface.cpp | 67 +++++++------------- packages/legacy/CosmosInterface.h | 19 ++---- packages/legacy/StatisticRecord.h | 2 +- packages/legacy/UT_Dictionary.cpp | 6 +- packages/legacy/UT_Dictionary.h | 6 +- packages/legacy/UT_Table.cpp | 18 +++--- packages/netsvc/EndpointProxy.cpp | 9 ++- packages/netsvc/NetsvcParms.cpp | 2 +- packages/netsvc/ProvisioningSystemLib.cpp | 2 +- platforms/linux/OsApi.cpp | 2 +- platforms/linux/SockLib.cpp | 4 +- plugins/gedi/plugin/FootprintReader.h | 7 +- plugins/icesat2/plugin/AncillaryFields.cpp | 2 +- plugins/icesat2/plugin/Atl08Dispatch.cpp | 2 +- plugins/icesat2/plugin/Atl08Dispatch.h | 2 +- plugins/pgc/plugin/PgcDemStripsRaster.cpp | 1 + project-config.cmake | 3 + scripts/extensions/asset.lua | 2 +- targets/slideruleearth-aws/Makefile | 2 +- 42 files changed, 113 insertions(+), 151 deletions(-) diff --git a/packages/aws/CredentialStore.cpp b/packages/aws/CredentialStore.cpp index ea13abfb3..cc67d18fa 100644 --- a/packages/aws/CredentialStore.cpp +++ b/packages/aws/CredentialStore.cpp @@ -107,7 +107,7 @@ CredentialStore::Credential CredentialStore::get (const char* host) /*---------------------------------------------------------------------------- * put *----------------------------------------------------------------------------*/ -bool CredentialStore::put (const char* host, Credential& credential) +bool CredentialStore::put (const char* host, const Credential& credential) { bool status = false; diff --git a/packages/aws/CredentialStore.h b/packages/aws/CredentialStore.h index 40dc19335..d3b194423 100644 --- a/packages/aws/CredentialStore.h +++ b/packages/aws/CredentialStore.h @@ -241,7 +241,7 @@ class CredentialStore static void deinit (void); static Credential get (const char* host); - static bool put (const char* host, Credential& credential); + static bool put (const char* host, const Credential& credential); static int luaGet (lua_State* L); static int luaPut (lua_State* L); diff --git a/packages/aws/S3CurlIODriver.cpp b/packages/aws/S3CurlIODriver.cpp index bbf75df0b..88c705add 100644 --- a/packages/aws/S3CurlIODriver.cpp +++ b/packages/aws/S3CurlIODriver.cpp @@ -109,7 +109,7 @@ static size_t curlWriteFixed(void *buffer, size_t size, size_t nmemb, void *user *----------------------------------------------------------------------------*/ static size_t curlWriteStreaming(void *buffer, size_t size, size_t nmemb, void *userp) { - List* rsps_set = (List*)userp; + List* rsps_set = reinterpret_cast*>(userp); streaming_data_t rsps; rsps.size = size * nmemb; rsps.data = new char [rsps.size]; diff --git a/packages/ccsds/CcsdsParserAOSFrameModule.cpp b/packages/ccsds/CcsdsParserAOSFrameModule.cpp index c3ec4f914..c55dca023 100644 --- a/packages/ccsds/CcsdsParserAOSFrameModule.cpp +++ b/packages/ccsds/CcsdsParserAOSFrameModule.cpp @@ -444,7 +444,7 @@ CcsdsParserAOSFrameModule::~CcsdsParserAOSFrameModule(void) /*---------------------------------------------------------------------------- * crc16 *----------------------------------------------------------------------------*/ -uint16_t CcsdsParserAOSFrameModule::crc16(uint8_t* data, uint32_t len, uint16_t crc) +uint16_t CcsdsParserAOSFrameModule::crc16(const uint8_t* data, uint32_t len, uint16_t crc) { static const uint16_t CrcTable[256]= { diff --git a/packages/ccsds/CcsdsParserAOSFrameModule.h b/packages/ccsds/CcsdsParserAOSFrameModule.h index d41bfdbe4..ffd58eff8 100644 --- a/packages/ccsds/CcsdsParserAOSFrameModule.h +++ b/packages/ccsds/CcsdsParserAOSFrameModule.h @@ -65,7 +65,7 @@ class CcsdsParserAOSFrameModule: public CcsdsParserModule CcsdsParserAOSFrameModule (lua_State* L, int scid, int vcid, int strip_size, uint8_t* sync_marker, int sync_size, int sync_offset, int fixed_size, int leading_size, int trailer_size); virtual ~CcsdsParserAOSFrameModule (void); - uint16_t crc16 (uint8_t* data, uint32_t len, uint16_t crc); + uint16_t crc16 (const uint8_t* data, uint32_t len, uint16_t crc); private: diff --git a/packages/ccsds/CcsdsParserZFrameModule.cpp b/packages/ccsds/CcsdsParserZFrameModule.cpp index 0b9278f5e..27dfbaa3d 100644 --- a/packages/ccsds/CcsdsParserZFrameModule.cpp +++ b/packages/ccsds/CcsdsParserZFrameModule.cpp @@ -245,20 +245,16 @@ void CcsdsParserZFrameModule::gotoInitState(bool reset) } else { - /* Check if Frame Complete */ - if(state == FRAME_Z) + /* Check if Frame is Not Complete - Stay in FANN */ + if(frameIndex < frameSize) { - /* Check if Frame is Not Complete - Stay in FANN */ - if(frameIndex < frameSize) - { - state = FRAME_FANN; - } - /* Check if Frame is Complete - Reset Indices */ - else - { - frameIndex = 0; - frameSize = 0; - } + state = FRAME_FANN; + } + /* Check if Frame is Complete - Reset Indices */ + else + { + frameIndex = 0; + frameSize = 0; } } } diff --git a/packages/ccsds/CcsdsRecord.cpp b/packages/ccsds/CcsdsRecord.cpp index fa0886e48..3e49bc235 100644 --- a/packages/ccsds/CcsdsRecord.cpp +++ b/packages/ccsds/CcsdsRecord.cpp @@ -333,7 +333,7 @@ void CcsdsRecord::populateHeader(void) /*---------------------------------------------------------------------------- * getPacketDefinition *----------------------------------------------------------------------------*/ -CcsdsRecord::pktDef_t* CcsdsRecord::getPacketDefinition(unsigned char* buffer, int size) +CcsdsRecord::pktDef_t* CcsdsRecord::getPacketDefinition(const unsigned char* buffer, int size) { if(size < 6) return NULL; diff --git a/packages/ccsds/CcsdsRecord.h b/packages/ccsds/CcsdsRecord.h index 40540a8b2..de81c3e0e 100644 --- a/packages/ccsds/CcsdsRecord.h +++ b/packages/ccsds/CcsdsRecord.h @@ -120,7 +120,7 @@ class CcsdsRecord: public RecordObject CcsdsRecord (void); void populateHeader (void); - pktDef_t* getPacketDefinition (unsigned char* buffer, int size); // overloaded RecordObject method + pktDef_t* getPacketDefinition (const unsigned char* buffer, int size); // overloaded RecordObject method }; /****************************************************************************** diff --git a/packages/core/Asset.cpp b/packages/core/Asset.cpp index 8e3669a76..711a31a3f 100644 --- a/packages/core/Asset.cpp +++ b/packages/core/Asset.cpp @@ -186,7 +186,7 @@ Asset::~Asset (void) /*---------------------------------------------------------------------------- * load *----------------------------------------------------------------------------*/ -int Asset::load (resource_t& resource) +int Asset::load (const resource_t& resource) { return resources.add(resource); } diff --git a/packages/core/Asset.h b/packages/core/Asset.h index 0dfb4d63a..a69ce4788 100644 --- a/packages/core/Asset.h +++ b/packages/core/Asset.h @@ -106,7 +106,7 @@ class Asset: public LuaObject virtual ~Asset (void); - int load (resource_t& resource); + int load (const resource_t& resource); resource_t& operator[] (int i); int size (void) const; diff --git a/packages/core/AssetIndex.h b/packages/core/AssetIndex.h index 310bfb662..1bc69832a 100644 --- a/packages/core/AssetIndex.h +++ b/packages/core/AssetIndex.h @@ -261,7 +261,7 @@ int AssetIndex::luaAdd (lua_State* L) try { /* Get Self */ - AssetIndex* lua_obj = (AssetIndex*)getLuaSelf(L, 1); + AssetIndex* lua_obj = reinterpret_cast*>(getLuaSelf(L, 1)); /* Create Resource Attributes */ T span = lua_obj->luatable2span(L, 2); @@ -289,7 +289,7 @@ int AssetIndex::luaQuery (lua_State* L) try { /* Get Self */ - AssetIndex* lua_obj = (AssetIndex*)getLuaSelf(L, 1); + AssetIndex* lua_obj = reinterpret_cast*>(getLuaSelf(L, 1)); /* Create Query Attributes */ T span = lua_obj->luatable2span(L, 2); @@ -334,7 +334,7 @@ int AssetIndex::luaDisplay (lua_State* L) try { /* Get Parameters */ - AssetIndex* lua_obj = (AssetIndex*)getLuaSelf(L, 1); + AssetIndex* lua_obj = reinterpret_cast*>(getLuaSelf(L, 1)); /* Display Tree */ lua_obj->display(); diff --git a/packages/core/Dictionary.h b/packages/core/Dictionary.h index 6af77b9b0..0c7cf0f35 100644 --- a/packages/core/Dictionary.h +++ b/packages/core/Dictionary.h @@ -686,21 +686,21 @@ unsigned int Dictionary::getNode(const char* key) const while(index != NULL_INDEX && hashTable[index].chain != EMPTY_ENTRY) { /* Compare Hash Key to Key */ - for(int i = 0; i < MAX_STR_SIZE; i++) + int i = 0; + while(hashTable[index].key[i] == key[i]) { - if(hashTable[index].key[i] != key[i]) + /* If there is no difference AND key is at null, return match */ + if(key[i] == '\0') { - /* As soon as there is a difference, go to next */ - index = hashTable[index].next; - break; - } - - if(key[i] == '\0') - { - /* If there is no difference AND key is at null, return match */ return index; } + + /* Goto next character */ + i++; } + + /* As soon as there is a difference, go to next */ + index = hashTable[index].next; } } diff --git a/packages/core/HttpClient.cpp b/packages/core/HttpClient.cpp index d0436523f..d99c6072f 100644 --- a/packages/core/HttpClient.cpp +++ b/packages/core/HttpClient.cpp @@ -468,11 +468,8 @@ HttpClient::rsps_t HttpClient::parseResponse (Publisher* outq, int timeout, int3 } /* Populate Response */ - if(rsps_bytes > 0) - { - memcpy(&rsps.response[rsps_index], &rspsBuf[line_start], rsps_bytes); - rsps.response[rsps_index + rsps_bytes] = '\0'; // ensure termination - } + memcpy(&rsps.response[rsps_index], &rspsBuf[line_start], rsps_bytes); + rsps.response[rsps_index + rsps_bytes] = '\0'; // ensure termination /* Update Indices */ rsps_index += rsps_bytes; diff --git a/packages/core/LuaEngine.h b/packages/core/LuaEngine.h index e3e51dd64..9069056c4 100644 --- a/packages/core/LuaEngine.h +++ b/packages/core/LuaEngine.h @@ -86,7 +86,7 @@ class LuaEngine *--------------------------------------------------------------------*/ LuaEngine (const char* name, int lua_argc, char lua_argv[][MAX_LUA_ARG], uint32_t trace_id=ORIGIN, luaStepHook hook=NULL, bool paused=false); // protected mode - LuaEngine (const char* script, const char* arg, uint32_t trace_id=ORIGIN, luaStepHook hook=NULL, bool paused=false); // direct mode + explicit LuaEngine (const char* script, const char* arg, uint32_t trace_id=ORIGIN, luaStepHook hook=NULL, bool paused=false); // direct mode ~LuaEngine (void); static void init (void); diff --git a/packages/core/LuaObject.cpp b/packages/core/LuaObject.cpp index 69e4c2385..f1bead735 100644 --- a/packages/core/LuaObject.cpp +++ b/packages/core/LuaObject.cpp @@ -82,7 +82,6 @@ int LuaObject::getLuaNumParms (lua_State* L) *----------------------------------------------------------------------------*/ int LuaObject::luaGetByName(lua_State* L) { - bool verbose = true; try { LuaObject* lua_obj = NULL; @@ -90,7 +89,6 @@ int LuaObject::luaGetByName(lua_State* L) { /* Get Parameters */ const char* name = getLuaString(L, 1); - verbose = getLuaBoolean(L, 2, true, true); /* Get Self */ lua_obj = globalObjects.get(name).lua_obj; @@ -104,7 +102,7 @@ int LuaObject::luaGetByName(lua_State* L) catch(const RunTimeException& e) { globalMut.unlock(); - if(verbose) mlog(e.level(), "Failed to get Lua object by name: %s", e.what()); + mlog(DEBUG, "Failed to get Lua object by name: %s", e.what()); lua_pushnil(L); } diff --git a/packages/core/MathLib.cpp b/packages/core/MathLib.cpp index 23fcb153c..44e667720 100644 --- a/packages/core/MathLib.cpp +++ b/packages/core/MathLib.cpp @@ -156,6 +156,11 @@ MathLib::point_t MathLib::coord2point (coord_t c, proj_t projection) p.x = EARTHRADIUS * lonrad; p.y = EARTHRADIUS * latrad; } + else + { + p.x = NAN; + p.y = NAN; + } return p; } diff --git a/packages/core/TimeLib.cpp b/packages/core/TimeLib.cpp index 6c4c12ec4..3ac02dfaa 100644 --- a/packages/core/TimeLib.cpp +++ b/packages/core/TimeLib.cpp @@ -426,7 +426,6 @@ int64_t TimeLib::str2gpstime (const char* time_str) int token_count = 1; int colon_count = 0; int dash_count = 0; - int space_count = 0; int tz_count = 0; int zulu_count = 0; token_ptr[0] = &time_buf[0]; @@ -442,7 +441,6 @@ int64_t TimeLib::str2gpstime (const char* time_str) } else if(time_str[i] == ' ') { - space_count++; is_delim = true; } else if(time_str[i] == '-') diff --git a/packages/geo/GdalRaster.cpp b/packages/geo/GdalRaster.cpp index 9d2f3ad3c..fd6ec2b19 100644 --- a/packages/geo/GdalRaster.cpp +++ b/packages/geo/GdalRaster.cpp @@ -985,11 +985,8 @@ RasterSubset* GdalRaster::getRasterSubset(uint32_t ulx, uint32_t uly, double map } catch (const RunTimeException &e) { - if(subset) - { - delete subset; - subset = NULL; - } + delete subset; + subset = NULL; mlog(e.level(), "Error subsetting: %s", e.what()); } diff --git a/packages/h5/H5Coro.cpp b/packages/h5/H5Coro.cpp index 1874f7f90..1a3444b69 100644 --- a/packages/h5/H5Coro.cpp +++ b/packages/h5/H5Coro.cpp @@ -3352,7 +3352,7 @@ int H5FileBuffer::shuffleChunk (const uint8_t* input, uint32_t input_size, uint8 { if(H5_ERROR_CHECKING) { - if(type_size < 0 || type_size > 8) + if(type_size <= 0 || type_size > 8) { throw RunTimeException(CRITICAL, RTE_ERROR, "invalid data size to perform shuffle on: %d", type_size); } diff --git a/packages/h5/H5DArray.cpp b/packages/h5/H5DArray.cpp index 98b227379..f016879a1 100644 --- a/packages/h5/H5DArray.cpp +++ b/packages/h5/H5DArray.cpp @@ -180,7 +180,6 @@ uint64_t H5DArray::serialize (uint8_t* buffer, int32_t start_element, uint32_t n /* Return Number of Bytes Serialized */ int64_t elements_available = h5f->info.elements - start_element; - if(elements_available < 0) elements_available = 0; - uint64_t elements_copied = MIN(elements_available, num_elements); + int64_t elements_copied = MAX(MIN(elements_available, num_elements), 0); return elements_copied * h5f->info.typesize; } diff --git a/packages/legacy/CcsdsPacketProcessor.cpp b/packages/legacy/CcsdsPacketProcessor.cpp index f357c4d02..9302b4af7 100644 --- a/packages/legacy/CcsdsPacketProcessor.cpp +++ b/packages/legacy/CcsdsPacketProcessor.cpp @@ -580,10 +580,6 @@ bool CcsdsPacketProcessor::resetProcessing (void) { if(pktProcessor[apid].enable && pktProcessor[apid].segments) { - for(int seg=0; seg < pktProcessor[apid].segments->length(); seg++) - { - delete pktProcessor[apid].segments->get(seg); - } pktProcessor[apid].segments->clear(); delete pktProcessor[apid].segments; } diff --git a/packages/legacy/CfsInterface.cpp b/packages/legacy/CfsInterface.cpp index 4663472c8..1b31e5e56 100644 --- a/packages/legacy/CfsInterface.cpp +++ b/packages/legacy/CfsInterface.cpp @@ -548,7 +548,7 @@ bool CfsInterface::validatePkt (unsigned char* pktbuf, int bytes) * * Notes: assumes primary header is present *----------------------------------------------------------------------------*/ -void CfsInterface::measurePkt (unsigned char* pktbuf, int bytes) +void CfsInterface::measurePkt (const unsigned char* pktbuf, int bytes) { if(pktbuf == NULL || bytes < 6) return; diff --git a/packages/legacy/CfsInterface.h b/packages/legacy/CfsInterface.h index b43456f85..c6d45ac38 100644 --- a/packages/legacy/CfsInterface.h +++ b/packages/legacy/CfsInterface.h @@ -141,7 +141,7 @@ class CfsInterface: public CommandableObject PktStats* createPktStat (uint16_t apid); bool validatePkt (unsigned char* pktbuf, int bytes); - void measurePkt (unsigned char* pktbuf, int bytes); + void measurePkt (const unsigned char* pktbuf, int bytes); }; diff --git a/packages/legacy/CosmosInterface.cpp b/packages/legacy/CosmosInterface.cpp index 72a66856b..be7377322 100644 --- a/packages/legacy/CosmosInterface.cpp +++ b/packages/legacy/CosmosInterface.cpp @@ -164,13 +164,6 @@ CosmosInterface::~CosmosInterface(void) delete [] tlmQName; delete [] cmdQName; - - tlmConnMut.lock(); - { - tlmConnections.flush(); - cmdConnections.flush(); - } - tlmConnMut.unlock(); } /*---------------------------------------------------------------------------- @@ -224,13 +217,8 @@ int CosmosInterface::tlmActiveHandler(int fd, int flags, void* parm) rqst->sock = new TcpSocket(NULL, fd); mlog(INFO, "Establishing new connection to %s:%d in %s", rqst->sock->getIpAddr() ? rqst->sock->getIpAddr() : "UNKNOWN", rqst->sock->getPort(), ci->getName()); - /* Register and Start Connection */ - ci->tlmConnMut.lock(); - { - ci->tlmConnections.add(rqst->sock->getUniqueId(), rqst); - rqst->pid = new Thread(telemetryThread, rqst, false); - } - ci->tlmConnMut.unlock(); + /* Start Connection */ + rqst->pid = new Thread(telemetryThread, rqst, false); } return 0; // return success @@ -254,13 +242,8 @@ int CosmosInterface::cmdActiveHandler(int fd, int flags, void* parm) rqst->sock = new TcpSocket(NULL, fd); mlog(INFO, "Establishing new connection to %s:%d in %s", rqst->sock->getIpAddr() ? rqst->sock->getIpAddr() : "UNKNOWN", rqst->sock->getPort(), ci->getName()); - /* Register and Start Connection */ - ci->cmdConnMut.lock(); - { - ci->cmdConnections.add(rqst->sock->getUniqueId(), rqst); - rqst->pid = new Thread(commandThread, rqst, false); - } - ci->cmdConnMut.unlock(); + /* Start Connection */ + rqst->pid = new Thread(commandThread, rqst, false); } return 0; // return success @@ -300,16 +283,10 @@ void* CosmosInterface::telemetryThread (void* parm) } mlog(DEBUG, "Terminating connection to %s in %s", rqst->sock->getIpAddr(), ci->getName()); - ci->tlmConnMut.lock(); - { - // !!! CANNOT access request values after this call !!! - // this will call freeConnectionEntry which - // closes the socket and deletes objects - ci->tlmConnections.remove(rqst->sock->getUniqueId()); - } - ci->tlmConnMut.unlock(); - delete[] buffer; + /* Clean Up */ + delete rqst; + delete [] buffer; return NULL; } @@ -319,7 +296,7 @@ void* CosmosInterface::telemetryThread (void* parm) *----------------------------------------------------------------------------*/ void* CosmosInterface::commandThread (void* parm) { - cmd_t* c = static_cast(parm); + cmd_t* rqst = static_cast(parm); unsigned char header_buf[HEADER_SIZE]; int header_index = 0; int packet_index = 0; @@ -327,12 +304,12 @@ void* CosmosInterface::commandThread (void* parm) unsigned char* packet_buf = new unsigned char[MAX_PACKET_SIZE]; - while(c->ci->interfaceActive) + while(rqst->ci->interfaceActive) { /* Read Header */ if(header_index != HEADER_SIZE) { - int bytes = c->sock->readBuffer(&header_buf[header_index], HEADER_SIZE - header_index); + int bytes = rqst->sock->readBuffer(&header_buf[header_index], HEADER_SIZE - header_index); if(bytes > 0) { header_index += bytes; @@ -361,26 +338,26 @@ void* CosmosInterface::commandThread (void* parm) /* Handle Loss of Synchronization */ if(!in_sync) { - mlog(CRITICAL, "Lost synchronization to COSMOS command interface in %s", c->ci->getName()); + mlog(CRITICAL, "Lost synchronization to COSMOS command interface in %s", rqst->ci->getName()); memmove(&header_buf[0], &header_buf[1], HEADER_SIZE - 1); header_index--; // shift down } } } - else if(!c->sock->isConnected()) + else if(!rqst->sock->isConnected()) { OsApi::sleep(1); } else if(bytes != TIMEOUT_RC) { - mlog(CRITICAL, "Failed to read header (%d) on %s command socket... fatal error, exiting command thread", bytes, c->ci->getName()); + mlog(CRITICAL, "Failed to read header (%d) on %s command socket... fatal error, exiting command thread", bytes, rqst->ci->getName()); break; } } /* Read Packet */ else if(packet_index < packet_size) { - int bytes = c->sock->readBuffer(&packet_buf[packet_index], packet_size - packet_index); + int bytes = rqst->sock->readBuffer(&packet_buf[packet_index], packet_size - packet_index); if(bytes > 0) { packet_index += bytes; @@ -388,12 +365,12 @@ void* CosmosInterface::commandThread (void* parm) { /* Post Packet */ int status = MsgQ::STATE_TIMEOUT; - while(c->ci->interfaceActive && status == MsgQ::STATE_TIMEOUT) + while(rqst->ci->interfaceActive && status == MsgQ::STATE_TIMEOUT) { - status = c->pub->postCopy(&packet_buf[0], packet_size, SYS_TIMEOUT); - if(status < 0 && status != MsgQ::STATE_TIMEOUT) + status = rqst->pub->postCopy(&packet_buf[0], packet_size, SYS_TIMEOUT); + if(status < 0) { - mlog(CRITICAL, "Message of size %d unable to be posted (%d) to output stream %s", bytes, status, c->pub->getName()); + mlog(CRITICAL, "Message of size %d unable to be posted (%d) to output stream %s", bytes, status, rqst->pub->getName()); break; } } @@ -404,19 +381,21 @@ void* CosmosInterface::commandThread (void* parm) packet_size = 0; } } - else if(!c->sock->isConnected()) + else if(!rqst->sock->isConnected()) { OsApi::sleep(1); } else if(bytes != TIMEOUT_RC) { - mlog(CRITICAL, "Failed to read packet (%d) on %s command socket... fatal error, exiting command thread", bytes, c->ci->getName()); + mlog(CRITICAL, "Failed to read packet (%d) on %s command socket... fatal error, exiting command thread", bytes, rqst->ci->getName()); break; } } } - delete[] packet_buf; + /* Clean Up */ + delete rqst; + delete [] packet_buf; return NULL; } diff --git a/packages/legacy/CosmosInterface.h b/packages/legacy/CosmosInterface.h index 84fcf262b..9c3e00aeb 100644 --- a/packages/legacy/CosmosInterface.h +++ b/packages/legacy/CosmosInterface.h @@ -100,9 +100,9 @@ class CosmosInterface: public CommandableObject pid = NULL; } ~tlm_t(void) - { if(sub) delete sub; - if(sock) delete sock; - if(pid) delete pid; } + { delete sub; + delete sock; + delete pid; } }; struct cmd_t @@ -119,14 +119,11 @@ class CosmosInterface: public CommandableObject pid = NULL; } ~cmd_t(void) - { if(pub) delete pub; - if(sock) delete sock; - if(pid) delete pid; } + { delete pub; + delete sock; + delete pid; } }; - typedef Ordering TlmOrdering; - typedef Ordering CmdOrdering; - /*-------------------------------------------------------------------- * Data *--------------------------------------------------------------------*/ @@ -137,15 +134,11 @@ class CosmosInterface: public CommandableObject // telemetry connections Thread* tlmListenerPid; listener_t tlmListener; - TlmOrdering tlmConnections; - Mutex tlmConnMut; const char* tlmQName; // command connections Thread* cmdListenerPid; listener_t cmdListener; - CmdOrdering cmdConnections; - Mutex cmdConnMut; const char* cmdQName; /*-------------------------------------------------------------------- diff --git a/packages/legacy/StatisticRecord.h b/packages/legacy/StatisticRecord.h index 086095db2..185dff903 100644 --- a/packages/legacy/StatisticRecord.h +++ b/packages/legacy/StatisticRecord.h @@ -294,7 +294,7 @@ typename StatisticRecord::clear_t StatisticRecord::str2clear(const char* s template void* StatisticRecord::telemetryThread(void* parm) { - StatisticRecord* procstat = (StatisticRecord*)parm; + StatisticRecord* procstat = reinterpret_cast*>(parm); int wait_counter = procstat->telemetryWaitSeconds; diff --git a/packages/legacy/UT_Dictionary.cpp b/packages/legacy/UT_Dictionary.cpp index 743287fd0..f272aa9f1 100644 --- a/packages/legacy/UT_Dictionary.cpp +++ b/packages/legacy/UT_Dictionary.cpp @@ -79,7 +79,7 @@ UT_Dictionary::~UT_Dictionary(void) /*---------------------------------------------------------------------------- * functionalUnitTestCmd - *----------------------------------------------------------------------------*/ -int UT_Dictionary::functionalUnitTestCmd (int argc, char argv[][MAX_CMD_SIZE]) +int UT_Dictionary::functionalUnitTestCmd (int argc, const char argv[][MAX_CMD_SIZE]) { (void)argc; @@ -284,7 +284,7 @@ int UT_Dictionary::functionalUnitTestCmd (int argc, char argv[][MAX_CMD_SIZE]) /*---------------------------------------------------------------------------- * iteratorUnitTestCmd - *----------------------------------------------------------------------------*/ -int UT_Dictionary::iteratorUnitTestCmd (int argc, char argv[][MAX_CMD_SIZE]) +int UT_Dictionary::iteratorUnitTestCmd (int argc, const char argv[][MAX_CMD_SIZE]) { (void)argc; @@ -408,7 +408,7 @@ int UT_Dictionary::iteratorUnitTestCmd (int argc, char argv[][MAX_CMD_SIZE]) /*---------------------------------------------------------------------------- * addWordSetCmd - *----------------------------------------------------------------------------*/ -int UT_Dictionary::addWordSetCmd (int argc, char argv[][MAX_CMD_SIZE]) +int UT_Dictionary::addWordSetCmd (int argc, const char argv[][MAX_CMD_SIZE]) { (void)argc; diff --git a/packages/legacy/UT_Dictionary.h b/packages/legacy/UT_Dictionary.h index a3a44d8c0..5d8b1cc96 100644 --- a/packages/legacy/UT_Dictionary.h +++ b/packages/legacy/UT_Dictionary.h @@ -73,9 +73,9 @@ class UT_Dictionary: public CommandableObject UT_Dictionary (CommandProcessor* cmd_proc, const char* obj_name); ~UT_Dictionary (void); - int functionalUnitTestCmd (int argc, char argv[][MAX_CMD_SIZE]); - int iteratorUnitTestCmd (int argc, char argv[][MAX_CMD_SIZE]); - int addWordSetCmd (int argc, char argv[][MAX_CMD_SIZE]); + int functionalUnitTestCmd (int argc, const char argv[][MAX_CMD_SIZE]); + int iteratorUnitTestCmd (int argc, const char argv[][MAX_CMD_SIZE]); + int addWordSetCmd (int argc, const char argv[][MAX_CMD_SIZE]); int createWordSet (const char* name, const char* filename); }; diff --git a/packages/legacy/UT_Table.cpp b/packages/legacy/UT_Table.cpp index fd8ac7ce6..b5a8c7a64 100644 --- a/packages/legacy/UT_Table.cpp +++ b/packages/legacy/UT_Table.cpp @@ -183,7 +183,7 @@ int UT_Table::testChaining(int argc, char argv[][MAX_CMD_SIZE]) int key, data; int size = 8; Table mytable(size); - int test_data[8] = {0,1,2,3,8,9,10,11}; + const int test_data[8] = {0,1,2,3,8,9,10,11}; failures = 0; @@ -222,9 +222,9 @@ int UT_Table::testRemoving(int argc, char argv[][MAX_CMD_SIZE]) int data; int size = 16; Table mytable(size); - int test_data[16] = {0, 16, 32, 1, 17, 33, 2, 18, 34, 3, 4, 5, 6, 7, 8, 9}; - int remove_order[16] = {0, 16, 32, 17, 33, 1, 34, 18, 2, 3, 4, 5, 6, 7, 8, 9}; - int check_order[16] = {0, 16, 32, 1, 1, 1, 2, 2, 2, 3, 4, 5, 6, 7, 8, 9}; + const int test_data[16] = {0, 16, 32, 1, 17, 33, 2, 18, 34, 3, 4, 5, 6, 7, 8, 9}; + const int remove_order[16] = {0, 16, 32, 17, 33, 1, 34, 18, 2, 3, 4, 5, 6, 7, 8, 9}; + const int check_order[16] = {0, 16, 32, 1, 1, 1, 2, 2, 2, 3, 4, 5, 6, 7, 8, 9}; failures = 0; @@ -262,7 +262,7 @@ int UT_Table::testDuplicates(int argc, char argv[][MAX_CMD_SIZE]) int key; int size = 16; Table mytable(size); - int test_data[16] = {0,16,32,1,17,33,2,18,34,3,4,5,6,7,8,9}; + const int test_data[16] = {0,16,32,1,17,33,2,18,34,3,4,5,6,7,8,9}; failures = 0; @@ -322,7 +322,7 @@ int UT_Table::testFullTable(int argc, char argv[][MAX_CMD_SIZE]) int key; int size = 8; Table mytable(size); - int test_data[8] = {0,1,2,3,4,5,6,7}; + const int test_data[8] = {0,1,2,3,4,5,6,7}; failures = 0; @@ -382,9 +382,9 @@ int UT_Table::testCollisions(int argc, char argv[][MAX_CMD_SIZE]) int key, data; int size = 16; Table mytable(size); - int test_data[16] = {0,16,32, 1,17,33, 2,18,34,40,50,66,48,35, 8, 9}; - int remove_order[16] = {0,16,32,17,33, 1,34,18, 2,40,50,66,48,35, 8, 9}; - int check_order[16] = {0,16,32, 1, 1, 1, 2, 2, 2,40,50,66,48,35, 8, 9}; + const int test_data[16] = {0,16,32, 1,17,33, 2,18,34,40,50,66,48,35, 8, 9}; + const int remove_order[16] = {0,16,32,17,33, 1,34,18, 2,40,50,66,48,35, 8, 9}; + const int check_order[16] = {0,16,32, 1, 1, 1, 2, 2, 2,40,50,66,48,35, 8, 9}; failures = 0; diff --git a/packages/netsvc/EndpointProxy.cpp b/packages/netsvc/EndpointProxy.cpp index c574eb4ef..bea145d62 100644 --- a/packages/netsvc/EndpointProxy.cpp +++ b/packages/netsvc/EndpointProxy.cpp @@ -117,11 +117,14 @@ int EndpointProxy::luaCreate (lua_State* L) { mlog(e.level(), "Error creating EndpointProxy: %s", e.what()); - for(int i = 0; i < _num_resources; i++) + if(_resources) // unnecessary check, but supplied for static analysis { - delete [] _resources[i]; + for(int i = 0; i < _num_resources; i++) + { + delete [] _resources[i]; + } + delete [] _resources; } - delete [] _resources; return returnLuaStatus(L, false); } diff --git a/packages/netsvc/NetsvcParms.cpp b/packages/netsvc/NetsvcParms.cpp index d38cd4907..596d8201f 100644 --- a/packages/netsvc/NetsvcParms.cpp +++ b/packages/netsvc/NetsvcParms.cpp @@ -207,7 +207,7 @@ void NetsvcParms::cleanup (void) const void NetsvcParms::get_lua_polygon (lua_State* L, int index, bool* provided) { /* Reset Provided */ - *provided = false; + if(provided) *provided = false; /* Must be table of coordinates */ if(lua_istable(L, index)) diff --git a/packages/netsvc/ProvisioningSystemLib.cpp b/packages/netsvc/ProvisioningSystemLib.cpp index 095cc41ed..5244d33bc 100644 --- a/packages/netsvc/ProvisioningSystemLib.cpp +++ b/packages/netsvc/ProvisioningSystemLib.cpp @@ -396,7 +396,7 @@ bool ProvisioningSystemLib::Authenticator::isValid (const char* token) *----------------------------------------------------------------------------*/ size_t ProvisioningSystemLib::writeData(void *buffer, size_t size, size_t nmemb, void *userp) { - List* rsps_set = (List*)userp; + List* rsps_set = reinterpret_cast*>(userp); data_t rsps; rsps.size = size * nmemb; diff --git a/platforms/linux/OsApi.cpp b/platforms/linux/OsApi.cpp index b71ca0f03..81d8eb51c 100644 --- a/platforms/linux/OsApi.cpp +++ b/platforms/linux/OsApi.cpp @@ -107,7 +107,7 @@ void OsApi::dupstr (char** dst, const char* src) assert(dst); if(*dst) delete [] *dst; int len = 0; - while( (len < (MAX_STR_SIZE - 1)) && (src[len] != '\0') ) len++; + while(src[len] != '\0') len++; *dst = new char[len + 1]; for(int k = 0; k < len; k++) (*dst)[k] = src[k]; (*dst)[len] = '\0'; diff --git a/platforms/linux/SockLib.cpp b/platforms/linux/SockLib.cpp index 28cb5888c..21d533b06 100644 --- a/platforms/linux/SockLib.cpp +++ b/platforms/linux/SockLib.cpp @@ -373,7 +373,7 @@ int SockLib::sockinfo(int fd, char** local_ipaddr, int* local_port, char** remot /* Populate Local IP Address */ if(local_ipaddr) { - *local_ipaddr = new char[16]; + *local_ipaddr = new char[17]; strncpy(*local_ipaddr, inet_ntoa(local_addr.sin_addr), 16); (*local_ipaddr)[15] = '\0'; } @@ -387,7 +387,7 @@ int SockLib::sockinfo(int fd, char** local_ipaddr, int* local_port, char** remot /* Populate Remote IP Address */ if(remote_ipaddr) { - *remote_ipaddr = new char[16]; + *remote_ipaddr = new char[17]; strncpy(*remote_ipaddr, inet_ntoa(remote_addr.sin_addr), 16); (*remote_ipaddr)[15] = '\0'; } diff --git a/plugins/gedi/plugin/FootprintReader.h b/plugins/gedi/plugin/FootprintReader.h index e92e15989..cc4eee53e 100644 --- a/plugins/gedi/plugin/FootprintReader.h +++ b/plugins/gedi/plugin/FootprintReader.h @@ -370,7 +370,6 @@ void FootprintReader::Region::polyregion (info_t* info) { /* Find First and Last Footprints in Polygon */ bool first_footprint_found = false; - bool last_footprint_found = false; int footprint = 0; while(footprint < lat.size) { @@ -393,10 +392,8 @@ void FootprintReader::Region::polyregion (info_t* info) first_footprint_found = true; first_footprint = footprint; } - else if(first_footprint_found && !last_footprint_found && !inclusion) + else if(first_footprint_found && !inclusion) { - /* Set Last Segment */ - last_footprint_found = true; break; // full extent found! } @@ -496,7 +493,7 @@ int FootprintReader::luaStats (lua_State* L) try { /* Get Self */ - lua_obj = (FootprintReader*)getLuaSelf(L, 1); + lua_obj = reinterpret_cast(getLuaSelf(L, 1)); } catch(const RunTimeException& e) { diff --git a/plugins/icesat2/plugin/AncillaryFields.cpp b/plugins/icesat2/plugin/AncillaryFields.cpp index 05c630045..1b4cf2a84 100644 --- a/plugins/icesat2/plugin/AncillaryFields.cpp +++ b/plugins/icesat2/plugin/AncillaryFields.cpp @@ -300,7 +300,7 @@ float* AncillaryFields::getValueAsFloat (uint8_t* buffer) *----------------------------------------------------------------------------*/ RecordObject* AncillaryFields::createFieldArrayRecord (uint64_t extent_id, vector& field_vec) { - if(field_vec.size() <= 0) return NULL; + if(field_vec.empty()) return NULL; int rec_size = offsetof(field_array_t, fields) + (sizeof(field_t) * field_vec.size()); RecordObject* rec = new RecordObject(ancFieldArrayRecType, rec_size); diff --git a/plugins/icesat2/plugin/Atl08Dispatch.cpp b/plugins/icesat2/plugin/Atl08Dispatch.cpp index 727ed411a..9c2b2ed7b 100644 --- a/plugins/icesat2/plugin/Atl08Dispatch.cpp +++ b/plugins/icesat2/plugin/Atl08Dispatch.cpp @@ -683,7 +683,7 @@ void Atl08Dispatch::phorealAlgorithm (Atl03Reader::extent_t* extent, vegetation_ /*---------------------------------------------------------------------------- * postResult *----------------------------------------------------------------------------*/ -void Atl08Dispatch::postResult (vegetation_t* result, RecordObject* ancrec) +void Atl08Dispatch::postResult (const vegetation_t* result, RecordObject* ancrec) { batchMutex.lock(); { diff --git a/plugins/icesat2/plugin/Atl08Dispatch.h b/plugins/icesat2/plugin/Atl08Dispatch.h index 00bdd4c0c..4d456a037 100644 --- a/plugins/icesat2/plugin/Atl08Dispatch.h +++ b/plugins/icesat2/plugin/Atl08Dispatch.h @@ -161,7 +161,7 @@ class Atl08Dispatch: public DispatchObject RecordObject* buildAncillaryRecord (Atl03Reader::extent_t* extent, recVec_t* records); void geolocateResult (Atl03Reader::extent_t* extent, vegetation_t& result); void phorealAlgorithm (Atl03Reader::extent_t* extent, vegetation_t& result); - void postResult (vegetation_t* result, RecordObject* ancrec); + void postResult (const vegetation_t* result, RecordObject* ancrec); static void quicksort (long* index_array, Atl03Reader::photon_t* ph_array, float Atl03Reader::photon_t::*field, int start, int end); static int quicksortpartition (long* index_array, Atl03Reader::photon_t* ph_array, float Atl03Reader::photon_t::*field, int start, int end); diff --git a/plugins/pgc/plugin/PgcDemStripsRaster.cpp b/plugins/pgc/plugin/PgcDemStripsRaster.cpp index a0e18a81b..e400692f7 100644 --- a/plugins/pgc/plugin/PgcDemStripsRaster.cpp +++ b/plugins/pgc/plugin/PgcDemStripsRaster.cpp @@ -217,6 +217,7 @@ bool PgcDemStripsRaster::findRasters(const OGRGeometry* geo) else fileName.clear(); raster_info_t flagsRinfo; + flagsRinfo.dataIsElevation = false; flagsRinfo.tag = FLAGS_TAG; flagsRinfo.fileName = fileName; diff --git a/project-config.cmake b/project-config.cmake index 1d4f4278d..6bf85b51a 100644 --- a/project-config.cmake +++ b/project-config.cmake @@ -64,6 +64,9 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug") "--suppress=constParameter:*/Dictionary.h" "--suppress=unreadVariable:*/TimeLib.cpp" "--suppress=invalidPointerCast:*/H5Array.h" + "--suppress=copyCtorPointerCopying:*/MsgQ.cpp" + "--suppress=knownConditionTrueFalse:*/packages/legacy/UT_*" + "--suppress=uninitStructMember:*/plugins/icesat2/plugin/Atl06Dispatch.cpp" "--error-exitcode=1" "-DLLONG_MAX" ) diff --git a/scripts/extensions/asset.lua b/scripts/extensions/asset.lua index 0240bca37..88c7e2cbd 100644 --- a/scripts/extensions/asset.lua +++ b/scripts/extensions/asset.lua @@ -128,7 +128,7 @@ local function loaddir(file) for k,v in pairs(directory) do -- see if asset already exists - local asset = core.getbyname(k, false) + local asset = core.getbyname(k) -- populate asset table if asset then diff --git a/targets/slideruleearth-aws/Makefile b/targets/slideruleearth-aws/Makefile index 0580a85a3..c5d2c07fb 100644 --- a/targets/slideruleearth-aws/Makefile +++ b/targets/slideruleearth-aws/Makefile @@ -75,7 +75,7 @@ MYIP ?= $(shell (ip route get 1 | sed -n 's/^.*src \([0-9.]*\) .*$$/\1/p')) ENVVER = $(shell git --git-dir ../../.git --work-tree ../../.git describe --abbrev --dirty --always --tags --long) USERCFG ?= -CLANG_VER = "-12" +CLANG_VER ?= "-12" CLANG_OPT = -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- CLANG_CFG = export CC=clang$(CLANG_VER) && export CXX=clang++$(CLANG_VER) From 74236921778e0f162d816b65973eade95416ac40 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Wed, 13 Dec 2023 13:32:43 +0000 Subject: [PATCH 06/15] fixed memory leak in curl lib for headers --- packages/netsvc/CurlLib.cpp | 18 +++++++++--------- packages/netsvc/CurlLib.h | 5 +++-- .../docker/sliderule/Dockerfile.runtime | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/netsvc/CurlLib.cpp b/packages/netsvc/CurlLib.cpp index f2e524807..9e2b28f5c 100644 --- a/packages/netsvc/CurlLib.cpp +++ b/packages/netsvc/CurlLib.cpp @@ -62,7 +62,7 @@ void CurlLib::deinit (void) /*---------------------------------------------------------------------------- * request *----------------------------------------------------------------------------*/ -long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname, vector* headers) +long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname, List* headers) { long http_code = 0; CURL* curl = NULL; @@ -123,11 +123,11 @@ long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* /* Add Headers */ struct curl_slist* hdr_slist = NULL; - if(headers && headers->size() > 0) + if(headers && headers->length() > 0) { - for(const char* hdr: *headers) + for(int i = 0; i < headers->length(); i++) { - hdr_slist = curl_slist_append(hdr_slist, hdr); + hdr_slist = curl_slist_append(hdr_slist, headers->get(i)); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hdr_slist); } @@ -328,7 +328,7 @@ long CurlLib::postAsRecord (const char* url, const char* data, Publisher* outq, /*---------------------------------------------------------------------------- * getHeaders *----------------------------------------------------------------------------*/ -int CurlLib::getHeaders (lua_State* L, int index, vector& header_list) +int CurlLib::getHeaders (lua_State* L, int index, List& header_list) { int num_hdrs = 0; @@ -344,7 +344,7 @@ int CurlLib::getHeaders (lua_State* L, int index, vector& header_li if(lua_isstring(L, -1)) { const char* header = StringLib::duplicate(LuaObject::getLuaString(L, -1)); - header_list.push_back(header); + header_list.add(header); num_hdrs++; } @@ -362,7 +362,7 @@ int CurlLib::getHeaders (lua_State* L, int index, vector& header_li int CurlLib::luaGet (lua_State* L) { bool status = false; - vector header_list; + List header_list(EXPECTED_MAX_HEADERS); try { @@ -405,7 +405,7 @@ int CurlLib::luaGet (lua_State* L) int CurlLib::luaPut (lua_State* L) { bool status = false; - vector header_list; + List header_list(EXPECTED_MAX_HEADERS); try { @@ -448,7 +448,7 @@ int CurlLib::luaPut (lua_State* L) int CurlLib::luaPost (lua_State* L) { bool status = false; - vector header_list; + List header_list(EXPECTED_MAX_HEADERS); try { diff --git a/packages/netsvc/CurlLib.h b/packages/netsvc/CurlLib.h index ac16961e6..d8e8a9bce 100644 --- a/packages/netsvc/CurlLib.h +++ b/packages/netsvc/CurlLib.h @@ -56,13 +56,13 @@ class CurlLib static void init (void); static void deinit (void); - static long request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false, vector* headers=NULL); + static long request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false, List* headers=NULL); static long get (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long put (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long post (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long postAsStream (const char* url, const char* data, Publisher* outq, bool with_terminator); static long postAsRecord (const char* url, const char* data, Publisher* outq, bool with_terminator, int timeout, bool* active=NULL); - static int getHeaders (lua_State* L, int index, vector& header_list); + static int getHeaders (lua_State* L, int index, List& header_list); static int luaGet (lua_State* L); static int luaPut (lua_State* L); static int luaPost (lua_State* L); @@ -76,6 +76,7 @@ class CurlLib static const int EXPECTED_RESPONSE_SEGMENTS = 16; static const int CONNECTION_TIMEOUT = 10L; // seconds static const int DATA_TIMEOUT = 60L; // seconds + static const int EXPECTED_MAX_HEADERS = 8; static const int RECOBJ_HDR_SIZE = 8; // bytes diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime index dcbcd2cd5..991d4b1a7 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime @@ -12,7 +12,7 @@ RUN make config-release INSTALL_DIR=/usr/local && make COPY config.json /usr/local/etc/sliderule/ # install runtime dependencies -FROM ubuntu:20.04 AS runtime +FROM amazonlinux:2023 AS runtime RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get install -y --no-install-recommends \ From d9de9d4dbf516a21cabae227e3943a0e3113620c Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Wed, 13 Dec 2023 13:37:13 +0000 Subject: [PATCH 07/15] refixed memory leak in curl lib for headers by changing to use string type --- packages/netsvc/CurlLib.cpp | 14 +++++++------- packages/netsvc/CurlLib.h | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/netsvc/CurlLib.cpp b/packages/netsvc/CurlLib.cpp index 9e2b28f5c..837f4cef4 100644 --- a/packages/netsvc/CurlLib.cpp +++ b/packages/netsvc/CurlLib.cpp @@ -62,7 +62,7 @@ void CurlLib::deinit (void) /*---------------------------------------------------------------------------- * request *----------------------------------------------------------------------------*/ -long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname, List* headers) +long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer, bool verify_hostname, List* headers) { long http_code = 0; CURL* curl = NULL; @@ -127,7 +127,7 @@ long CurlLib::request (EndpointObject::verb_t verb, const char* url, const char* { for(int i = 0; i < headers->length(); i++) { - hdr_slist = curl_slist_append(hdr_slist, headers->get(i)); + hdr_slist = curl_slist_append(hdr_slist, headers->get(i)->c_str()); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hdr_slist); } @@ -328,7 +328,7 @@ long CurlLib::postAsRecord (const char* url, const char* data, Publisher* outq, /*---------------------------------------------------------------------------- * getHeaders *----------------------------------------------------------------------------*/ -int CurlLib::getHeaders (lua_State* L, int index, List& header_list) +int CurlLib::getHeaders (lua_State* L, int index, List& header_list) { int num_hdrs = 0; @@ -343,7 +343,7 @@ int CurlLib::getHeaders (lua_State* L, int index, List& header_list lua_rawgeti(L, index, i+1); if(lua_isstring(L, -1)) { - const char* header = StringLib::duplicate(LuaObject::getLuaString(L, -1)); + string* header = new string(LuaObject::getLuaString(L, -1)); header_list.add(header); num_hdrs++; } @@ -362,7 +362,7 @@ int CurlLib::getHeaders (lua_State* L, int index, List& header_list int CurlLib::luaGet (lua_State* L) { bool status = false; - List header_list(EXPECTED_MAX_HEADERS); + List header_list(EXPECTED_MAX_HEADERS); try { @@ -405,7 +405,7 @@ int CurlLib::luaGet (lua_State* L) int CurlLib::luaPut (lua_State* L) { bool status = false; - List header_list(EXPECTED_MAX_HEADERS); + List header_list(EXPECTED_MAX_HEADERS); try { @@ -448,7 +448,7 @@ int CurlLib::luaPut (lua_State* L) int CurlLib::luaPost (lua_State* L) { bool status = false; - List header_list(EXPECTED_MAX_HEADERS); + List header_list(EXPECTED_MAX_HEADERS); try { diff --git a/packages/netsvc/CurlLib.h b/packages/netsvc/CurlLib.h index d8e8a9bce..9568439d5 100644 --- a/packages/netsvc/CurlLib.h +++ b/packages/netsvc/CurlLib.h @@ -56,13 +56,13 @@ class CurlLib static void init (void); static void deinit (void); - static long request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false, List* headers=NULL); + static long request (EndpointObject::verb_t verb, const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false, List* headers=NULL); static long get (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long put (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long post (const char* url, const char* data, const char** response, int* size, bool verify_peer=false, bool verify_hostname=false); static long postAsStream (const char* url, const char* data, Publisher* outq, bool with_terminator); static long postAsRecord (const char* url, const char* data, Publisher* outq, bool with_terminator, int timeout, bool* active=NULL); - static int getHeaders (lua_State* L, int index, List& header_list); + static int getHeaders (lua_State* L, int index, List& header_list); static int luaGet (lua_State* L); static int luaPut (lua_State* L); static int luaPost (lua_State* L); From 675e3cc303938d5cabe486e3d910efa86e17e535 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Wed, 13 Dec 2023 18:28:31 +0000 Subject: [PATCH 08/15] docker image for sliderule using al2023 --- .../docker/sliderule/Dockerfile.buildenv | 37 ++++++++----------- .../docker/sliderule/Dockerfile.runtime | 21 +++++------ .../docker/sliderule/libdep.lock | 12 +++--- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv index c1b1c39d1..095d695da 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv @@ -1,30 +1,25 @@ -FROM ubuntu:20.04 +FROM amazonlinux:2023 # install build dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - pkg-config \ +RUN dnf update \ + && dnf groupinstall -y "Development Tools" \ + && dnf install -y \ cmake \ - git \ - curl \ - libreadline-dev \ - liblua5.3-dev \ - libcurl4-openssl-dev \ - libssl-dev \ - uuid-dev \ - zlib1g-dev \ - libtiff-dev \ - libsqlite3-dev \ - sqlite3 \ + readline-devel \ + lua-devel \ + openssl-devel \ + libuuid-devel \ + libtiff-devel \ + sqlite-devel \ + curl-devel \ + python-devel \ meson \ llvm \ - clang-12 \ - clang-tidy \ + clang \ + clang-tools-extra \ cppcheck \ - && rm -rf /var/lib/apt/lists/* + && dnf clean all \ + && rm -rf /var/cache/yum # install rapidjson dependency WORKDIR / diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime index 991d4b1a7..fb9ae96e9 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.runtime @@ -13,22 +13,21 @@ COPY config.json /usr/local/etc/sliderule/ # install runtime dependencies FROM amazonlinux:2023 AS runtime -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y --no-install-recommends \ - libreadline8 \ - liblua5.3 \ - libssl1.1 \ - libtiff5 \ - zlib1g \ - curl \ - && rm -rf /var/lib/apt/lists/* +RUN dnf update \ + && dnf install -y \ + readline \ + lua \ + openssl \ + libtiff \ + zlib \ + && dnf clean all \ + && rm -rf /var/cache/yum # copy sliderule installation from buildenv COPY --from=buildenv /usr/local /usr/local # configure any new shared libraries -RUN ldconfig +RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/local.conf && ldconfig # set entrypoint to bash shell (expect it to be overridden) ENTRYPOINT ["/bin/sh"] diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.lock b/targets/slideruleearth-aws/docker/sliderule/libdep.lock index 5c3db78fc..064765bfe 100644 --- a/targets/slideruleearth-aws/docker/sliderule/libdep.lock +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.lock @@ -1,8 +1,8 @@ -PDAL: 2.6.0-37-g7d3ec480e -PROJ: 9.0.0-1040-gdecf6afd -arrow: apache-arrow-15.0.0.dev-226-g081b4022f -gdal: v3.8.0RC1-531-gf014e7bda2 -geos: 3.12.0-100-g1d1ad6b21 +PDAL: 2.6.0-38-g65be31031 +PROJ: 9.0.0-1046-g657c69c6 +arrow: apache-arrow-15.0.0.dev-260-g4aa9f604d +gdal: v3.8.0RC1-587-gd26bb28615 +geos: 3.12.0-107-g0aef713ac libgeotiff: 1.7.1-44-g1f3e10b rapidjson: v1.1.0-745-g6089180e -pistache: 0.0.5-65-g31bc54d +pistache: 0.0.5-70-gea688d0 From 84c07102d3b59dcd7c66d31133a57bb507f3c4a2 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Thu, 11 Jan 2024 15:20:02 +0000 Subject: [PATCH 09/15] updated ami for al2023 --- .../docker/sliderule/Dockerfile.buildenv | 3 ++ .../packer/sliderule-base.pkr.hcl | 36 +++++++------------ 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv index 095d695da..0e408583d 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv @@ -97,6 +97,9 @@ RUN meson setup build RUN meson install -C build RUN ldconfig +# configure any new shared libraries +RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/local.conf && ldconfig + # support interactive mode WORKDIR / CMD ["/bin/bash"] \ No newline at end of file diff --git a/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl b/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl index 8fa797c8c..d9236e5af 100644 --- a/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl +++ b/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl @@ -21,12 +21,12 @@ source "amazon-ebs" "base-image" { region = var.region source_ami_filter { filters = { - name = "ubuntu/images/*ubuntu-focal-20.04-arm64-server-*" + name = "amazon/al2023-ami-2023.3.-*" root-device-type = "ebs" virtualization-type = "hvm" } most_recent = true - owners = ["099720109477"] + owners = ["137112412989"] } force_deregister = true # force_delete_snapshot = true @@ -45,33 +45,22 @@ build { provisioner "shell" { inline = [ - "sudo apt-get update", - "sudo apt-get install -y ca-certificates curl gnupg lsb-release", - "sudo mkdir -p /etc/apt/keyrings", - "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg", - "echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null", - "sudo apt-get update", - "sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin", - "sudo usermod -aG docker ubuntu", - "sudo systemctl enable docker", + "sudo dnf -y upgrade --refresh", + "sudo dnf -y install docker", + "sudo usermod -aG docker ec2-user", - "sudo apt-get update && sudo apt-get install -y zip curl", - "curl -O 'https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip'", - "unzip awscli-exe-linux-aarch64.zip", - "sudo ./aws/install", - - "wget https://github.com/docker/compose/releases/download/v2.11.2/docker-compose-linux-aarch64", + "wget https://github.com/docker/compose/releases/download/v2.23.2/docker-compose-linux-aarch64", "sudo mv docker-compose-linux-aarch64 /usr/local/bin/docker-compose", "sudo chmod +x /usr/local/bin/docker-compose", - "wget https://github.com/prometheus/node_exporter/releases/download/v1.2.2/node_exporter-1.2.2.linux-arm64.tar.gz", - "tar -xvzf node_exporter-1.2.2.linux-arm64.tar.gz", - "sudo mv node_exporter-1.2.2.linux-arm64/node_exporter /usr/local/bin/node_exporter", + "wget https://github.com/prometheus/node_exporter/releases/download/v1.7.0/node_exporter-1.7.0.linux-arm64.tar.gz", + "tar -xvzf node_exporter-1.7.0.linux-arm64.tar.gz", + "sudo mv node_exporter-1.7.0.linux-arm64/node_exporter /usr/local/bin/node_exporter", "sudo mv /tmp/node_exporter.service /etc/systemd/system/node_exporter.service", "sudo systemctl daemon-reload", "sudo systemctl enable node_exporter.service", - "wget https://github.com/grafana/loki/releases/download/v2.4.0/promtail-linux-arm64.zip", + "wget https://github.com/grafana/loki/releases/download/v2.9.3/promtail-linux-arm64.zip", "unzip promtail-linux-arm64.zip", "sudo mv promtail-linux-arm64 /usr/local/bin/promtail", "sudo mv /tmp/promtail.yml /etc/", @@ -79,12 +68,11 @@ build { "sudo systemctl daemon-reload", "sudo systemctl enable promtail.service", - "wget https://github.com/grafana/loki/releases/download/v2.4.2/logcli-linux-arm64.zip", + "wget https://github.com/grafana/loki/releases/download/v2.9.3/logcli-linux-arm64.zip", "unzip logcli-linux-arm64.zip", "sudo mv logcli-linux-arm64 /usr/local/bin/logcli", - "sudo apt-get -y autoremove", - "sudo apt-get -y clean", + "sudo dnf -y clean all" ] } } From 7a0a07e5cfc2734f740b090eb0f663f30fc5daf6 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Thu, 11 Jan 2024 18:23:49 +0000 Subject: [PATCH 10/15] fixed ami filter and username for al2023 --- targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl b/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl index d9236e5af..1d11b6652 100644 --- a/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl +++ b/targets/slideruleearth-aws/packer/sliderule-base.pkr.hcl @@ -21,7 +21,7 @@ source "amazon-ebs" "base-image" { region = var.region source_ami_filter { filters = { - name = "amazon/al2023-ami-2023.3.-*" + name = "al2023-ami-2023.3*-arm64" root-device-type = "ebs" virtualization-type = "hvm" } @@ -30,7 +30,7 @@ source "amazon-ebs" "base-image" { } force_deregister = true # force_delete_snapshot = true - ssh_username = "ubuntu" + ssh_username = "ec2-user" } # a build block invokes sources and runs provisioning steps on them. From d7973a55e405363785c8123f1708e1100fb83d94 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Thu, 11 Jan 2024 21:31:09 +0000 Subject: [PATCH 11/15] remove defaulted clang version --- targets/slideruleearth-aws/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/targets/slideruleearth-aws/Makefile b/targets/slideruleearth-aws/Makefile index c5d2c07fb..f033d41d9 100644 --- a/targets/slideruleearth-aws/Makefile +++ b/targets/slideruleearth-aws/Makefile @@ -75,7 +75,7 @@ MYIP ?= $(shell (ip route get 1 | sed -n 's/^.*src \([0-9.]*\) .*$$/\1/p')) ENVVER = $(shell git --git-dir ../../.git --work-tree ../../.git describe --abbrev --dirty --always --tags --long) USERCFG ?= -CLANG_VER ?= "-12" +CLANG_VER ?= "" CLANG_OPT = -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- CLANG_CFG = export CC=clang$(CLANG_VER) && export CXX=clang++$(CLANG_VER) From ae7f5367b9d311a3b7da6224823db913b796e859 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Thu, 11 Jan 2024 21:32:11 +0000 Subject: [PATCH 12/15] removed clang version from sigview linux makefile --- targets/sigview-linux/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/targets/sigview-linux/Makefile b/targets/sigview-linux/Makefile index 09865c994..beb299951 100644 --- a/targets/sigview-linux/Makefile +++ b/targets/sigview-linux/Makefile @@ -8,7 +8,7 @@ SLIDERULE_BUILD_DIR = $(BUILD)/sliderule SLIDERULE_STAGE_DIR = $(STAGE)/atlas ATLAS_BUILD_DIR = $(BUILD)/altas -CLANG_VER = "-12" +CLANG_VER = "" CLANG_OPT = -DCMAKE_USER_MAKE_RULES_OVERRIDE=$(ROOT)/platforms/linux/ClangOverrides.txt -D_CMAKE_TOOLCHAIN_PREFIX=llvm- CLANG_CFG = export CC=clang$(CLANG_VER) && export CXX=clang++$(CLANG_VER) From bd579526b4ba5a1da6493361875da2437be48ba1 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Fri, 12 Jan 2024 09:32:26 -0500 Subject: [PATCH 13/15] updated dockerfile for build environment to address changes in GDAL build process --- .../docker/sliderule/Dockerfile.buildenv | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv index 0e408583d..cb80f65fe 100644 --- a/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv +++ b/targets/slideruleearth-aws/docker/sliderule/Dockerfile.buildenv @@ -74,20 +74,20 @@ WORKDIR / RUN git clone https://github.com/OSGeo/gdal.git RUN mkdir -p /build/gdal WORKDIR /build/gdal -RUN cmake /gdal -DCMAKE_BUILD_TYPE=Release +RUN cmake /gdal -DCMAKE_BUILD_TYPE=Release -DBUILD_APPS=OFF -DGDAL_USE_SWIG:BOOL=OFF -DBUILD_PYTHON_BINDINGS:BOOL=OFF RUN make -j8 RUN make install RUN ldconfig # install pdal dependency -WORKDIR / -RUN git clone https://github.com/PDAL/PDAL.git -RUN mkdir -p /build/pdal -WORKDIR /build/pdal -RUN cmake /PDAL -DCMAKE_BUILD_TYPE=Release -RUN make -j8 -RUN make install -RUN ldconfig +#WORKDIR / +#RUN git clone https://github.com/PDAL/PDAL.git +#RUN mkdir -p /build/pdal +#WORKDIR /build/pdal +#RUN cmake /PDAL -DCMAKE_BUILD_TYPE=Release +#RUN make -j8 +#RUN make install +#RUN ldconfig # install pistache dependency WORKDIR / From faf0973437f1e172aa295a7896868d9bec186fd3 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Fri, 12 Jan 2024 14:41:02 +0000 Subject: [PATCH 14/15] updated buildenv dependencies --- .../docker/sliderule/libdep.lock | 19 +++++++++++++------ .../docker/sliderule/libdep.sh | 8 ++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.lock b/targets/slideruleearth-aws/docker/sliderule/libdep.lock index 064765bfe..7316bee3d 100644 --- a/targets/slideruleearth-aws/docker/sliderule/libdep.lock +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.lock @@ -1,8 +1,15 @@ -PDAL: 2.6.0-38-g65be31031 -PROJ: 9.0.0-1046-g657c69c6 -arrow: apache-arrow-15.0.0.dev-260-g4aa9f604d -gdal: v3.8.0RC1-587-gd26bb28615 -geos: 3.12.0-107-g0aef713ac +PDAL: +PROJ: 9.0.0-1085-ga50673d1 +arrow: apache-arrow-15.0.0.dev-421-g3cc04f1e8 +gdal: v3.8.0RC1-839-g41bbd3bf4e +geos: 3.12.0-111-g6f70b63a0 libgeotiff: 1.7.1-44-g1f3e10b rapidjson: v1.1.0-745-g6089180e -pistache: 0.0.5-70-gea688d0 +pistache: 0.0.5-75-g1c733a1 +arrow: apache-arrow-15.0.0.dev-421-g3cc04f1e8 +gdal: v3.8.0RC1-839-g41bbd3bf4e +geos: 3.12.0-111-g6f70b63a0 +libgeotiff: 1.7.1-44-g1f3e10b +pistache: 0.0.5-75-g1c733a1 +PROJ: 9.0.0-1085-ga50673d1 +rapidjson: v1.1.0-745-g6089180e diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.sh b/targets/slideruleearth-aws/docker/sliderule/libdep.sh index d4823c600..518ff7793 100755 --- a/targets/slideruleearth-aws/docker/sliderule/libdep.sh +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.sh @@ -1,8 +1,8 @@ -echo PDAL: $(git --git-dir /PDAL/.git --work-tree /PDAL describe --abbrev --dirty --always --tags --long) > /host/libdep.lock -echo PROJ: $(git --git-dir /PROJ/.git --work-tree /PROJ describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo arrow: $(git --git-dir /arrow/.git --work-tree /arrow describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo gdal: $(git --git-dir /gdal/.git --work-tree /gdal describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo geos: $(git --git-dir /geos/.git --work-tree /geos describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo libgeotiff: $(git --git-dir /libgeotiff/.git --work-tree /libgeotiff describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock -echo rapidjson: $(git --git-dir /rapidjson/.git --work-tree /rapidjson describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock -echo pistache: $(git --git-dir /pistache/.git --work-tree /pistache describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock \ No newline at end of file +echo pistache: $(git --git-dir /pistache/.git --work-tree /pistache describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +#echo PDAL: $(git --git-dir /PDAL/.git --work-tree /PDAL describe --abbrev --dirty --always --tags --long) > /host/libdep.lock +echo PROJ: $(git --git-dir /PROJ/.git --work-tree /PROJ describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo rapidjson: $(git --git-dir /rapidjson/.git --work-tree /rapidjson describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock \ No newline at end of file From 8b1a630ef3fca98d6e76b697d31316b7c64e307a Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Fri, 12 Jan 2024 14:43:32 +0000 Subject: [PATCH 15/15] fixed bug in libdep script --- targets/slideruleearth-aws/docker/sliderule/libdep.lock | 8 -------- targets/slideruleearth-aws/docker/sliderule/libdep.sh | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.lock b/targets/slideruleearth-aws/docker/sliderule/libdep.lock index 7316bee3d..25bc4b3a4 100644 --- a/targets/slideruleearth-aws/docker/sliderule/libdep.lock +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.lock @@ -1,11 +1,3 @@ -PDAL: -PROJ: 9.0.0-1085-ga50673d1 -arrow: apache-arrow-15.0.0.dev-421-g3cc04f1e8 -gdal: v3.8.0RC1-839-g41bbd3bf4e -geos: 3.12.0-111-g6f70b63a0 -libgeotiff: 1.7.1-44-g1f3e10b -rapidjson: v1.1.0-745-g6089180e -pistache: 0.0.5-75-g1c733a1 arrow: apache-arrow-15.0.0.dev-421-g3cc04f1e8 gdal: v3.8.0RC1-839-g41bbd3bf4e geos: 3.12.0-111-g6f70b63a0 diff --git a/targets/slideruleearth-aws/docker/sliderule/libdep.sh b/targets/slideruleearth-aws/docker/sliderule/libdep.sh index 518ff7793..b17aee6eb 100755 --- a/targets/slideruleearth-aws/docker/sliderule/libdep.sh +++ b/targets/slideruleearth-aws/docker/sliderule/libdep.sh @@ -1,8 +1,8 @@ -echo arrow: $(git --git-dir /arrow/.git --work-tree /arrow describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock +echo arrow: $(git --git-dir /arrow/.git --work-tree /arrow describe --abbrev --dirty --always --tags --long) > /host/libdep.lock echo gdal: $(git --git-dir /gdal/.git --work-tree /gdal describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo geos: $(git --git-dir /geos/.git --work-tree /geos describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo libgeotiff: $(git --git-dir /libgeotiff/.git --work-tree /libgeotiff describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo pistache: $(git --git-dir /pistache/.git --work-tree /pistache describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock -#echo PDAL: $(git --git-dir /PDAL/.git --work-tree /PDAL describe --abbrev --dirty --always --tags --long) > /host/libdep.lock +#echo PDAL: $(git --git-dir /PDAL/.git --work-tree /PDAL describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo PROJ: $(git --git-dir /PROJ/.git --work-tree /PROJ describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock echo rapidjson: $(git --git-dir /rapidjson/.git --work-tree /rapidjson describe --abbrev --dirty --always --tags --long) >> /host/libdep.lock \ No newline at end of file