Merge branch 'al2023'

SlideRuleEarth · Jan 12, 2024 · f4cbf03 · f4cbf03
2 parents 48eae07 + 8b1a630
commit f4cbf03
Show file tree

Hide file tree

Showing 56 changed files with 641 additions and 384 deletions.
diff --git a/docs/rtd/source/tutorials/developer.rst b/docs/rtd/source/tutorials/developer.rst
@@ -10,5 +10,6 @@ Developer
    developer/docker_cli_tips.md
    developer/pub_to_pypi.md
    developer/ubuntu_arm_setup.md
+   developer/amazon_linux_arm_setup.md
    developer/certbot_instructions.md
    developer/backdoor_access.md
diff --git a/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md b/docs/rtd/source/tutorials/developer/amazon_linux_arm_setup.md
@@ -0,0 +1,274 @@
+# Setting Up Amazon Linux Development Environment
+
+2023-12-11
+
+## Overview
+
+These steps setup a development environment for SlideRule.  The target platform is a Graviton3 processor running Amazon Linux 2023 in AWS.
+
+## Steps
+
+### 1. Logging in the first time
+
+```bash
+ssh -i .ssh/<mykey>.pem ec2-user@<ip address>
+sudo dnf upgrade --refresh
+```
+
+### 2. Creating Users
+
+```bash
+sudo useradd -m -s /usr/bin/bash <username>
+sudo passwd <username>
+sudo usermod -aG wheel <username>
+su - <username>
+```
+
+### 3. Setup Bash
+
+Replace the appropriate section in the .bashrc file with the contents below
+```bash
+parse_git_branch() {
+     git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/'
+}
+PS1="\n\[\033[01;32m\]\u@\h\[\033[00m\]\[\033[33m\] [\$CONDA_DEFAULT_ENV]\$(parse_git_branch):\[\033[01;34m\]\w\[\033[00m\]\n\$ "
+```
+
+Setup core dumps by adding the following to the end of the .bashrc file
+```bash
+ulimit -c unlimited
+```
+
+### 4. Managing Keys for Remote Login
+
+- create a local key on your laptop or desktop for remote use: `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`
+- your new key is located in `.ssh/id_rsa.pub`
+- to log into a remote server with this key, while on the remote server copy and paste the key into the file `~/.ssh/authorized_keys`
+
+### 5. Managing Keys for GitHub
+
+- create a key on the server, while logged into your account: `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`
+- add the key as an ssh key to your github account
+
+### 6. Install and Configure Git
+
+```bash
+sudo dnf install git
+```
+
+Create the local file `~/.gitconfig` in the user's home directory with the following contents:
+```yml
+[user]
+	name = <name>
+	email = <email>
+
+[push]
+    default = simple
+
+[diff]
+    tool = vimdiff
+
+[merge]
+    tool = vimdiff
+
+[difftool]
+    prompt = false
+
+[mergetool]
+    keepBackup = false
+
+[difftool "vimdiff"]
+    cmd = vimdiff "$LOCAL" "$REMOTE"
+
+[alias]
+    dd = difftool --dir-diff
+
+[credential "https://github.com"]
+	helper =
+	helper = !/usr/bin/gh auth git-credential
+
+[credential "https://gist.github.com"]
+	helper =
+	helper = !/usr/bin/gh auth git-credential
+```
+
+### 7. Clone Projects
+
+```bash
+mkdir meta
+cd meta
+git clone git@github.com:ICESat2-SlideRule/sliderule.git
+git clone git@github.com:ICESat2-SlideRule/sliderule-python.git
+```
+
+### 8. Installing and Configuring Docker
+
+```bash
+sudo dnf install docker
+sudo usermod -aG docker <username>
+newgrp docker
+```
+
+Then install Docker Compose plugin
+```bash
+DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
+mkdir -p $DOCKER_CONFIG/cli-plugins
+curl -SL https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-aarch64 -o $DOCKER_CONFIG/cli-plugins/docker-compose
+chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
+```
+
+### 9. Install Dependencies got Local Build
+
+```bash
+sudo dnf groupinstall "Development Tools"
+sudo dnf install \
+  cmake \
+  readline-devel \
+  lua-devel \
+  openssl-devel
+  libuuid-devel \
+  libtiff-devel \
+  sqlite-devel \
+  curl-devel \
+  python-devel \
+  meson \
+  llvm \
+  clang \
+  clang-tools-extra \
+  cppcheck
+```
+
+In an editor (e.g. vi), create a file in the `/etc/ld.so.conf.d/` directory called `local.conf` and in it put the one line below, and run `sudo ldconfig` afterwards. This allows any applications that are linked to libraries installed in `/usr/local/lib64` to be found.
+```bash
+/usr/local/lib64
+```
+
+Go through and install all of the dependencies for SlideRule/
+```bash
+# install rapidjson dependency
+git clone https://github.com/Tencent/rapidjson.git
+cd rapidjson
+mkdir build
+cd build
+cmake ..
+make -j8
+sudo make install
+sudo ldconfig
+
+# install arrow dependency
+git clone https://github.com/apache/arrow.git
+cd arrow
+mkdir build
+cd build
+cmake ../cpp -DARROW_PARQUET=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON
+make -j8
+sudo make install
+sudo ldconfig
+
+# install proj9 gdal/pdal dependency
+git clone https://github.com/OSGeo/PROJ.git
+cd PROJ
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release
+make -j8
+sudo make install
+sudo ldconfig
+
+# install geotiff gdal/pdal dependency
+git clone https://github.com/OSGeo/libgeotiff.git
+cd libgeotiff
+mkdir build
+cd build
+cmake ../libgeotiff -DCMAKE_BUILD_TYPE=Release
+make -j8
+sudo make install
+sudo ldconfig
+
+# install geos gdal dependency
+git clone https://github.com/libgeos/geos.git
+cd geos
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release
+make -j8
+sudo make install
+sudo ldconfig
+
+# install gdal dependency
+git clone https://github.com/OSGeo/gdal.git
+cd gdal
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=Release
+make -j8
+sudo make install
+sudo ldconfig
+
+# install pistache dependency
+git clone https://github.com/pistacheio/pistache.git
+cd pistache
+meson setup build
+sudo meson install -C build
+sudo ldconfig
+```
+
+### 10. Install and Configure Miniconda
+
+```bash
+wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-aarch64.sh
+chmod +x Miniconda3-py39_4.12.0-Linux-aarch64.sh
+./Miniconda3-py39_4.12.0-Linux-aarch64.sh
+conda config --set changeps1 False
+conda config --set auto_activate_base false
+```
+
+Once you have miniconda setup, you can navigate to the `sliderule-python` repository and run `conda env create -f environment.yml` to create a `sliderule_env` environment with everything you will need for the SlideRule Python client.
+
+### 11. Install GitHub Command Line Client
+
+```bash
+sudo dnf install 'dnf-command(config-manager)'
+sudo dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+sudo dnf install gh
+```
+
+### 12. Install and Configure AWS Command Line Client
+
+```bash
+sudo dnf install awscli
+```
+
+Make sure to setup an initial .aws/credentials file so that it has the sliderule profile access key and secret access key.  The credentials file will look something like:
+```
+[default]
+aws_access_key_id = _
+aws_secret_access_key = _
+aws_session_token = _
+
+[sliderule]
+aws_access_key_id = _
+aws_secret_access_key = _
+```
+
+To populate the default keys and session token, run:
+```bash
+aws --profile=sliderule sts get-session-token --serial-number arn:aws:iam::$account_number:mfa/$user_name --token-code=$code
+```
+
+To login to the AWS Elastic Container Registry, run:
+```bash
+aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account_number.dkr.ecr.$region.amazonaws.com
+```
+
+### 13. Install Terraform & Packer
+
+```bash
+wget https://releases.hashicorp.com/terraform/1.3.1/terraform_1.3.1_linux_arm64.zip
+unzip terraform_1.3.1_linux_arm64.zip
+sudo mv terraform /usr/local/bin/
+
+wget https://releases.hashicorp.com/packer/1.8.3/packer_1.8.3_linux_arm64.zip
+unzip packer_1.8.3_linux_arm64.zip
+sudo mv packer /usr/local/bin/
+```
diff --git a/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md b/docs/rtd/source/tutorials/developer/ubuntu_arm_setup.md
@@ -1,4 +1,4 @@
-# Setting Up Development Environment
+# Setting Up Ubuntu Development Environment
 
 2022-10-07
 

diff --git a/packages/aws/CredentialStore.cpp b/packages/aws/CredentialStore.cpp
@@ -107,7 +107,7 @@ CredentialStore::Credential CredentialStore::get (const char* host)
 /*----------------------------------------------------------------------------
  * put
  *----------------------------------------------------------------------------*/
-bool CredentialStore::put (const char* host, Credential& credential)
+bool CredentialStore::put (const char* host, const Credential& credential)
 {
     bool status = false;
 

diff --git a/packages/aws/CredentialStore.h b/packages/aws/CredentialStore.h
@@ -241,7 +241,7 @@ class CredentialStore
         static void         deinit  (void);
 
         static Credential   get     (const char* host);
-        static bool         put     (const char* host, Credential& credential);
+        static bool         put     (const char* host, const Credential& credential);
 
         static int          luaGet  (lua_State* L);
         static int          luaPut  (lua_State* L);

diff --git a/packages/aws/S3CurlIODriver.cpp b/packages/aws/S3CurlIODriver.cpp
@@ -109,7 +109,7 @@ static size_t curlWriteFixed(void *buffer, size_t size, size_t nmemb, void *user
  *----------------------------------------------------------------------------*/
 static size_t curlWriteStreaming(void *buffer, size_t size, size_t nmemb, void *userp)
 {
-    List<streaming_data_t>* rsps_set = (List<streaming_data_t>*)userp;
+    List<streaming_data_t>* rsps_set = reinterpret_cast<List<streaming_data_t>*>(userp);
     streaming_data_t rsps;
     rsps.size = size * nmemb;
     rsps.data = new char [rsps.size];

diff --git a/packages/ccsds/CcsdsParserAOSFrameModule.cpp b/packages/ccsds/CcsdsParserAOSFrameModule.cpp
@@ -444,7 +444,7 @@ CcsdsParserAOSFrameModule::~CcsdsParserAOSFrameModule(void)
 /*----------------------------------------------------------------------------
  * crc16
  *----------------------------------------------------------------------------*/
-uint16_t CcsdsParserAOSFrameModule::crc16(uint8_t* data, uint32_t len, uint16_t crc)
+uint16_t CcsdsParserAOSFrameModule::crc16(const uint8_t* data, uint32_t len, uint16_t crc)
 {
     static const uint16_t CrcTable[256]=
     {

diff --git a/packages/ccsds/CcsdsParserAOSFrameModule.h b/packages/ccsds/CcsdsParserAOSFrameModule.h
@@ -65,7 +65,7 @@ class CcsdsParserAOSFrameModule: public CcsdsParserModule
                     CcsdsParserAOSFrameModule   (lua_State* L, int scid, int vcid, int strip_size, uint8_t* sync_marker, int sync_size, int sync_offset, int fixed_size, int leading_size, int trailer_size);
         virtual     ~CcsdsParserAOSFrameModule  (void);
 
-        uint16_t    crc16                       (uint8_t* data, uint32_t len, uint16_t crc);
+        uint16_t    crc16                       (const uint8_t* data, uint32_t len, uint16_t crc);
 
     private:
 

diff --git a/packages/ccsds/CcsdsParserZFrameModule.cpp b/packages/ccsds/CcsdsParserZFrameModule.cpp
@@ -245,20 +245,16 @@ void CcsdsParserZFrameModule::gotoInitState(bool reset)
     }
     else
     {
-        /* Check if Frame Complete */
-        if(state == FRAME_Z)
+        /* Check if Frame is Not Complete - Stay in FANN */
+        if(frameIndex < frameSize)
         {
-            /* Check if Frame is Not Complete - Stay in FANN */
-            if(frameIndex < frameSize)
-            {
-                state = FRAME_FANN;
-            }
-            /* Check if Frame is Complete - Reset Indices */
-            else
-            {
-                frameIndex = 0;
-                frameSize = 0;
-            }
+            state = FRAME_FANN;
+        }
+        /* Check if Frame is Complete - Reset Indices */
+        else
+        {
+            frameIndex = 0;
+            frameSize = 0;
         }
     }
 }

diff --git a/packages/ccsds/CcsdsRecord.cpp b/packages/ccsds/CcsdsRecord.cpp
@@ -333,7 +333,7 @@ void CcsdsRecord::populateHeader(void)
 /*----------------------------------------------------------------------------
  * getPacketDefinition
  *----------------------------------------------------------------------------*/
-CcsdsRecord::pktDef_t* CcsdsRecord::getPacketDefinition(unsigned char* buffer, int size)
+CcsdsRecord::pktDef_t* CcsdsRecord::getPacketDefinition(const unsigned char* buffer, int size)
 {
     if(size < 6) return NULL;
 

diff --git a/packages/ccsds/CcsdsRecord.h b/packages/ccsds/CcsdsRecord.h
@@ -120,7 +120,7 @@ class CcsdsRecord: public RecordObject
 
                     CcsdsRecord         (void);
         void        populateHeader      (void);
-        pktDef_t*   getPacketDefinition (unsigned char* buffer, int size); // overloaded RecordObject method
+        pktDef_t*   getPacketDefinition (const unsigned char* buffer, int size); // overloaded RecordObject method
 };
 
 /******************************************************************************

diff --git a/packages/core/Asset.cpp b/packages/core/Asset.cpp
@@ -186,7 +186,7 @@ Asset::~Asset (void)
 /*----------------------------------------------------------------------------
  * load
  *----------------------------------------------------------------------------*/
-int Asset::load (resource_t& resource)
+int Asset::load (const resource_t& resource)
 {
     return resources.add(resource);
 }

diff --git a/packages/core/Asset.h b/packages/core/Asset.h
@@ -106,7 +106,7 @@ class Asset: public LuaObject
 
         virtual         ~Asset          (void);
 
-        int             load            (resource_t& resource);
+        int             load            (const resource_t& resource);
         resource_t&     operator[]      (int i);
 
         int             size            (void) const;