2024.01.01 release

Signed-off-by: junbo <68558268+junbo75@users.noreply.github.com>
aws · Feb 16, 2024 · 1a48363 · 1a48363
1 parent 330a587
commit 1a48363
Show file tree

Hide file tree

Showing 37 changed files with 241 additions and 77 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,13 @@
 # Change Log
 This file is used to list changes made in each release of Research and Engineering Studio (RES).
 
+2024.01.01
+------
+
+**BUG FIXES**
+
+- Upgraded CDK Lambda runtime to nodejs18.x
+
 2024.01
 ------
 
@@ -27,4 +34,4 @@ This file is used to list changes made in each release of Research and Engineeri
 - Added validation for FSx ONTAP filesystem creation
 - Narrowed installation IAM permissions
 - Skipped deletion of batteries included related resources
-- VDI no longer tries to mount filesystems after removing filesystem from project
+- VDI no longer tries to mount filesystems after removing filesystem from project
diff --git a/RES_VERSION.txt b/RES_VERSION.txt
@@ -1 +1 @@
-2024.01
+2024.01.01
diff --git a/deployment/ecr/idea-administrator/Dockerfile b/deployment/ecr/idea-administrator/Dockerfile
@@ -0,0 +1,55 @@
+FROM public.ecr.aws/docker/library/python:3.9.16-slim
+
+WORKDIR /root
+
+RUN apt-get update && \
+    apt-get -y install \
+    curl \
+    tar \
+    unzip \
+    locales \
+    && apt-get clean
+
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LC_ALL="en_US.UTF-8" \
+    LC_CTYPE="en_US.UTF-8" \
+    LANG="en_US.UTF-8"
+
+RUN sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen \
+    && locale-gen "en_US.UTF-8" \
+    && dpkg-reconfigure locales
+
+# install aws cli
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip -qq awscliv2.zip && \
+    ./aws/install && \
+    rm -rf ./aws awscliv2.zip
+
+# install nvm and node
+RUN set -uex && \
+    apt-get update && \
+    apt-get install -y ca-certificates curl gnupg && \
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+    | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+    NODE_MAJOR=18 && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" \
+    > /etc/apt/sources.list.d/nodesource.list && \
+    apt-get update && \
+	apt-get install nodejs -y
+
+# add all packaged artifacts to container
+ARG PUBLIC_ECR_TAG
+ENV PUBLIC_ECR_TAG=${PUBLIC_ECR_TAG}
+ADD all-*.tar.gz cfn_params_2_values.sh /root/.idea/downloads/
+
+# install administrator app
+RUN mkdir -p /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG} && \
+    tar -xvf /root/.idea/downloads/idea-administrator-*.tar.gz -C /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG} && \
+    /bin/bash /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG}/install.sh && \
+    rm -rf /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG}
+
+CMD ["bash"]
+
+
diff --git a/deployment/ecr/idea-administrator/cfn_params_2_values.sh b/deployment/ecr/idea-administrator/cfn_params_2_values.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+aws_partition=${1}
+aws_region=${2}
+aws_account_id=${3}
+aws_dns_suffix=${4}
+cluster_name=${5}
+administrator_email=${6}
+ssh_key_pair_name=${7}
+client_ip1=${8}
+client_ip2=${9}
+vpc_id=${10}
+pub_subnets=${11}
+pvt_subnets=${12}
+storage_home_provider=${13}
+home_fs_id=${14}
+
+values_file="/root/.idea/clusters/${5}/${2}/values.yml"
+
+prt_subnets(){
+    for sn in $(echo $1| tr ',' ' ')
+    do
+        echo "- ${sn}"
+    done
+}
+
+dir_name=$(dirname ${values_file})
+
+mkdir -p ${dir_name}
+
+rm -f ${values_file}
+cat << EOF1 > ${values_file}
+_regenerate: false
+aws_partition: ${aws_partition}
+aws_region: ${aws_region}
+aws_account_id: ${aws_account_id}
+aws_dns_suffix: ${aws_dns_suffix}
+cluster_name: ${cluster_name}
+administrator_email: ${administrator_email}
+ssh_key_pair_name: ${ssh_key_pair_name}
+client_ip:
+- ${client_ip1}
+- ${client_ip2}
+alb_public: true
+use_vpc_endpoints: true
+directory_service_provider: aws_managed_activedirectory
+enable_aws_backup: true
+kms_key_type: aws-managed
+use_existing_vpc: true
+vpc_id: ${vpc_id}
+existing_resources:
+- subnets:public
+- subnets:private
+- shared-storage:home
+public_subnet_ids:
+EOF1
+prt_subnets ${pub_subnets} >> ${values_file}
+cat << EOF2 >> ${values_file}
+private_subnet_ids:
+EOF2
+prt_subnets ${pvt_subnets} >> ${values_file}
+cat << EOF3 >> ${values_file}
+storage_home_provider: ${storage_home_provider}
+use_existing_home_fs: true
+existing_home_fs_id: ${home_fs_id}
+enabled_modules:
+- metrics
+- virtual-desktop-controller
+- bastion-host
+metrics_provider: cloudwatch
+base_os: amazonlinux2
+instance_type: m5.large
+volume_size: '200'
+EOF3
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -3,10 +3,10 @@ alembic==1.7.7
 arrow==1.2.1
 astroid==2.12.11
 attrs==21.4.0
-aws-cdk-asset-awscli-v1==2.2.52
-aws-cdk-asset-kubectl-v20==2.1.1
-aws-cdk-asset-node-proxy-agent-v5==2.0.42
-aws-cdk-lib==2.63.0
+aws-cdk-asset-awscli-v1~=2.2
+aws-cdk-asset-kubectl-v20~=2.1
+aws-cdk-asset-node-proxy-agent-v6~=2.0
+aws-cdk-lib==2.*
 banal==1.0.6
 blinker==1.4
 boto3==1.26.61
@@ -15,7 +15,7 @@ cacheout==0.13.1
 cachetools==5.1.0
 cattrs==22.1.0
 cdk-bootstrapless-synthesizer==2.3.2
-cdk-nag==2.18.17
+cdk-nag~=2.28
 certifi==2023.7.22
 cffi==1.15.0
 cfn-flip==1.3.0
@@ -43,7 +43,7 @@ ipaddress==1.0.23
 isort==5.10.1
 jinja2==3.1.2
 jmespath==1.0.0
-jsii==1.74.0
+jsii~=1.94
 lazy-object-proxy==1.7.1
 ldappool==3.0.0
 mako==1.2.4

diff --git a/requirements/idea-administrator.in b/requirements/idea-administrator.in
@@ -2,7 +2,7 @@
 colored
 ipaddress
 sanic==23.6.0
-aws-cdk-lib==2.63.0
+aws-cdk-lib==2.*
 cdk-nag
 prettytable
-defusedxml
+defusedxml
diff --git a/requirements/idea-administrator.txt b/requirements/idea-administrator.txt
@@ -2,17 +2,17 @@ aiofiles==23.2.1
 alembic==1.8.0
 arrow==1.2.1
 attrs==21.4.0
-aws-cdk-asset-awscli-v1==2.2.52
-aws-cdk-asset-kubectl-v20==2.1.1
-aws-cdk-asset-node-proxy-agent-v5==2.0.42
-aws-cdk-lib==2.63.0
+aws-cdk-asset-awscli-v1~=2.2
+aws-cdk-asset-kubectl-v20~=2.1
+aws-cdk-asset-node-proxy-agent-v6~=2.0
+aws-cdk-lib==2.*
 banal==1.0.6
 blinker==1.4
 boto3==1.26.61
 botocore==1.29.61
 cacheout==0.13.1
 cattrs==22.1.0
-cdk-nag==2.18.17
+cdk-nag~=2.28
 certifi==2023.7.22
 cffi==1.15.0
 cfn-flip==1.3.0
@@ -33,7 +33,7 @@ idna==3.3
 ipaddress==1.0.23
 jinja2==3.1.2
 jmespath==1.0.0
-jsii==1.74.0
+jsii~=1.94
 mako==1.2.4
 markupsafe==2.1.1
 multidict==6.0.2

diff --git a/software_versions.yml b/software_versions.yml
@@ -1,4 +1,4 @@
-aws_cdk_version: 2.63.0
+aws_cdk_version: 2.*
 node_version: 18.18.0
 nvm_version: 0.39.0
 python_version: 3.9.16
diff --git a/source/idea/README.md b/source/idea/README.md
@@ -2,7 +2,7 @@
 
 # Documentation
 
-https://docs.ide-on-aws.com
+https://docs.aws.amazon.com/res/latest/ug/overview.html
 
 # Directories
 
@@ -14,10 +14,6 @@ Contains the files related to the initial install of the Research and Engineerin
 
 Contains the jinja templates needed throughout the solution.
 
-## idea-cli
-
-Contains files for CLI setup.
-
 ## idea-cluster-manager
 
 Contains everything needed for the cluster manager module (server set-up, api, etc).
@@ -28,11 +24,6 @@ Defines the data models used throughout the solution.
 
 ## idea-dcv-connection-gateway
 
-
-## idea-scheduler
-
-Contains everything needing for the scheduler module (server set-up, api, etc).
-
 ## idea-sdk
 
 Contains functions and base classes that are re-used throughout Research and Engineering Studio.

diff --git a/source/idea/batteries_included/parameters/directoryservice.py b/source/idea/batteries_included/parameters/directoryservice.py
@@ -116,7 +116,7 @@ class DirectoryServiceParameters(Base):
         Attributes(
             id=DirectoryServiceKey.DOMAIN_TLS_CERTIFICATE_SECRET_ARN,
             type="String",
-            description="(Optional) Domain TLS Certificate Secret ARN",
+            description="(Optional) AD Domain TLS Certificate Secret ARN",
         )
     )
     enable_ldap_id_mapping: str = Base.parameter(

diff --git a/source/idea/idea-administrator/README.md b/source/idea/idea-administrator/README.md
@@ -2,11 +2,11 @@
 
 # Documentation
 
-https://docs.ide-on-aws.com
+https://docs.aws.amazon.com/res/latest/ug/overview.html
 
 # Directories
 
-The idea-adminsitrator folder contains various files related to the initial install of IDEA via using the CDK and the cli commands for res-admin app (cluster install and management).
+The idea-adminsitrator folder contains various files related to the initial install of IDEA via CDK.
 
 ## install
 

diff --git a/source/idea/idea-administrator/install/install.sh b/source/idea/idea-administrator/install/install.sh
@@ -14,7 +14,7 @@
 # RES Administrator Installation Script
 
 IDEA_APP_DEPLOY_DIR="/root/.idea"
-IDEA_CDK_VERSION="2.63.0"
+IDEA_CDK_VERSION="2.*"
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 

diff --git a/source/idea/idea-administrator/src/README.md b/source/idea/idea-administrator/src/README.md
@@ -2,15 +2,15 @@
 
 # Documentation
 
-https://docs.ide-on-aws.com
+https://docs.aws.amazon.com/res/latest/ug/overview.html
 
 # Directories
 
-The idea-adminsitrator folder contains various files related to the initial install of IDEA via using the CDK and the cli commands for res-admin app (cluster install and management).
+The idea-adminsitrator folder contains various files related to the initial install of IDEA via CDK (cluster install and management).
 
 ## ideaadministrator
 
-Contains the CDK and idea administrator app source files. The main entry point for the app is app_main.py which defines the CLI commands and their functions.
+Contains the CDK.
 
 ### app
 

diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/common.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/common.py
@@ -110,6 +110,10 @@ def __init__(self, context: AdministratorContext,
             vpc_subnets=vpc_subnets,
             log_retention_role=log_retention_role)
 
+        self.add_nag_suppression(suppressions=[
+            IdeaNagSuppression(rule_id='AwsSolutions-L1', reason='Lambda runtime uses Python 3.9 by default.')
+        ])
+
 
 class Policy(SocaBaseConstruct, iam.Policy):
 

diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bastion_host_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bastion_host_stack.py
@@ -183,6 +183,7 @@ def build_ec2_instance(self):
                 ec2.CfnInstance.BlockDeviceMappingProperty(
                     device_name=block_device_name,
                     ebs=ec2.CfnInstance.EbsProperty(
+                        encrypted=True,
                         volume_size=volume_size,
                         volume_type='gp3'
                     )

diff --git a/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py b/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py
@@ -12,4 +12,4 @@
 # pkg config for soca-admin. no dependencies.
 
 __name__ = 'idea-administrator'
-__version__ = '2024.01'
+__version__ = '2024.01.01'
diff --git a/source/idea/idea-cluster-manager/README.md b/source/idea/idea-cluster-manager/README.md
@@ -2,7 +2,7 @@
 
 # Documentation
 
-https://docs.ide-on-aws.com
+https://docs.aws.amazon.com/res/latest/ug/overview.html
 
 # Directories
 

diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/README.md b/source/idea/idea-cluster-manager/src/ideaclustermanager/README.md
@@ -2,7 +2,7 @@
 
 # Documentation
 
-https://docs.ide-on-aws.com
+https://docs.aws.amazon.com/res/latest/ug/overview.html
 
 # Directories
 

diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/accounts_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/accounts_service.py
@@ -735,7 +735,7 @@ def modify_user(self, user: User, email_verified: bool = False) -> User:
     def activate_user(self, existing_user: User):
         if not existing_user.is_active:
             username = existing_user.username
-            for additional_group in existing_user.additional_groups:
+            for additional_group in existing_user.additional_groups or []:
                 try:
                     self.logger.info(f'Adding username {username} to additional group: {additional_group}')
                     self.context.accounts.add_users_to_group([username], additional_group, bypass_active_user_check=True)

diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/adsync/adsync_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/adsync/adsync_service.py
@@ -123,7 +123,7 @@ def consolidate_ldap_users_by_unique_identifier(
         self, ldap_users: list, ldap_users_by_unique_identifier: dict, ldap_group_name: str = None
     ):
         for ldap_user in filter(lambda u: 'cn' in u, ldap_users):
-            cn = str(ldap_user['cn'])
+            cn = str(ldap_user['cn']).lower()
             if cn not in ldap_users_by_unique_identifier:
                 ldap_users_by_unique_identifier[cn] = ldap_user
 

diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/project_tasks.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/project_tasks.py
@@ -68,10 +68,10 @@ def invoke(self, payload: Dict):
         project_id = payload['project_id']
         project = self.context.projects.projects_dao.get_project_by_id(project_id)
         if project['enabled']:
-            groups_added = payload.get('groups_added', [])
-            groups_removed = payload.get('groups_removed', [])
-            users_added = payload.get('users_added', [])
-            users_removed = payload.get('users_removed', [])
+            groups_added = payload.get('groups_added') or []
+            groups_removed = payload.get('groups_removed') or []
+            users_added = payload.get('users_added') or []
+            users_removed = payload.get('users_removed') or []
 
             for username in users_removed:
                 self.context.projects.user_projects_dao.delete_user_project(