Fixed some AWS Settings. (#8094)

S3/Cloudfront keys were put in config.aws.keys, which did not reflect what those keys did. Moved them to config.aws.s3, where it is more obvious what they have access to. EKS/ECR keys were not in the AWS config/settings at all. Added new config child 'eks' to store them. Also, these keys were named as environment variables as 'AWS_ACCESS_KEY'/'AWS_SECRET', which did not indicate what they had access to. Renamed these references to 'EKS_AWS_ACCESS_KEY'/ 'EKS_AWS_SECRET'. Updated aws-setting schema to reflect this change. Added migration to add this column and update s3 and eks columns with this changed schema and values. Made custom dbToSchema resolvers non-async. They were returning promises instead of values, which were not being merged properly with the base settings.
EtherealEngine · Jun 20, 2023 · bdd538a · bdd538a
1 parent 081a387
commit bdd538a
Show file tree

Hide file tree

Showing 22 changed files with 161 additions and 53 deletions.
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -42,8 +42,8 @@ variables and scripts deal with AWS SDKs. If you want to deploy to GKE or anothe
 have to modify some of these variables, downloads, and calls to use another service.
 
 This action uses several other secrets to control what repo it is interacting with:
-*`AWS_ACCESS_KEY`: The public key of an IAM user that has EKS access
-*`AWS_SECRET`: The secret key of an IAM user that has EKS access
+*`EKS_AWS_ACCESS_KEY`: The public key of an IAM user that has EKS access
+*`EKS_AWS_SECRET`: The secret key of an IAM user that has EKS access
 *`AWS_REGION`: The region of the EKS cluster/ECR repo to which this built image will be pushed and deployed
 *`CLUSTER_NAME`: The name of the cluster to which this built image will be deployed
 *`DOCKER_LABEL`: In the builder service, this is the name of a Docker Hub repo that the final image
@@ -88,8 +88,8 @@ variables and scripts deal with AWS SDKs. If you want to deploy to GKE or anothe
 have to modify some of these variables, downloads, and calls to use another service.
 
 This action uses several other secrets to control what repo it is interacting with:
-*`AWS_ACCESS_KEY`: The public key of an IAM user that has EKS access
-*`AWS_SECRET`: The secret key of an IAM user that has EKS access
+*`EKS_AWS_ACCESS_KEY`: The public key of an IAM user that has EKS access
+*`EKS_AWS_SECRET`: The secret key of an IAM user that has EKS access
 *`AWS_REGION`: The region of the EKS cluster/ECR repo to which this built image will be pushed and deployed
 *`CLUSTER_NAME`: The name of the cluster to which this built image will be deployed
 *`DOCKER_LABEL`: In the builder service, this is the name of a Docker Hub repo that the final image

diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml
@@ -41,10 +41,10 @@ jobs:
       - name: Setup Helm
         run: scripts/setup_helm_builder.sh
       - name: Setup AWS
-        run: scripts/setup_aws_builder.sh $AWS_ACCESS_KEY $AWS_SECRET $AWS_REGION $CLUSTER_NAME
+        run: scripts/setup_aws_builder.sh $EKS_AWS_ACCESS_KEY $EKS_AWS_SECRET $AWS_REGION $CLUSTER_NAME
         env:
-          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
-          AWS_SECRET: ${{ secrets.AWS_SECRET }}
+          EKS_AWS_ACCESS_KEY: ${{ secrets.EKS_AWS_ACCESS_KEY }}
+          EKS_AWS_SECRET: ${{ secrets.EKS_AWS_SECRET }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }}
       - name: Space debug

diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml
@@ -40,10 +40,10 @@ jobs:
       - name: Setup Helm
         run: scripts/setup_helm_builder.sh
       - name: Setup AWS
-        run: scripts/setup_aws_builder.sh $AWS_ACCESS_KEY $AWS_SECRET $AWS_REGION $CLUSTER_NAME
+        run: scripts/setup_aws_builder.sh $EKS_AWS_ACCESS_KEY $EKS_AWS_SECRET $AWS_REGION $CLUSTER_NAME
         env:
-          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
-          AWS_SECRET: ${{ secrets.AWS_SECRET }}
+          EKS_AWS_ACCESS_KEY: ${{ secrets.EKS_AWS_ACCESS_KEY }}
+          EKS_AWS_SECRET: ${{ secrets.EKS_AWS_SECRET }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }}
       - name: Space debug

diff --git a/.github/workflows/publish-gh-container.yml b/.github/workflows/publish-gh-container.yml
@@ -26,10 +26,10 @@ jobs:
         with:
           node-version: 18.x
       - name: Setup AWS
-        run: scripts/setup_aws.sh $AWS_ACCESS_KEY $AWS_SECRET $AWS_REGION $CLUSTER_NAME
+        run: scripts/setup_aws.sh $EKS_AWS_ACCESS_KEY $EKS_AWS_SECRET $AWS_REGION $CLUSTER_NAME
         env:
-          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
-          AWS_SECRET: ${{ secrets.AWS_SECRET }}
+          EKS_AWS_ACCESS_KEY: ${{ secrets.EKS_AWS_ACCESS_KEY }}
+          EKS_AWS_SECRET: ${{ secrets.EKS_AWS_SECRET }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }}
       - name: Build Docker Image

diff --git a/.github/workflows/qat-deploy.yml b/.github/workflows/qat-deploy.yml
@@ -40,10 +40,10 @@ jobs:
       - name: Setup Helm
         run: scripts/setup_helm_builder.sh
       - name: Setup AWS
-        run: scripts/setup_aws_builder.sh $AWS_ACCESS_KEY $AWS_SECRET $AWS_REGION $CLUSTER_NAME
+        run: scripts/setup_aws_builder.sh $EKS_AWS_ACCESS_KEY $EKS_AWS_SECRET $AWS_REGION $CLUSTER_NAME
         env:
-          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
-          AWS_SECRET: ${{ secrets.AWS_SECRET }}
+          EKS_AWS_ACCESS_KEY: ${{ secrets.EKS_AWS_ACCESS_KEY }}
+          EKS_AWS_SECRET: ${{ secrets.EKS_AWS_SECRET }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }}
       - name: Space debug

diff --git a/packages/client-core/i18n/en/admin.json b/packages/client-core/i18n/en/admin.json
@@ -263,6 +263,7 @@
       "keys": "Keys",
       "accessKeyId": "Access Key ID",
       "secretAccessKey": "Secret Access Key",
+      "eks": "EKS",
       "route53": "Route53",
       "hostedZoneId": "Hosted Zone ID",
       "s3": "S3",

diff --git a/packages/client-core/src/admin/components/Project/BuildStatusDrawer.tsx b/packages/client-core/src/admin/components/Project/BuildStatusDrawer.tsx
@@ -61,7 +61,7 @@ const BuildStatusDrawer = ({ open, onClose }: Props) => {
   const { t } = useTranslation()
   const page = useHookstate(0)
   const rowsPerPage = useHookstate(10)
-  const selectedStatusId = useHookstate('')
+  const selectedStatusId = useHookstate(0)
   const logsModalOpen = useHookstate(false)
 
   const fieldOrder = useHookstate('desc')
@@ -81,7 +81,7 @@ const BuildStatusDrawer = ({ open, onClose }: Props) => {
 
   const handleCloseLogsModal = () => {
     logsModalOpen.set(false)
-    selectedStatusId.set('')
+    selectedStatusId.set(0)
   }
   const createData = (el: BuildStatusType) => {
     return {

diff --git a/packages/client-core/src/admin/components/Setting/Aws.tsx b/packages/client-core/src/admin/components/Setting/Aws.tsx
@@ -111,24 +111,38 @@ const Aws = () => {
       </Typography>
       <Grid container spacing={3}>
         <Grid item xs={12} sm={6} md={6}>
-          <Typography className={styles.settingsSubHeading}>{t('admin:components.setting.keys')}</Typography>
+          <Typography className={styles.settingsSubHeading}>{t('admin:components.setting.eks')}</Typography>
 
           <InputText
             name="accessKeyId"
             label={t('admin:components.setting.accessKeyId')}
-            value={awsSetting?.keys?.accessKeyId || ''}
+            value={awsSetting?.eks?.accessKeyId || ''}
             disabled
           />
 
           <InputText
             name="secretAccessKey"
             label={t('admin:components.setting.secretAccessKey')}
-            value={awsSetting?.keys?.secretAccessKey || ''}
+            value={awsSetting?.eks?.secretAccessKey || ''}
             disabled
           />
 
           <Typography className={styles.settingsSubHeading}>{t('admin:components.setting.s3')}</Typography>
 
+          <InputText
+            name="accessKeyId"
+            label={t('admin:components.setting.accessKeyId')}
+            value={awsSetting?.s3?.accessKeyId || ''}
+            disabled
+          />
+
+          <InputText
+            name="secretAccessKey"
+            label={t('admin:components.setting.secretAccessKey')}
+            value={awsSetting?.s3?.secretAccessKey || ''}
+            disabled
+          />
+
           <InputText
             name="endpoint"
             label={t('admin:components.setting.endpoint')}

diff --git a/packages/engine/src/schemas/setting/aws-setting.schema.ts b/packages/engine/src/schemas/setting/aws-setting.schema.ts
@@ -49,13 +49,24 @@ export const awsRoute53Schema = Type.Object(
 )
 export type AwsRoute53Type = Static<typeof awsRoute53Schema>
 
+export const awsEksSchema = Type.Object(
+  {
+    accessKeyId: Type.String(),
+    secretAccessKey: Type.String()
+  },
+  { $id: 'AwsEks', additionalProperties: false }
+)
+export type AwsEksType = Static<typeof awsEksSchema>
+
 export const awsS3Schema = Type.Object(
   {
+    accessKeyId: Type.String(),
     endpoint: Type.String(),
     staticResourceBucket: Type.String(),
     region: Type.String(),
     avatarDir: Type.String(),
-    s3DevMode: Type.String()
+    s3DevMode: Type.String(),
+    secretAccessKey: Type.String()
   },
   { $id: 'AwsS3', additionalProperties: false }
 )
@@ -89,7 +100,7 @@ export const awsSettingSchema = Type.Object(
     id: Type.String({
       format: 'uuid'
     }),
-    keys: Type.Ref(awsKeysSchema),
+    eks: Type.Ref(awsEksSchema),
     route53: Type.Ref(awsRoute53Schema),
     s3: Type.Ref(awsS3Schema),
     cloudfront: Type.Ref(awsCloudFrontSchema),
@@ -101,16 +112,16 @@ export const awsSettingSchema = Type.Object(
 )
 export type AwsSettingType = Static<typeof awsSettingSchema>
 
-export type AwsSettingDatabaseType = Omit<AwsSettingType, 'keys' | 'route53' | 's3' | 'cloudfront' | 'sms'> & {
-  keys: string
+export type AwsSettingDatabaseType = Omit<AwsSettingType, 'eks' | 'route53' | 's3' | 'cloudfront' | 'sms'> & {
+  eks: string
   route53: string
   s3: string
   cloudfront: string
   sms: string
 }
 
 // Schema for creating new entries
-export const awsSettingDataSchema = Type.Pick(awsSettingSchema, ['keys', 'route53', 's3', 'cloudfront', 'sms'], {
+export const awsSettingDataSchema = Type.Pick(awsSettingSchema, ['eks', 'route53', 's3', 'cloudfront', 'sms'], {
   $id: 'AwsSettingData'
 })
 export type AwsSettingData = Static<typeof awsSettingDataSchema>

diff --git a/packages/engine/src/schemas/setting/email-setting.schema.ts b/packages/engine/src/schemas/setting/email-setting.schema.ts
@@ -57,7 +57,7 @@ export type EmailAuthType = Static<typeof emailAuthSchema>
 export const emailSmtpSchema = Type.Object(
   {
     host: Type.String(),
-    port: Type.String(),
+    port: Type.Number(),
     secure: Type.Boolean(),
     auth: Type.Ref(emailAuthSchema)
   },

diff --git a/packages/server-core/src/appconfig.ts b/packages/server-core/src/appconfig.ts
@@ -308,10 +308,6 @@ const authentication = {
  * AWS
  */
 const aws = {
-  keys: {
-    accessKeyId: process.env.STORAGE_AWS_ACCESS_KEY_ID!,
-    secretAccessKey: process.env.STORAGE_AWS_ACCESS_KEY_SECRET!
-  },
   route53: {
     hostedZoneId: process.env.ROUTE53_HOSTED_ZONE_ID!,
     keys: {
@@ -320,6 +316,8 @@ const aws = {
     }
   },
   s3: {
+    accessKeyId: process.env.STORAGE_AWS_ACCESS_KEY_ID!,
+    secretAccessKey: process.env.STORAGE_AWS_ACCESS_KEY_SECRET!,
     endpoint: process.env.STORAGE_S3_ENDPOINT!,
     staticResourceBucket: process.env.STORAGE_S3_STATIC_RESOURCE_BUCKET!,
     region: process.env.STORAGE_S3_REGION!,
@@ -331,6 +329,10 @@ const aws = {
     distributionId: process.env.STORAGE_CLOUDFRONT_DISTRIBUTION_ID!,
     region: process.env.STORAGE_CLOUDFRONT_REGION || process.env.STORAGE_S3_REGION
   },
+  eks: {
+    accessKeyId: process.env.EKS_AWS_ACCESS_KEY!,
+    secretAccessKey: process.env.EKS_AWS_SECRET!
+  },
   sms: {
     accessKeyId: process.env.AWS_SMS_ACCESS_KEY_ID!,
     applicationId: process.env.AWS_SMS_APPLICATION_ID!,

diff --git a/packages/server-core/src/hooks/convert-video.ts b/packages/server-core/src/hooks/convert-video.ts
@@ -59,7 +59,7 @@ export {}
 // const dashManifestName = 'manifest.mpd';
 // const createStaticResourceHook = createStaticResource();
 //
-// const s3 = new AWS.S3({ ...config.aws.keys });
+// const s3 = new AWS.S3({ ...config.aws.s3 });
 //
 // const s3BlobStore = new S3BlobStore({
 //   client: s3,

diff --git a/packages/server-core/src/media/storageprovider/s3.storage.ts b/packages/server-core/src/media/storageprovider/s3.storage.ts
@@ -72,8 +72,8 @@ export class S3Provider implements StorageProviderInterface {
    */
   provider: S3Client = new S3Client({
     credentials: {
-      accessKeyId: config.aws.keys.accessKeyId,
-      secretAccessKey: config.aws.keys.secretAccessKey
+      accessKeyId: config.aws.s3.accessKeyId,
+      secretAccessKey: config.aws.s3.secretAccessKey
     },
     endpoint: config.server.storageProviderExternalEndpoint
       ? config.server.storageProviderExternalEndpoint
@@ -110,8 +110,8 @@ export class S3Provider implements StorageProviderInterface {
   private cloudfront: CloudFrontClient = new CloudFrontClient({
     region: config.aws.cloudfront.region,
     credentials: {
-      accessKeyId: config.aws.keys.accessKeyId,
-      secretAccessKey: config.aws.keys.secretAccessKey
+      accessKeyId: config.aws.s3.accessKeyId,
+      secretAccessKey: config.aws.s3.secretAccessKey
     }
   })
 

diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts
@@ -697,8 +697,8 @@ export const findBuilderTags = async (): Promise<Array<BuilderTag>> => {
   if (publicECRExec) {
     const ecr = new ECRPUBLICClient({
       credentials: {
-        accessKeyId: process.env.AWS_ACCESS_KEY as string, //FIXME Replace these with proper EKS user credentials from config once it stores those credentials somewhere
-        secretAccessKey: process.env.AWS_SECRET as string
+        accessKeyId: config.aws.eks.accessKeyId,
+        secretAccessKey: config.aws.eks.secretAccessKey
       },
       region: 'us-east-1'
     })
@@ -726,8 +726,8 @@ export const findBuilderTags = async (): Promise<Array<BuilderTag>> => {
   } else if (privateECRExec) {
     const ecr = new ECRClient({
       credentials: {
-        accessKeyId: process.env.AWS_ACCESS_KEY as string, //FIXME Replace these with proper EKS user credentials from config once it stores those credentials somewhere
-        secretAccessKey: process.env.AWS_SECRET as string
+        accessKeyId: config.aws.eks.accessKeyId,
+        secretAccessKey: config.aws.eks.secretAccessKey
       },
       region: privateECRExec[1]
     })

diff --git a/packages/server-core/src/setting/aws-setting/aws-setting.hooks.ts b/packages/server-core/src/setting/aws-setting/aws-setting.hooks.ts
@@ -29,6 +29,7 @@ import { iff, isProvider } from 'feathers-hooks-common'
 
 import {
   awsCloudFrontSchema,
+  awsEksSchema,
   awsKeysSchema,
   awsRoute53Schema,
   awsS3Schema,
@@ -53,6 +54,8 @@ import {
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const awsKeysValidator = getValidator(awsKeysSchema, dataValidator)
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
+const awsEksValidator = getValidator(awsEksSchema, dataValidator)
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const awsRoute53Validator = getValidator(awsRoute53Schema, dataValidator)
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 const awsS3Validator = getValidator(awsS3Schema, dataValidator)

diff --git a/packages/server-core/src/setting/aws-setting/aws-setting.resolvers.ts b/packages/server-core/src/setting/aws-setting/aws-setting.resolvers.ts
@@ -29,6 +29,7 @@ import { v4 } from 'uuid'
 
 import {
   AwsCloudFrontType,
+  AwsEksType,
   AwsKeysType,
   AwsRoute53Type,
   AwsS3Type,
@@ -43,13 +44,13 @@ import { getDateTimeSql } from '../../util/get-datetime-sql'
 
 export const awsSettingResolver = resolve<AwsSettingType, HookContext>({})
 
-export const awsDbToSchema = async (rawData: AwsSettingDatabaseType): Promise<AwsSettingType> => {
-  let keys = JSON.parse(rawData.keys) as AwsKeysType
+export const awsDbToSchema = (rawData: AwsSettingDatabaseType): AwsSettingType => {
+  let eks = JSON.parse(rawData.eks || '{}') as AwsEksType
 
   // Usually above JSON.parse should be enough. But since our pre-feathers 5 data
   // was serialized multiple times, therefore we need to parse it twice.
-  if (typeof keys === 'string') {
-    keys = JSON.parse(keys)
+  if (typeof eks === 'string') {
+    eks = JSON.parse(eks)
   }
 
   let route53 = JSON.parse(rawData.route53) as AwsRoute53Type
@@ -91,7 +92,7 @@ export const awsDbToSchema = async (rawData: AwsSettingDatabaseType): Promise<Aw
 
   return {
     ...rawData,
-    keys,
+    eks,
     route53,
     s3,
     cloudfront,

diff --git a/packages/server-core/src/setting/aws-setting/aws-setting.seed.ts b/packages/server-core/src/setting/aws-setting/aws-setting.seed.ts
@@ -38,10 +38,6 @@ export async function seed(knex: Knex): Promise<void> {
   const seedData: AwsSettingDatabaseType[] = await Promise.all(
     [
       {
-        keys: JSON.stringify({
-          accessKeyId: process.env.STORAGE_AWS_ACCESS_KEY_ID,
-          secretAccessKey: process.env.STORAGE_AWS_ACCESS_KEY_SECRET
-        }),
         route53: JSON.stringify({
           hostedZoneId: process.env.ROUTE53_HOSTED_ZONE_ID,
           keys: {
@@ -50,12 +46,18 @@ export async function seed(knex: Knex): Promise<void> {
           }
         }),
         s3: JSON.stringify({
+          accessKeyId: process.env.STORAGE_AWS_ACCESS_KEY_ID,
+          secretAccessKey: process.env.STORAGE_AWS_ACCESS_KEY_SECRET,
           endpoint: process.env.STORAGE_S3_ENDPOINT,
           staticResourceBucket: process.env.STORAGE_S3_STATIC_RESOURCE_BUCKET,
           region: process.env.STORAGE_S3_REGION,
           avatarDir: process.env.STORAGE_S3_AVATAR_DIRECTORY,
           s3DevMode: process.env.STORAGE_S3_DEV_MODE
         }),
+        eks: JSON.stringify({
+          accessKeyId: process.env.EKS_AWS_ACCESS_KEY_ID,
+          secretAccessKey: process.env.EKS_AWS_ACCESS_KEY_SECRET
+        }),
         cloudfront: JSON.stringify({
           domain: process.env.STORAGE_CLOUDFRONT_DOMAIN,
           distributionId: process.env.STORAGE_CLOUDFRONT_DISTRIBUTION_ID,