From 60efdd9068d13cb6eef3094f02d7edc59eaa50a5 Mon Sep 17 00:00:00 2001 From: Antonis Kouzoupis Date: Wed, 16 Dec 2020 09:41:18 +0100 Subject: [PATCH] [instance_profile_permissions] Reorganize instance profile section and add permissions for CloudWatch (#197) --- docs/hopsworksai/aws/getting_started.md | 31 ++++++------ .../aws/instance_profile_permissions.md | 47 +++++++++++++++++++ .../aws/restrictive_permissions.md | 4 ++ docs/hopsworksai/aws/s3_permissions.md | 26 ---------- 4 files changed, 67 insertions(+), 41 deletions(-) create mode 100644 docs/hopsworksai/aws/instance_profile_permissions.md delete mode 100644 docs/hopsworksai/aws/s3_permissions.md diff --git a/docs/hopsworksai/aws/getting_started.md b/docs/hopsworksai/aws/getting_started.md index c77850a9b0..926e3a6146 100644 --- a/docs/hopsworksai/aws/getting_started.md +++ b/docs/hopsworksai/aws/getting_started.md @@ -212,12 +212,20 @@ Paste the *Access Key ID* and the *Secret Access Key* into Hopsworks.ai and clic

-## Step 2: Creating and configuring a storage +## Step 2: Creating Instance profile + +Hopsworks cluster nodes need access to certain resources such as S3 bucket and CloudWatch. + +Follow the instructions in this guide to create an IAM instance profile with access to your S3 bucket: [Guide](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html) + +When creating the policy, paste the following in the JSON tab. +{!hopsworksai/aws/instance_profile_permissions.md!} + +## Step 3: Creating storage The Hopsworks clusters deployed by hopsworks.ai store their data in an S3 bucket in your AWS account. To enable this you need to create an S3 bucket and an instance profile to give cluster nodes access to the bucket. -### Step 2.1: Creating an S3 bucket Proceed to the [S3 Management Console](https://s3.console.aws.amazon.com/s3/home) and click on *Create bucket*:

@@ -239,17 +247,10 @@ Name your bucket and select the region where your Hopsworks cluster will run. Cl

-### Step 2.2: Creating an instance profile and giving it access to the bucket - -Follow the instructions in this guide to create an IAM instance profile with access to your S3 bucket: [Guide](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html) - -When creating the policy, paste the following in the JSON tab. -{!hopsworksai/aws/s3_permissions.md!} - -## Step 3: Create an SSH key +## Step 4: Create an SSH key When deploying clusters, Hopsworks.ai installs an ssh key on the cluster's instances so that you can access them if necessary. For this purpose, you need to add an ssh key to your AWS EC2 environment. This can be done in two ways: [creating a new key pair](#step-31-create-a-new-key-pair) or [importing an existing key pair](#step-32-import-a-key-pair). -### Step 3.1: Create a new key pair +### Step 4.1: Create a new key pair Proceed to [Key pairs in the EC2 console](https://us-east-2.console.aws.amazon.com/ec2/v2/home?#KeyPairs) and click on *Create key pair*

@@ -271,7 +272,7 @@ Name your key, select the file format you prefer and click on *Create key pair*.

-### Step 3.2: Import a key pair +### Step 4.2: Import a key pair Proceed to [Key pairs in the EC2 console](https://us-east-2.console.aws.amazon.com/ec2/v2/home?#KeyPairs), click on *Action* and click on *Import key pair*

@@ -292,7 +293,7 @@ Name your key pair, upload your public key and click on *Import key pair*.

-## Step 4: Deploying a Hopsworks cluster +## Step 5: Deploying a Hopsworks cluster In Hopsworks.ai, select *Create cluster*: @@ -387,7 +388,7 @@ As soon as the cluster has started, you will be able to log in to your new Hopsw

-## Step 5: Outside Access to the Feature Store +## Step 6: Outside Access to the Feature Store By default, only the Hopsworks UI is made available to clients on external networks, like the Internet. To integrate with external platforms and access APIs for services such as the Feature Store, you have to open the service's ports. @@ -403,7 +404,7 @@ Open ports by going to the *Services* tab, selecting a service, and pressing *Up

-## Step 6: Next steps +## Step 7: Next steps Check out our other guides for how to get started with Hopsworks and the Feature Store: diff --git a/docs/hopsworksai/aws/instance_profile_permissions.md b/docs/hopsworksai/aws/instance_profile_permissions.md new file mode 100644 index 0000000000..8ebb3334e5 --- /dev/null +++ b/docs/hopsworksai/aws/instance_profile_permissions.md @@ -0,0 +1,47 @@ + +Replace *BUCKET_NAME* with appropriate S3 bucket name. +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "hopsworksaiInstanceProfile", + "Effect": "Allow", + "Action": [ + "S3:PutObject", + "S3:ListBucket", + "S3:GetBucketLocation", + "S3:GetObject", + "S3:DeleteObject", + "S3:AbortMultipartUpload", + "S3:ListBucketMultipartUploads" + ], + "Resource": [ + "arn:aws:s3:::BUCKET_NAME/*", + "arn:aws:s3:::BUCKET_NAME" + ] + }, + { + "Effect": "Allow", + "Action": [ + "cloudwatch:PutMetricData", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + "logs:DescribeLogGroups", + "logs:CreateLogStream", + "logs:CreateLogGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ssm:GetParameter" + ], + "Resource": "arn:aws:ssm:*:*:parameter/AmazonCloudWatch-*" + } + ] +} +``` diff --git a/docs/hopsworksai/aws/restrictive_permissions.md b/docs/hopsworksai/aws/restrictive_permissions.md index 0ea08550a5..50de46511a 100644 --- a/docs/hopsworksai/aws/restrictive_permissions.md +++ b/docs/hopsworksai/aws/restrictive_permissions.md @@ -32,6 +32,10 @@ Follow this guide to create a role to be used by EC2 with no permissions attache [Creating a Role for an AWS Service (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-service.html). Take note of the ARN of the role you just created. +For the suggested permissions check [our guide here](../getting_started/#step-2-creating-instance-profile). **Required** permissions are +only those which give access to the **S3 bucket**. If you select only the required permissions, then enabling `Collect logs to CloudWatch` feature +will not have any effect. + ## Step 3: Set permissions of the cross-account role During the account setup for Hopsworks.ai, you were asked to create and provide a cross-account role. diff --git a/docs/hopsworksai/aws/s3_permissions.md b/docs/hopsworksai/aws/s3_permissions.md deleted file mode 100644 index bdd6a3e3f0..0000000000 --- a/docs/hopsworksai/aws/s3_permissions.md +++ /dev/null @@ -1,26 +0,0 @@ - -Replace *BUCKET_NAME* with appropriate S3 bucket name. -```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "HopsFSS3Permissions", - "Effect": "Allow", - "Action": [ - "S3:PutObject", - "S3:ListBucket", - "S3:GetBucketLocation", - "S3:GetObject", - "S3:DeleteObject", - "S3:AbortMultipartUpload", - "S3:ListBucketMultipartUploads" - ], - "Resource": [ - "arn:aws:s3:::BUCKET_NAME/*", - "arn:aws:s3:::BUCKET_NAME" - ] - } - ] -} -```