From 4e4e64bfa777f74ff29c88509e155899decf9b1b Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Sat, 31 Jul 2021 20:25:11 -0700 Subject: [PATCH 1/2] Document permissions for AWS (DynamoDB & Redshift) Signed-off-by: Tsotne Tabidze --- docs/reference/offline-stores/redshift.md | 163 ++++++++++++++++++ docs/reference/online-stores/dynamodb.md | 60 +++++++ .../providers/amazon-web-services.md | 90 ---------- 3 files changed, 223 insertions(+), 90 deletions(-) diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md index 029bd2b4f0..5ac1bba96e 100644 --- a/docs/reference/offline-stores/redshift.md +++ b/docs/reference/offline-stores/redshift.md @@ -29,3 +29,166 @@ offline_store: Configuration options are available [here](https://github.com/feast-dev/feast/blob/bf557bcb72c7878a16dccb48443bbbe9dc3efa49/sdk/python/feast/infra/offline_stores/redshift.py#L22). +### Permissions + +Feast requires the following permissions in order to execute commands for Redshift offline store: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CommandPermissionsResources
Apply +

redshift-data:DescribeTable

+

redshift:GetClusterCredentials

+ 		+

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

+

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

+

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

+
Materialize +

redshift-data:ExecuteStatement

+ 		+

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

+
+

redshift-data:DescribeStatement

+ 		*
+

s3:ListBucket

+

s3:GetObject

+

s3:DeleteObject

+ 		+

arn:aws:s3:::<bucket_name>

+

arn:aws:s3:::<bucket_name>/*

+
Get Historical Features +

redshift-data:ExecuteStatement

+

redshift:GetClusterCredentials

+ 		+

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

+

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

+

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

+
+

redshift-data:DescribeStatement

+ 		*
+

s3:ListBucket

+

s3:GetObject

+

s3:PutObject

+

s3:DeleteObject

+ 		+

arn:aws:s3:::<bucket_name>

+

arn:aws:s3:::<bucket_name>/*

+
+ +The following inline policy can be used to grant Feast the necessary permissions: + +```json +{ + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::/*", + "arn:aws:s3:::" + ] + }, + { + "Action": [ + "redshift-data:DescribeTable", + "redshift:GetClusterCredentials", + "redshift-data:ExecuteStatement" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:redshift:::dbuser:/", + "arn:aws:redshift:::dbname:/", + "arn:aws:redshift:::cluster:" + ] + }, + { + "Action": [ + "redshift-data:DescribeStatement" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} +``` + +In addition to this, Redshift offline store requires an IAM role that will be used by Redshift itself to interact with S3. More concretely, Redshift has to use this IAM role to run UNLOAD and COPY commands. Once created, this IAM role needs to be configured in `feature_store.yaml` file as `offline_store: iam_role`. + +The following inline policy can be used to grant Redshift necessary permissions to access S3: + +```json +{ + "Statement": [ + { + "Action": "s3:*", + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::feast-integration-tests", + "arn:aws:s3:::feast-integration-tests/*" + ] + } + ], + "Version": "2012-10-17" +} +``` + +While the following trust relationship is necessary to make sure that Redshift, and only Redshift can assume this role: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "redshift.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` \ No newline at end of file diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index 21e3e586c6..984762c791 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -25,3 +25,63 @@ online_store: {% endcode %} Configuration options are available [here](https://github.com/feast-dev/feast/blob/17bfa6118d6658d2bff53d7de8e2ccef5681714d/sdk/python/feast/infra/online_stores/dynamodb.py#L36). + +### Permissions + +Feast requires the following permissions in order to execute commands for DynamoDB online store: + + + + + + + + + + + + + + + + + + + + + + + + + + +
CommandPermissionsResources
Apply +

dynamodb:CreateTable

+

dynamodb:DescribeTable

+

dynamodb:DeleteTable

+ 		arn:aws:dynamodb:<region>:<account_id>:table/*
Materialize +

dynamodb.BatchWriteItem

+ 		arn:aws:dynamodb:<region>:<account_id>:table/*
Get Online Featuresdynamodb.GetItemarn:aws:dynamodb:<region>:<account_id>:table/*
+ +The following inline policy can be used to grant Feast the necessary permissions: + +```json +{ + "Statement": [ + { + "Action": [ + "dynamodb:CreateTable", + "dynamodb:DescribeTable", + "dynamodb:DeleteTable", + "dynamodb:BatchWriteItem", + "dynamodb:GetItem" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:dynamodb:::table/*" + ] + } + ], + "Version": "2012-10-17" +} +``` \ No newline at end of file diff --git a/docs/reference/providers/amazon-web-services.md b/docs/reference/providers/amazon-web-services.md index 026cd581f9..677bc04a55 100644 --- a/docs/reference/providers/amazon-web-services.md +++ b/docs/reference/providers/amazon-web-services.md @@ -25,93 +25,3 @@ offline_store: iam_role: arn:aws:iam::123456789012:role/redshift_s3_access_role ``` {% endcode %} - - From b7d8baae3fa8f07e7fd0de55b696ba754f7dd21a Mon Sep 17 00:00:00 2001 From: Tsotne Tabidze Date: Sat, 31 Jul 2021 20:36:12 -0700 Subject: [PATCH 2/2] Add dynamodb & redshift pages in parent pages Signed-off-by: Tsotne Tabidze --- .../getting-started/create-a-feature-repository.md | 14 +++++++------- docs/reference/data-sources/README.md | 1 + docs/reference/offline-stores/README.md | 1 + docs/reference/online-stores/README.md | 1 + docs/reference/online-stores/dynamodb.md | 4 +++- docs/reference/providers/README.md | 1 + 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/getting-started/create-a-feature-repository.md b/docs/getting-started/create-a-feature-repository.md index 9c50b74c41..4047062d0a 100644 --- a/docs/getting-started/create-a-feature-repository.md +++ b/docs/getting-started/create-a-feature-repository.md @@ -24,13 +24,13 @@ Creating a new Feast repository in /<...>/tiny_pika. {% tab title="AWS template" %} ```text feast init -t aws -[?] AWS Region (e.g. us-west-2): ... -[?] Redshift Cluster ID: ... -[?] Redshift Database Name: ... -[?] Redshift User Name: ... -[?] Redshift S3 Staging Location (s3://*): ... -[?] Redshift IAM Role for S3 (arn:aws:iam::*:role/*): ... -[?] Should I upload example data to Redshift (overwriting 'feast_driver_hourly_stats' table)? (Y/n): +AWS Region (e.g. us-west-2): ... +Redshift Cluster ID: ... +Redshift Database Name: ... +Redshift User Name: ... +Redshift S3 Staging Location (s3://*): ... +Redshift IAM Role for S3 (arn:aws:iam::*:role/*): ... +Should I upload example data to Redshift (overwriting 'feast_driver_hourly_stats' table)? (Y/n): Creating a new Feast repository in /<...>/tiny_pika. ``` diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index 8435391241..2a130b2943 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -6,3 +6,4 @@ Please see [Data Source](../../concepts/feature-view.md#data-source) for an expl {% page-ref page="bigquery.md" %} +{% page-ref page="redshift.md" %} diff --git a/docs/reference/offline-stores/README.md b/docs/reference/offline-stores/README.md index 5bf7639504..7cd2ee4a47 100644 --- a/docs/reference/offline-stores/README.md +++ b/docs/reference/offline-stores/README.md @@ -6,3 +6,4 @@ Please see [Offline Store](../../concepts/offline-store.md) for an explanation o {% page-ref page="bigquery.md" %} +{% page-ref page="redshift.md" %} diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 0a240ffb10..dd7275aa5e 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -8,3 +8,4 @@ Please see [Online Store](../../concepts/online-store.md) for an explanation of {% page-ref page="datastore.md" %} +{% page-ref page="dynamodb.md" %} diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index 984762c791..e3830963df 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -84,4 +84,6 @@ The following inline policy can be used to grant Feast the necessary permissions ], "Version": "2012-10-17" } -``` \ No newline at end of file +``` + +Lastly, this IAM role needs to be associated with the desired Redshift cluster. Please follow the official AWS guide for the necessary steps [here](https://docs.aws.amazon.com/redshift/latest/dg/c-getting-started-using-spectrum-add-role.html). \ No newline at end of file diff --git a/docs/reference/providers/README.md b/docs/reference/providers/README.md index ffc1c9805f..0a8aa8cc5a 100644 --- a/docs/reference/providers/README.md +++ b/docs/reference/providers/README.md @@ -6,3 +6,4 @@ Please see [Provider](../../concepts/provider.md) for an explanation of provider {% page-ref page="google-cloud-platform.md" %} +{% page-ref page="amazon-web-services.md" %}