-
Notifications
You must be signed in to change notification settings - Fork 0
/
serverless.yaml
121 lines (114 loc) · 3.13 KB
/
serverless.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
frameworkVersion: ">=3.21.0 <4.0.0"
service: okdata-pipeline
provider:
name: aws
ecr:
images:
okdata-pipeline:
path: ./
platform: linux/amd64
region: ${opt:region, 'eu-west-1'}
stage: ${opt:stage, 'dev'}
timeout: 60
deploymentBucket:
name: ${self:custom.deploymentBucket.${self:provider.stage}, self:custom.deploymentBucket.dev}
serverSideEncryption: AES256
deploymentPrefix: deploy
logRetentionInDays: 90
iam:
role:
permissionsBoundary: "arn:aws:iam::${aws:accountId}:policy/oslokommune/oslokommune-boundary"
managedPolicies:
- 'arn:aws:iam::${aws:accountId}:policy/pipeline_component_policy'
- 'arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess'
tags:
GIT_REV: ${git:branch}:${git:sha1}
VERSION: ${self:custom.version}
environment:
BUCKET_NAME: ok-origo-dataplatform-${self:provider.stage}
GIT_REV: ${git:branch}:${git:sha1}
KEYCLOAK_SERVER: ${ssm:/dataplatform/shared/keycloak-server-url}
SERVICE_NAME: ${self:service}
STAGE: ${self:provider.stage}
STATUS_API_URL: ${ssm:/dataplatform/shared/api-gateway-url}/status-api
VERSION: ${self:custom.version}
OKDATA_ENVIRONMENT: ${self:custom.okdataEnvironment.${self:provider.stage}, self:custom.okdataEnvironment.dev}
OKDATA_CLIENT_ID: ${self:service}
package:
patterns:
- '!**/*'
- okdata/**/*.py
functions:
csv-to-parquet:
image:
name: okdata-pipeline
command:
- okdata.pipeline.converters.csv.handler.csv_to_parquet
memorySize: 10240
timeout: 900
csv-to-delta:
image:
name: okdata-pipeline
command:
- okdata.pipeline.converters.csv.handler.csv_to_delta
json-to-delta:
image:
name: okdata-pipeline
command:
- okdata.pipeline.converters.json.handlers.json_to_delta
invoke-lambda:
image:
name: okdata-pipeline
command:
- okdata.pipeline.lambda_invoker.invoke_lambda
validate-csv:
image:
name: okdata-pipeline
command:
- okdata.pipeline.validators.csv.validator.validate_csv
validate-json:
image:
name: okdata-pipeline
command:
- okdata.pipeline.validators.json.handler.validate_json
write-s3:
image:
name: okdata-pipeline
command:
- okdata.pipeline.writers.s3.handlers.write_s3
timeout: 900
tracing: Active
xls-to-csv:
image:
name: okdata-pipeline
command:
- okdata.pipeline.converters.xls.handlers.xlsx_to_csv
memorySize: 2048
timeout: 240
xlsx-to-delta:
image:
name: okdata-pipeline
command:
- okdata.pipeline.converters.xls.handlers.xlsx_to_delta
memorySize: 2048
timeout: 240
plugins:
- serverless-better-credentials # must be first
- serverless-plugin-git-variables
- serverless-prune-plugin
resources:
Description: |
${self:service} ${git:branch}:${git:sha1}
${git:message}
custom:
version: 0.1.0
prune:
automatic: true
number: 3
exportGitVariables: false
deploymentBucket:
prod: ok-origo-dataplatform-config-prod
dev: ok-origo-dataplatform-config-dev
okdataEnvironment:
prod: prod
dev: dev