-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Set resource limits for connector definitions: persistence layer (#10481
- Loading branch information
Showing
11 changed files
with
124 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
airbyte-config/models/src/main/resources/types/ActorDefinitionResourceRequirements.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
--- | ||
"$schema": http://json-schema.org/draft-07/schema# | ||
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/ActorDefinitionResourceRequirements.yaml | ||
title: ActorDefinitionResourceRequirements | ||
description: actor definition specific resource requirements | ||
type: object | ||
additionalProperties: true | ||
properties: | ||
default: | ||
description: if set, these are the requirements that should be set for ALL jobs run for this actor definition. | ||
"$ref": ResourceRequirements.yaml | ||
jobSpecific: | ||
type: array | ||
items: | ||
"$ref": "#/definitions/JobTypeResourceLimit" | ||
definitions: | ||
JobTypeResourceLimit: | ||
description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set. | ||
type: object | ||
required: | ||
- jobtype | ||
- resourceRequirements | ||
properties: | ||
jobType: | ||
"$ref": JobType.yaml | ||
resourceRequirements: | ||
"$ref": ResourceRequirements.yaml |
14 changes: 14 additions & 0 deletions
14
airbyte-config/models/src/main/resources/types/JobType.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
--- | ||
"$schema": http://json-schema.org/draft-07/schema# | ||
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/JobType.yaml | ||
title: JobType | ||
description: enum that describes the different types of jobs that the platform runs. | ||
type: string | ||
enum: | ||
- get_spec | ||
- check_connection | ||
- discover_schema | ||
- sync | ||
- reset_connection | ||
- connection_updater | ||
- replicate |
17 changes: 17 additions & 0 deletions
17
airbyte-config/models/src/main/resources/types/ResourceRequirements.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
--- | ||
"$schema": http://json-schema.org/draft-07/schema# | ||
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/ResourceRequirements.yaml | ||
title: ResourceRequirements | ||
description: generic configuration for pod source requirements | ||
type: object | ||
additionalProperties: true | ||
properties: | ||
# todo (cgardens) - should be camel case for consistency. | ||
cpu_request: | ||
type: string | ||
cpu_limit: | ||
type: string | ||
memory_request: | ||
type: string | ||
memory_limit: | ||
type: string |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
33 changes: 33 additions & 0 deletions
33
...te/db/instance/configs/migrations/V0_35_32_001__AddConnectorDefinitionResourceLimits.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
/* | ||
* Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
*/ | ||
|
||
package io.airbyte.db.instance.configs.migrations; | ||
|
||
import org.flywaydb.core.api.migration.BaseJavaMigration; | ||
import org.flywaydb.core.api.migration.Context; | ||
import org.jooq.DSLContext; | ||
import org.jooq.impl.DSL; | ||
import org.jooq.impl.SQLDataType; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
|
||
public class V0_35_32_001__AddConnectorDefinitionResourceLimits extends BaseJavaMigration { | ||
|
||
private static final Logger LOGGER = LoggerFactory.getLogger(V0_35_32_001__AddConnectorDefinitionResourceLimits.class); | ||
|
||
@Override | ||
public void migrate(final Context context) throws Exception { | ||
LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); | ||
|
||
final DSLContext ctx = DSL.using(context.getConnection()); | ||
addResourceReqsToActorDefs(ctx); | ||
} | ||
|
||
public static void addResourceReqsToActorDefs(final DSLContext ctx) { | ||
ctx.alterTable("actor_definition") | ||
.addColumnIfNotExists(DSL.field("resource_requirements", SQLDataType.JSONB.nullable(true))) | ||
.execute(); | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters