-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
16 changed files
with
127 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 20 additions & 0 deletions
20
...ping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package io.airbyte.integrations.base.destination.typing_deduping; | ||
|
|
||
| /** | ||
| * In general, callers should not directly instantiate this class. Use | ||
| * {@link SqlGenerator#buildColumnId(String)} instead. | ||
| * | ||
| * @param name the name of the column in the final table. Callers should prefer | ||
| * {@link #name(String)} when using the column in a query. | ||
| * @param originalName the name of the field in the raw JSON blob | ||
| * @param canonicalName the name of the field according to the destination. Used for deduping. | ||
| * Useful if a destination warehouse handles columns ignoring case, but preserves case in the | ||
| * table schema. | ||
| */ | ||
| public record ColumnId(String name, String originalName, String canonicalName) { | ||
|
|
||
| public String name(final String quote) { | ||
| return quote + name + quote; | ||
| } | ||
|
|
||
| } |
18 changes: 18 additions & 0 deletions
18
...src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package io.airbyte.integrations.base.destination.typing_deduping; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| public record ParsedCatalog(List<StreamConfig> streams) { | ||
|
|
||
| public StreamConfig getStream(String namespace, String name) { | ||
| return streams.stream() | ||
| .filter(s -> s.id().originalNamespace().equals(namespace) && s.id().originalName().equals(name)) | ||
| .findFirst() | ||
| .orElseThrow(() -> new IllegalArgumentException(String.format( | ||
| "Could not find stream %s.%s out of streams %s", | ||
| namespace, | ||
| name, | ||
| streams.stream().map(stream -> stream.id().originalNamespace() + "." + stream.id().originalName()).toList()))); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 16 additions & 0 deletions
16
.../src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package io.airbyte.integrations.base.destination.typing_deduping; | ||
|
|
||
| import io.airbyte.protocol.models.v0.DestinationSyncMode; | ||
| import io.airbyte.protocol.models.v0.SyncMode; | ||
| import java.util.LinkedHashMap; | ||
| import java.util.List; | ||
| import java.util.Optional; | ||
|
|
||
| public record StreamConfig(StreamId id, | ||
| SyncMode syncMode, | ||
| DestinationSyncMode destinationSyncMode, | ||
| List<ColumnId> primaryKey, | ||
| Optional<ColumnId> cursor, | ||
| LinkedHashMap<ColumnId, AirbyteType> columns) { | ||
|
|
||
| } |
45 changes: 45 additions & 0 deletions
45
...ping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| package io.airbyte.integrations.base.destination.typing_deduping; | ||
|
|
||
| /** | ||
| * In general, callers should not directly instantiate this class. Use | ||
| * {@link SqlGenerator#buildStreamId(String, String, String)} instead. | ||
| * <p> | ||
| * All names/namespaces are intended to be quoted, but do not explicitly contain quotes. For | ||
| * example, finalName might be "foo bar"; the caller is required to wrap that in quotes before using | ||
| * it in a query. | ||
| * | ||
| * @param finalNamespace the namespace where the final table will be created | ||
| * @param finalName the name of the final table | ||
| * @param rawNamespace the namespace where the raw table will be created (typically "airbyte") | ||
| * @param rawName the name of the raw table (typically namespace_name, but may be different if there | ||
| * are collisions). There is no rawNamespace because we assume that we're writing raw tables | ||
| * to the airbyte namespace. | ||
| */ | ||
| public record StreamId(String finalNamespace, String finalName, String rawNamespace, String rawName, | ||
| String originalNamespace, String originalName) { | ||
|
|
||
| /** | ||
| * Most databases/warehouses use a `schema.name` syntax to identify tables. This is a convenience | ||
| * method to generate that syntax. | ||
| */ | ||
| public String finalTableId(String quote) { | ||
| return quote + finalNamespace + quote + "." + quote + finalName + quote; | ||
| } | ||
|
|
||
| public String finalTableId(String suffix, String quote) { | ||
| return quote + finalNamespace + quote + "." + quote + finalName + suffix + quote; | ||
| } | ||
|
|
||
| public String rawTableId(String quote) { | ||
| return quote + rawNamespace + quote + "." + quote + rawName + quote; | ||
| } | ||
|
|
||
| public String finalName(final String quote) { | ||
| return quote + finalName + quote; | ||
| } | ||
|
|
||
| public String finalNamespace(final String quote) { | ||
| return quote + finalNamespace + quote; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.