From fa78c3f1da746ff379a57466761f3edec2cb2e63 Mon Sep 17 00:00:00 2001 From: git-phu Date: Wed, 16 Mar 2022 23:55:13 -0700 Subject: [PATCH 1/3] listGrantableActorDefinitions --- .../config/persistence/ConfigRepository.java | 32 +++++++++++++++++++ .../ConfigRepositoryE2EReadWriteTest.java | 16 ++++++++++ 2 files changed, 48 insertions(+) diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 460e0899ba3a..545acb449c94 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -207,6 +208,18 @@ public List listGrantedSourceDefinitions(final UUID wo includeTombstones(ACTOR_DEFINITION.TOMBSTONE, includeTombstones)); } + public List> listGrantableSourceDefinitions(final UUID workspaceId, + final boolean includeTombstones) + throws IOException { + return listActorDefinitionsJoinedWithGrants( + workspaceId, + JoinType.LEFT_OUTER_JOIN, + ActorType.source, + record -> actorDefinitionWithGrantStatus(record, DbConverter::buildStandardSourceDefinition), + ACTOR_DEFINITION.CUSTOM.eq(false), + includeTombstones(ACTOR_DEFINITION.TOMBSTONE, includeTombstones)); + } + public void writeStandardSourceDefinition(final StandardSourceDefinition sourceDefinition) throws JsonValidationException, IOException { persistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, sourceDefinition.getSourceDefinitionId().toString(), sourceDefinition); } @@ -286,6 +299,18 @@ public List listGrantedDestinationDefinitions(fin includeTombstones(ACTOR_DEFINITION.TOMBSTONE, includeTombstones)); } + public List> listGrantableDestinationDefinitions(final UUID workspaceId, + final boolean includeTombstones) + throws IOException { + return listActorDefinitionsJoinedWithGrants( + workspaceId, + JoinType.LEFT_OUTER_JOIN, + ActorType.destination, + record -> actorDefinitionWithGrantStatus(record, DbConverter::buildStandardDestinationDefinition), + ACTOR_DEFINITION.CUSTOM.eq(false), + includeTombstones(ACTOR_DEFINITION.TOMBSTONE, includeTombstones)); + } + public void writeStandardDestinationDefinition(final StandardDestinationDefinition destinationDefinition) throws JsonValidationException, IOException { persistence.writeConfig( @@ -389,6 +414,13 @@ private List listActorDefinitionsJoinedWithGrants(final UUID workspaceId, .toList(); } + private Entry actorDefinitionWithGrantStatus(final Record outerJoinRecord, + final Function recordToActorDefinition) { + final T actorDefinition = recordToActorDefinition.apply(outerJoinRecord); + final boolean granted = outerJoinRecord.get(ACTOR_DEFINITION_WORKSPACE_GRANT.WORKSPACE_ID) != null; + return Map.entry(actorDefinition, granted); + } + /** * Returns source with a given id. Does not contain secrets. To hydrate with secrets see { @link * SecretsRepositoryReader#getSourceConnectionWithSecrets(final UUID sourceId) }. diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 8c658ab35548..6ffcbf33ce2d 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -38,6 +38,8 @@ import java.sql.SQLException; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -234,6 +236,7 @@ public void testListPublicSourceDefinitions() throws IOException { public void testSourceDefinitionGrants() throws IOException { final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); final StandardSourceDefinition grantableDefinition1 = MockData.grantableSourceDefinition1(); + final StandardSourceDefinition grantableDefinition2 = MockData.grantableSourceDefinition2(); final StandardSourceDefinition customDefinition = MockData.customSourceDefinition(); configRepository.writeActorDefinitionWorkspaceGrant(customDefinition.getSourceDefinitionId(), workspaceId); @@ -241,6 +244,12 @@ public void testSourceDefinitionGrants() throws IOException { final List actualGrantedDefinitions = configRepository .listGrantedSourceDefinitions(workspaceId, false); assertThat(actualGrantedDefinitions).hasSameElementsAs(List.of(grantableDefinition1, customDefinition)); + + final List> actualGrantableDefinitions = configRepository + .listGrantableSourceDefinitions(workspaceId, false); + assertThat(actualGrantableDefinitions).hasSameElementsAs(List.of( + Map.entry(grantableDefinition1, true), + Map.entry(grantableDefinition2, false))); } @Test @@ -253,6 +262,7 @@ public void testListPublicDestinationDefinitions() throws IOException { public void testDestinationDefinitionGrants() throws IOException { final UUID workspaceId = MockData.standardWorkspaces().get(0).getWorkspaceId(); final StandardDestinationDefinition grantableDefinition1 = MockData.grantableDestinationDefinition1(); + final StandardDestinationDefinition grantableDefinition2 = MockData.grantableDestinationDefinition2(); final StandardDestinationDefinition customDefinition = MockData.cusstomDestinationDefinition(); configRepository.writeActorDefinitionWorkspaceGrant(customDefinition.getDestinationDefinitionId(), workspaceId); @@ -260,6 +270,12 @@ public void testDestinationDefinitionGrants() throws IOException { final List actualGrantedDefinitions = configRepository .listGrantedDestinationDefinitions(workspaceId, false); assertThat(actualGrantedDefinitions).hasSameElementsAs(List.of(grantableDefinition1, customDefinition)); + + final List> actualGrantableDefinitions = configRepository + .listGrantableDestinationDefinitions(workspaceId, false); + assertThat(actualGrantableDefinitions).hasSameElementsAs(List.of( + Map.entry(grantableDefinition1, true), + Map.entry(grantableDefinition2, false))); } } From f8f0915f91280ca3944723462ab40baf530146c1 Mon Sep 17 00:00:00 2001 From: git-phu Date: Tue, 22 Mar 2022 18:59:35 -0700 Subject: [PATCH 2/3] listPrivateSourceDefinitions --- .../airbyte/server/apis/ConfigurationApi.java | 2 +- .../handlers/SourceDefinitionsHandler.java | 19 +++++++ .../SourceDefinitionsHandlerTest.java | 53 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index afa2087070a6..8cf4166b23ac 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -331,7 +331,7 @@ public SourceDefinitionReadList listLatestSourceDefinitions() { @Override public PrivateSourceDefinitionReadList listPrivateSourceDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return null; + return execute(() -> sourceDefinitionsHandler.listPrivateSourceDefinitions(workspaceIdRequestBody)); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java index 02dbbc4477e0..228305fb2bc1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceDefinitionsHandler.java @@ -7,6 +7,8 @@ import static io.airbyte.server.ServerConstants.DEV_IMAGE_TAG; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.api.model.PrivateSourceDefinitionRead; +import io.airbyte.api.model.PrivateSourceDefinitionReadList; import io.airbyte.api.model.ReleaseStage; import io.airbyte.api.model.SourceDefinitionCreate; import io.airbyte.api.model.SourceDefinitionIdRequestBody; @@ -35,6 +37,7 @@ import java.net.URISyntaxException; import java.time.LocalDate; import java.util.List; +import java.util.Map.Entry; import java.util.UUID; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -129,6 +132,22 @@ public SourceDefinitionReadList listSourceDefinitionsForWorkspace(final Workspac configRepository.listGrantedSourceDefinitions(workspaceIdRequestBody.getWorkspaceId(), false))); } + public PrivateSourceDefinitionReadList listPrivateSourceDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) + throws IOException { + final List> standardSourceDefinitionBooleanMap = + configRepository.listGrantableSourceDefinitions(workspaceIdRequestBody.getWorkspaceId(), false); + return toPrivateSourceDefinitionReadList(standardSourceDefinitionBooleanMap); + } + + private static PrivateSourceDefinitionReadList toPrivateSourceDefinitionReadList(final List> defs) { + final List reads = defs.stream() + .map(entry -> new PrivateSourceDefinitionRead() + .sourceDefinition(buildSourceDefinitionRead(entry.getKey())) + .granted(entry.getValue())) + .collect(Collectors.toList()); + return new PrivateSourceDefinitionReadList().sourceDefinitions(reads); + } + public SourceDefinitionRead getSourceDefinition(final SourceDefinitionIdRequestBody sourceDefinitionIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { return buildSourceDefinitionRead(configRepository.getStandardSourceDefinition(sourceDefinitionIdRequestBody.getSourceDefinitionId())); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java index 5fb72c1a7bc0..b4aaceac3411 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceDefinitionsHandlerTest.java @@ -15,6 +15,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.api.model.PrivateSourceDefinitionRead; +import io.airbyte.api.model.PrivateSourceDefinitionReadList; import io.airbyte.api.model.ReleaseStage; import io.airbyte.api.model.SourceDefinitionCreate; import io.airbyte.api.model.SourceDefinitionIdRequestBody; @@ -43,6 +45,7 @@ import java.net.URISyntaxException; import java.time.LocalDate; import java.util.Collections; +import java.util.Map; import java.util.UUID; import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; @@ -180,6 +183,56 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept actualSourceDefinitionReadList.getSourceDefinitions()); } + @Test + @DisplayName("listPrivateSourceDefinitions should return the right list") + void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { + final StandardSourceDefinition sourceDefinition2 = generateSourceDefinition(); + + when(configRepository.listGrantableSourceDefinitions(workspaceId, false)).thenReturn( + Lists.newArrayList( + Map.entry(sourceDefinition, false), + Map.entry(sourceDefinition2, true))); + + final SourceDefinitionRead expectedSourceDefinitionRead1 = new SourceDefinitionRead() + .sourceDefinitionId(sourceDefinition.getSourceDefinitionId()) + .name(sourceDefinition.getName()) + .dockerRepository(sourceDefinition.getDockerRepository()) + .dockerImageTag(sourceDefinition.getDockerImageTag()) + .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) + .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .releaseStage(ReleaseStage.fromValue(sourceDefinition.getReleaseStage().value())) + .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + + final SourceDefinitionRead expectedSourceDefinitionRead2 = new SourceDefinitionRead() + .sourceDefinitionId(sourceDefinition2.getSourceDefinitionId()) + .name(sourceDefinition2.getName()) + .dockerRepository(sourceDefinition.getDockerRepository()) + .dockerImageTag(sourceDefinition.getDockerImageTag()) + .documentationUrl(new URI(sourceDefinition.getDocumentationUrl())) + .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .releaseStage(ReleaseStage.fromValue(sourceDefinition.getReleaseStage().value())) + .releaseDate(LocalDate.parse(sourceDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(sourceDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); + + final PrivateSourceDefinitionRead expectedSourceDefinitionOptInRead1 = + new PrivateSourceDefinitionRead().sourceDefinition(expectedSourceDefinitionRead1).granted(false); + + final PrivateSourceDefinitionRead expectedSourceDefinitionOptInRead2 = + new PrivateSourceDefinitionRead().sourceDefinition(expectedSourceDefinitionRead2).granted(true); + + final PrivateSourceDefinitionReadList actualSourceDefinitionOptInReadList = sourceDefinitionsHandler.listPrivateSourceDefinitions( + new WorkspaceIdRequestBody().workspaceId(workspaceId)); + + assertEquals( + Lists.newArrayList(expectedSourceDefinitionOptInRead1, expectedSourceDefinitionOptInRead2), + actualSourceDefinitionOptInReadList.getSourceDefinitions()); + } + @Test @DisplayName("getSourceDefinition should return the right source") void testGetSourceDefinition() throws JsonValidationException, ConfigNotFoundException, IOException, URISyntaxException { From bab763163401edf4d5fea2f2455cef538a5b3c01 Mon Sep 17 00:00:00 2001 From: git-phu Date: Tue, 22 Mar 2022 18:59:50 -0700 Subject: [PATCH 3/3] listPrivateDestinationDefinitions --- .../airbyte/server/apis/ConfigurationApi.java | 2 +- .../DestinationDefinitionsHandler.java | 20 +++++++ .../DestinationDefinitionsHandlerTest.java | 54 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 8cf4166b23ac..678b6763f470 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -518,7 +518,7 @@ public DestinationDefinitionReadList listLatestDestinationDefinitions() { @Override public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return null; + return execute(() -> destinationDefinitionsHandler.listPrivateDestinationDefinitions(workspaceIdRequestBody)); } @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 8d52e96f3432..4b43fae9e6bc 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -13,6 +13,8 @@ import io.airbyte.api.model.DestinationDefinitionReadList; import io.airbyte.api.model.DestinationDefinitionUpdate; import io.airbyte.api.model.DestinationRead; +import io.airbyte.api.model.PrivateDestinationDefinitionRead; +import io.airbyte.api.model.PrivateDestinationDefinitionReadList; import io.airbyte.api.model.ReleaseStage; import io.airbyte.api.model.WorkspaceIdRequestBody; import io.airbyte.commons.docker.DockerUtils; @@ -35,6 +37,7 @@ import java.net.URISyntaxException; import java.time.LocalDate; import java.util.List; +import java.util.Map.Entry; import java.util.UUID; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -133,6 +136,23 @@ public DestinationDefinitionReadList listDestinationDefinitionsForWorkspace(fina configRepository.listGrantedDestinationDefinitions(workspaceIdRequestBody.getWorkspaceId(), false))); } + public PrivateDestinationDefinitionReadList listPrivateDestinationDefinitions(final WorkspaceIdRequestBody workspaceIdRequestBody) + throws IOException { + final List> standardDestinationDefinitionBooleanMap = + configRepository.listGrantableDestinationDefinitions(workspaceIdRequestBody.getWorkspaceId(), false); + return toPrivateDestinationDefinitionReadList(standardDestinationDefinitionBooleanMap); + } + + private static PrivateDestinationDefinitionReadList toPrivateDestinationDefinitionReadList( + final List> defs) { + final List reads = defs.stream() + .map(entry -> new PrivateDestinationDefinitionRead() + .destinationDefinition(buildDestinationDefinitionRead(entry.getKey())) + .granted(entry.getValue())) + .collect(Collectors.toList()); + return new PrivateDestinationDefinitionReadList().destinationDefinitions(reads); + } + public DestinationDefinitionRead getDestinationDefinition(final DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { return buildDestinationDefinitionRead( diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index 71eb43a017f2..990c75f70507 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -21,6 +21,8 @@ import io.airbyte.api.model.DestinationDefinitionUpdate; import io.airbyte.api.model.DestinationRead; import io.airbyte.api.model.DestinationReadList; +import io.airbyte.api.model.PrivateDestinationDefinitionRead; +import io.airbyte.api.model.PrivateDestinationDefinitionReadList; import io.airbyte.api.model.ReleaseStage; import io.airbyte.api.model.WorkspaceIdRequestBody; import io.airbyte.commons.docker.DockerUtils; @@ -42,6 +44,7 @@ import java.net.URISyntaxException; import java.time.LocalDate; import java.util.Collections; +import java.util.Map; import java.util.UUID; import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; @@ -181,6 +184,57 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE actualDestinationDefinitionReadList.getDestinationDefinitions()); } + @Test + @DisplayName("listPrivateDestinationDefinitions should return the right list") + void testListPrivateDestinationDefinitions() throws IOException, URISyntaxException { + final StandardDestinationDefinition destinationDefinition2 = generateDestinationDefinition(); + + when(configRepository.listGrantableDestinationDefinitions(workspaceId, false)).thenReturn( + Lists.newArrayList( + Map.entry(destinationDefinition, false), + Map.entry(destinationDefinition2, true))); + + final DestinationDefinitionRead expectedDestinationDefinitionRead1 = new DestinationDefinitionRead() + .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()) + .name(destinationDefinition.getName()) + .dockerRepository(destinationDefinition.getDockerRepository()) + .dockerImageTag(destinationDefinition.getDockerImageTag()) + .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) + .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) + .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest()))); + + final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() + .destinationDefinitionId(destinationDefinition2.getDestinationDefinitionId()) + .name(destinationDefinition2.getName()) + .dockerRepository(destinationDefinition.getDockerRepository()) + .dockerImageTag(destinationDefinition.getDockerImageTag()) + .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) + .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) + .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) + .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .resourceRequirements(new io.airbyte.api.model.ActorDefinitionResourceRequirements() + ._default(new io.airbyte.api.model.ResourceRequirements() + .cpuRequest(destinationDefinition2.getResourceRequirements().getDefault().getCpuRequest()))); + + final PrivateDestinationDefinitionRead expectedDestinationDefinitionOptInRead1 = + new PrivateDestinationDefinitionRead().destinationDefinition(expectedDestinationDefinitionRead1).granted(false); + + final PrivateDestinationDefinitionRead expectedDestinationDefinitionOptInRead2 = + new PrivateDestinationDefinitionRead().destinationDefinition(expectedDestinationDefinitionRead2).granted(true); + + final PrivateDestinationDefinitionReadList actualDestinationDefinitionOptInReadList = + destinationDefinitionsHandler.listPrivateDestinationDefinitions( + new WorkspaceIdRequestBody().workspaceId(workspaceId)); + + assertEquals( + Lists.newArrayList(expectedDestinationDefinitionOptInRead1, expectedDestinationDefinitionOptInRead2), + actualDestinationDefinitionOptInReadList.getDestinationDefinitions()); + } + @Test @DisplayName("getDestinationDefinition should return the right destination") void testGetDestination() throws JsonValidationException, ConfigNotFoundException, IOException, URISyntaxException {