-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into cus2240-lookml-ingestion-fails
- Loading branch information
Showing
112 changed files
with
3,382 additions
and
263 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,33 @@ | ||
| import setuptools | ||
| import os | ||
|
|
||
| folders = ["./smoke-test/tests"] | ||
|
|
||
| for folder in folders: | ||
| print(f"Checking folder {folder}") | ||
| a = [i for i in setuptools.find_packages(folder) if "cypress" not in i] | ||
| b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i] | ||
| packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i] | ||
| namespace_packages = [ | ||
| i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i | ||
| ] | ||
|
|
||
| in_a_not_b = set(a) - set(b) | ||
| in_b_not_a = set(b) - set(a) | ||
| print("Packages found:", packages) | ||
| print("Namespace packages found:", namespace_packages) | ||
|
|
||
| in_packages_not_namespace = set(packages) - set(namespace_packages) | ||
| in_namespace_not_packages = set(namespace_packages) - set(packages) | ||
|
|
||
| if in_packages_not_namespace: | ||
| print(f"Packages not in namespace packages: {in_packages_not_namespace}") | ||
| if in_namespace_not_packages: | ||
| print(f"Namespace packages not in packages: {in_namespace_not_packages}") | ||
| for pkg in in_namespace_not_packages: | ||
| pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep)) | ||
| print(f"Contents of {pkg_path}:") | ||
| print(os.listdir(pkg_path)) | ||
|
|
||
| assert ( | ||
| len(in_a_not_b) == 0 | ||
| ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}" | ||
| len(in_packages_not_namespace) == 0 | ||
| ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}" | ||
| assert ( | ||
| len(in_b_not_a) == 0 | ||
| ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}" | ||
| len(in_namespace_not_packages) == 0 | ||
| ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
57 changes: 57 additions & 0 deletions
57
...din/datahub/graphql/resolvers/settings/docPropagation/DocPropagationSettingsResolver.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package com.linkedin.datahub.graphql.resolvers.settings.docPropagation; | ||
|
|
||
| import com.linkedin.datahub.graphql.QueryContext; | ||
| import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; | ||
| import com.linkedin.datahub.graphql.generated.DocPropagationSettings; | ||
| import com.linkedin.metadata.service.SettingsService; | ||
| import com.linkedin.settings.global.GlobalSettingsInfo; | ||
| import graphql.schema.DataFetcher; | ||
| import graphql.schema.DataFetchingEnvironment; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** Retrieves the Global Settings related to the Actions feature. */ | ||
| @Slf4j | ||
| public class DocPropagationSettingsResolver | ||
| implements DataFetcher<CompletableFuture<DocPropagationSettings>> { | ||
|
|
||
| private final SettingsService _settingsService; | ||
|
|
||
| public DocPropagationSettingsResolver(final SettingsService settingsService) { | ||
| _settingsService = Objects.requireNonNull(settingsService, "settingsService must not be null"); | ||
| } | ||
|
|
||
| @Override | ||
| public CompletableFuture<DocPropagationSettings> get(final DataFetchingEnvironment environment) | ||
| throws Exception { | ||
| final QueryContext context = environment.getContext(); | ||
| return GraphQLConcurrencyUtils.supplyAsync( | ||
| () -> { | ||
| try { | ||
| final GlobalSettingsInfo globalSettings = | ||
| _settingsService.getGlobalSettings(context.getOperationContext()); | ||
| final DocPropagationSettings defaultSettings = new DocPropagationSettings(); | ||
| defaultSettings.setDocColumnPropagation(true); | ||
| return globalSettings != null && globalSettings.hasDocPropagation() | ||
| ? mapDocPropagationSettings(globalSettings.getDocPropagation()) | ||
| : defaultSettings; | ||
| } catch (Exception e) { | ||
| throw new RuntimeException("Failed to retrieve Action Settings", e); | ||
| } | ||
| }, | ||
| this.getClass().getSimpleName(), | ||
| "get"); | ||
| } | ||
|
|
||
| private static DocPropagationSettings mapDocPropagationSettings( | ||
| @Nonnull final com.linkedin.settings.global.DocPropagationFeatureSettings settings) { | ||
| final DocPropagationSettings result = new DocPropagationSettings(); | ||
|
|
||
| // Map docColumnPropagation settings field | ||
| result.setDocColumnPropagation(settings.isColumnPropagationEnabled()); | ||
|
|
||
| return result; | ||
| } | ||
| } |
77 changes: 77 additions & 0 deletions
77
...tahub/graphql/resolvers/settings/docPropagation/UpdateDocPropagationSettingsResolver.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package com.linkedin.datahub.graphql.resolvers.settings.docPropagation; | ||
|
|
||
| import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; | ||
|
|
||
| import com.linkedin.datahub.graphql.QueryContext; | ||
| import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; | ||
| import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; | ||
| import com.linkedin.datahub.graphql.exception.AuthorizationException; | ||
| import com.linkedin.datahub.graphql.generated.UpdateDocPropagationSettingsInput; | ||
| import com.linkedin.metadata.service.SettingsService; | ||
| import com.linkedin.settings.global.DocPropagationFeatureSettings; | ||
| import com.linkedin.settings.global.GlobalSettingsInfo; | ||
| import graphql.schema.DataFetcher; | ||
| import graphql.schema.DataFetchingEnvironment; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import javax.annotation.Nonnull; | ||
|
|
||
| /** Resolver responsible for updating the actions settings. */ | ||
| public class UpdateDocPropagationSettingsResolver | ||
| implements DataFetcher<CompletableFuture<Boolean>> { | ||
|
|
||
| private final SettingsService _settingsService; | ||
|
|
||
| public UpdateDocPropagationSettingsResolver(@Nonnull final SettingsService settingsService) { | ||
| _settingsService = Objects.requireNonNull(settingsService, "settingsService must not be null"); | ||
| } | ||
|
|
||
| @Override | ||
| public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment) | ||
| throws Exception { | ||
| final QueryContext context = environment.getContext(); | ||
| final UpdateDocPropagationSettingsInput input = | ||
| bindArgument(environment.getArgument("input"), UpdateDocPropagationSettingsInput.class); | ||
|
|
||
| return GraphQLConcurrencyUtils.supplyAsync( | ||
| () -> { | ||
| if (AuthorizationUtils.canManageFeatures(context)) { | ||
| try { | ||
| // First, fetch the existing global settings. This does a R-M-F. | ||
| final GlobalSettingsInfo maybeGlobalSettings = | ||
| _settingsService.getGlobalSettings(context.getOperationContext()); | ||
|
|
||
| final GlobalSettingsInfo newGlobalSettings = | ||
| maybeGlobalSettings != null ? maybeGlobalSettings : new GlobalSettingsInfo(); | ||
|
|
||
| final DocPropagationFeatureSettings newDocPropagationSettings = | ||
| newGlobalSettings.hasDocPropagation() | ||
| ? newGlobalSettings.getDocPropagation() | ||
| : new DocPropagationFeatureSettings().setEnabled(true); | ||
|
|
||
| // Next, patch the actions settings. | ||
| updateDocPropagationSettings(newDocPropagationSettings, input); | ||
| newGlobalSettings.setDocPropagation(newDocPropagationSettings); | ||
|
|
||
| // Finally, write back to GMS. | ||
| _settingsService.updateGlobalSettings( | ||
| context.getOperationContext(), newGlobalSettings); | ||
| return true; | ||
| } catch (Exception e) { | ||
| throw new RuntimeException( | ||
| String.format("Failed to update action settings! %s", input), e); | ||
| } | ||
| } | ||
| throw new AuthorizationException( | ||
| "Unauthorized to perform this action. Please contact your DataHub administrator."); | ||
| }, | ||
| this.getClass().getSimpleName(), | ||
| "get"); | ||
| } | ||
|
|
||
| private static void updateDocPropagationSettings( | ||
| @Nonnull final com.linkedin.settings.global.DocPropagationFeatureSettings settings, | ||
| @Nonnull final UpdateDocPropagationSettingsInput input) { | ||
| settings.setColumnPropagationEnabled(input.getDocColumnPropagation()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
54 changes: 54 additions & 0 deletions
54
.../src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DocumentationMapper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| package com.linkedin.datahub.graphql.types.common.mappers; | ||
|
|
||
| import com.linkedin.common.urn.Urn; | ||
| import com.linkedin.datahub.graphql.QueryContext; | ||
| import com.linkedin.datahub.graphql.generated.DataHubConnection; | ||
| import com.linkedin.datahub.graphql.generated.Documentation; | ||
| import com.linkedin.datahub.graphql.generated.DocumentationAssociation; | ||
| import com.linkedin.datahub.graphql.generated.EntityType; | ||
| import com.linkedin.datahub.graphql.types.mappers.ModelMapper; | ||
| import java.util.stream.Collectors; | ||
| import javax.annotation.Nonnull; | ||
| import javax.annotation.Nullable; | ||
|
|
||
| public class DocumentationMapper | ||
| implements ModelMapper<com.linkedin.common.Documentation, Documentation> { | ||
|
|
||
| public static final DocumentationMapper INSTANCE = new DocumentationMapper(); | ||
|
|
||
| public static Documentation map( | ||
| @Nullable final QueryContext context, | ||
| @Nonnull final com.linkedin.common.Documentation metadata) { | ||
| return INSTANCE.apply(context, metadata); | ||
| } | ||
|
|
||
| @Override | ||
| public Documentation apply( | ||
| @Nullable final QueryContext context, | ||
| @Nonnull final com.linkedin.common.Documentation input) { | ||
| final Documentation result = new Documentation(); | ||
| result.setDocumentations( | ||
| input.getDocumentations().stream() | ||
| .map(docAssociation -> mapDocAssociation(context, docAssociation)) | ||
| .collect(Collectors.toList())); | ||
| return result; | ||
| } | ||
|
|
||
| private DocumentationAssociation mapDocAssociation( | ||
| @Nullable final QueryContext context, | ||
| @Nonnull final com.linkedin.common.DocumentationAssociation association) { | ||
| final DocumentationAssociation result = new DocumentationAssociation(); | ||
| result.setDocumentation(association.getDocumentation()); | ||
| if (association.getAttribution() != null) { | ||
| result.setAttribution(MetadataAttributionMapper.map(context, association.getAttribution())); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| private DataHubConnection mapConnectionEntity(@Nonnull final Urn urn) { | ||
| DataHubConnection connection = new DataHubConnection(); | ||
| connection.setUrn(urn.toString()); | ||
| connection.setType(EntityType.DATAHUB_CONNECTION); | ||
| return connection; | ||
| } | ||
| } |
Oops, something went wrong.