Skip to content

Commit

Permalink
POC revamping implementation
Browse files Browse the repository at this point in the history 
squash me

Signed-off-by: Ljupcho Palashevski <lpalashevski@gmail.com>
  • Loading branch information
@tijanapavicic @lpalashevski
tijanapavicic authored and lpalashevski committed Sep 8, 2023
1 parent 358543d commit 7b9ce1c
Show file tree
Hide file tree
Showing 6 changed files with 194 additions and 64 deletions.
2 changes: 0 additions & 2 deletions bom/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ ext {
servletVersion = '4.0.1'
jakartaServletVersion = '6.0.0'
sleepycatVersion = '18.3.12'
snakeyamlVersion = '2.0'
slf4jVersion = '2.0.6'
snappyVersion = '1.1.10.2'
springbootVersion = '3.1.1'
Expand Down Expand Up @@ -255,7 +254,6 @@ dependencies {
// testng also used in our 'source' code to support unit tests
api("org.testng:testng:${testngVersion}")
api("joda-time:joda-time:${jodatimeVersion}")
api("org.yaml:snakeyaml:${snakeyamlVersion}")
api("org.antlr:antlr-runtime:${antlrVersion}")
api("org.antlr:ST4:${ST4Version}")
api("org.apache.jena:jena-arq:${jenaVersion}")
Expand Down
2 changes: 2 additions & 0 deletions open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ dependencies {
implementation 'org.projectlombok:lombok'
// implementation 'org.yaml:snakeyaml'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
implementation 'com.fasterxml.jackson.core:jackson-core'
annotationProcessor 'org.projectlombok:lombok'

runtimeOnly 'io.micrometer:micrometer-registry-prometheus'
Expand All @@ -46,6 +47,7 @@ dependencies {
implementation project(path: ':open-metadata-implementation:adapters:open-connectors:connector-configuration-factory')
implementation project(path: ':open-metadata-implementation:repository-services:repository-services-implementation')

implementation 'com.google.guava:guava'
/* IN DEVELOPMENT */

/* Pulling dependencies for some fo the sub-systems enabling 'Metadata Access Store' services */
Expand Down
157 changes: 112 additions & 45 deletions ...src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,24 @@
package org.odpi.openmetadata.serverchassis.springboot.config;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.google.common.io.Files;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory;
import org.odpi.openmetadata.adminservices.configuration.properties.LocalRepositoryConfig;
import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig;
import org.odpi.openmetadata.adminservices.configuration.properties.RepositoryServicesConfig;
import org.odpi.openmetadata.frameworks.connectors.properties.beans.Connection;
import org.odpi.openmetadata.frameworks.connectors.properties.beans.ConnectorType;
import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory;
import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions;
import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.yaml.snakeyaml.Yaml;
import org.springframework.core.io.Resource;

import java.io.IOException;
import java.util.List;
Expand All @@ -28,63 +30,84 @@

@EnableConfigurationProperties(OMAGServerProperties.class)
@Configuration
@Slf4j
public class OMAGConfigHelper {

@Getter final OMAGServerProperties serverProperties;
@Getter OMAGServerConfig omagServerConfig;
private static final Logger LOG = LoggerFactory.getLogger(OMAGConfigHelper.class);
private final ObjectMapper mapper;
public static final String WRONG_EXTENSION_MESSAGE = "Unallowed config file extension, " +
"allowed config file extensions are: " + Extensions.stream();
@Getter
final OMAGServerProperties serverProperties;
private final ObjectMapper jsonObjectMapper;
private final ObjectMapper yamlObjectMapper;
@Getter
OMAGServerConfig omagServerConfig;
ConnectorConfigurationFactory connectorConfigurationFactory;
OMRSConfigurationFactory omrsConfigurationFactory;

@Autowired
public OMAGConfigHelper(OMAGServerProperties properties, ObjectMapper objectMapper, ConnectorConfigurationFactory connectorConfigurationFactory, OMRSConfigurationFactory omrsConfigurationFactory) {
public OMAGConfigHelper(OMAGServerProperties properties,
@Qualifier("jsonObjectMapper") ObjectMapper jsonObjectMapper,
@Qualifier("yamlObjectMapper") ObjectMapper yamlObjectMapper,
ConnectorConfigurationFactory connectorConfigurationFactory,
OMRSConfigurationFactory omrsConfigurationFactory) {
this.serverProperties = properties;
this.mapper = objectMapper;
this.jsonObjectMapper = jsonObjectMapper;
this.yamlObjectMapper = yamlObjectMapper;
this.connectorConfigurationFactory = connectorConfigurationFactory;
this.omrsConfigurationFactory = omrsConfigurationFactory;
}


/**
* Method implementing logic for deciding proper configuration source
* and loading its content into configuration object
* @throws OMAGServerActivationError
*/
public void loadConfig() throws OMAGServerActivationError {

try {
if (serverProperties.getServerConfigFile() != null) {
if (isConfigurationFileProvided()) {

Resource serverConfigFile = serverProperties.getServerConfigFile();

//TODO: This is POC implementation.
// Better way to identify content type may be required.

if (!isJsonConfigurationFile(serverConfigFile) && !isYamlConfigurationFile(serverConfigFile)) {
log.error(WRONG_EXTENSION_MESSAGE);
throw new OMAGServerActivationError(WRONG_EXTENSION_MESSAGE);
}

LOG.info("Configuration {}", serverProperties.getServerConfigFile());
//TODO: This is POC implementation. Better way to identify content type may be required.
if (Objects.requireNonNull(serverProperties.getServerConfigFile().getFilename()).endsWith("json")) {
omagServerConfig = mapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class);
} else {
log.info("{} based configuration with server-config-file {}", getFileExtension(serverConfigFile), serverConfigFile);

if (isJsonConfigurationFile(serverConfigFile)) {
omagServerConfig = jsonObjectMapper.reader()
.readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class);

} else if (isYamlConfigurationFile(serverConfigFile)) {
//TODO: This is POC implementation. We need better code to deal with yaml i.e. one option is to use Jackson and Yaml data format.
// Yaml yaml = new Yaml();
// omagServerConfig = yaml.loadAs(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class);
ObjectMapper yamlmapper = new ObjectMapper(new YAMLFactory());
omagServerConfig = yamlmapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class);
// omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class);

omagServerConfig = yamlObjectMapper.reader()
.readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class);
}

} else if (serverProperties.getServer() != null) {
} else if (isPropertiesConfiguration()) {

log.info("Configuring server using omag. application properties [EXPERIMENTAL]");

LOG.info("Configuration document for server not provided");
LOG.info("[EXPERIMENTAL] Configuring server using omag. application properties");
OMAGServerProperties.Server omagServer = serverProperties.getServer();

omagServerConfig = new OMAGServerConfig();
omagServerConfig.setLocalServerName(serverProperties.getServer().getName());
omagServerConfig.setLocalServerUserId(serverProperties.getServer().getUser());
omagServerConfig.setMaxPageSize(serverProperties.getServer().getMaxPageSize());
omagServerConfig.setLocalServerName(omagServer.getName());
omagServerConfig.setLocalServerUserId(omagServer.getUser());
omagServerConfig.setMaxPageSize(omagServer.getMaxPageSize());

if (serverProperties.getServer().getRepositoryServices() != null) {
if (omagServer.getRepositoryServices() != null) {
RepositoryServicesConfig repositoryServicesConfig = omrsConfigurationFactory.getDefaultRepositoryServicesConfig();
if (serverProperties.getServer().getRepositoryServices().getAuditLogConnectors() != null) {
List<Connection> auditLogConnectors = serverProperties.getServer().getRepositoryServices().getAuditLogConnectors().stream().map(o -> {
Connection c = new Connection();
ConnectorType ct = new ConnectorType();
ct.setConnectorProviderClassName(o.getProviderClassName());
c.setConnectorType(ct);
//TODO: Fix binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!!
c.setConfigurationProperties(o.getConfigurationProperties());
return c;
}).collect(Collectors.toList());
repositoryServicesConfig.setAuditLogConnections(auditLogConnectors);

if (omagServer.getRepositoryServices().getAuditLogConnectors() != null) {

repositoryServicesConfig.setAuditLogConnections(getAuditLogConnectors());
}
//TODO: Work in progress currently using default in-mem local repo.
LocalRepositoryConfig localRepositoryConfig = getLocalRepositoryConfig();
Expand All @@ -93,21 +116,65 @@ public void loadConfig() throws OMAGServerActivationError {
}
}

LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName());
log.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName());

} catch (Exception e) {
LOG.info("Configuration document cannot be loaded from the resource provided - check application configuration");
} catch (IOException e) {
log.info("Configuration document cannot be loaded from the resource provided - check application configuration");
throw new OMAGServerActivationError(
String.format("Configuration document cannot be loaded from the resource provided - check application configuration"), e);
}
}

private String getFileExtension(Resource serverConfigFile) {
return Files.getFileExtension(serverConfigFile.getFilename());
}

private boolean isConfigurationFileProvided() {
return serverProperties.getServerConfigFile() != null;
}

private boolean isJsonConfigurationFile(Resource serverConfigFile) {
return Extensions.JSON.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile)));
}

private boolean isYamlConfigurationFile(Resource serverConfigFile) {
return Extensions.YAML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile)))
|| Extensions.YML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile)));
}

private boolean isPropertiesConfiguration() {
return serverProperties.getServer() != null;
}

private List<Connection> getAuditLogConnectors() {

return serverProperties.getServer()
.getRepositoryServices()
.getAuditLogConnectors()
.stream()
.map(o -> getConnection(o))
.collect(Collectors.toList());
}

private Connection getConnection(OMAGServerProperties.Connector o) {
Connection c = new Connection();
ConnectorType ct = new ConnectorType();
ct.setConnectorProviderClassName(o.getProviderClassName());
c.setConnectorType(ct);
//TODO: Fix binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!!
c.setConfigurationProperties(o.getConfigurationProperties());
return c;
}

private LocalRepositoryConfig getLocalRepositoryConfig() {
LocalRepositoryConfig localRepositoryConfig = new LocalRepositoryConfig();
localRepositoryConfig.setEventsToSendRule(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getEventsToSend());
localRepositoryConfig.setEventsToSaveRule(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getEventsToSave());
localRepositoryConfig.setMetadataCollectionId(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getMetadataCollectionId());

OMAGServerProperties.LocalRepository localRepository =
getServerProperties().getServer().getRepositoryServices().getLocalRepository();

localRepositoryConfig.setEventsToSendRule(localRepository.getEventsToSend());
localRepositoryConfig.setEventsToSaveRule(localRepository.getEventsToSave());
localRepositoryConfig.setMetadataCollectionId(localRepository.getMetadataCollectionId());
localRepositoryConfig.setLocalRepositoryLocalConnection(connectorConfigurationFactory.getInMemoryLocalRepositoryLocalConnection());
return localRepositoryConfig;
}
Expand Down
2 changes: 0 additions & 2 deletions ...main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@

import lombok.Getter;
import lombok.Setter;
import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig;
import org.odpi.openmetadata.adminservices.configuration.properties.OpenMetadataExchangeRule;
import org.odpi.openmetadata.adminservices.configuration.registration.ServerTypeClassification;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.core.io.Resource;
import org.springframework.validation.annotation.Validated;
Expand Down
60 changes: 45 additions & 15 deletions ...java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,38 @@
/* Copyright Contributors to the ODPi Egeria project. */
package org.odpi.openmetadata.serverchassis.springboot.config;


import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
import lombok.extern.slf4j.Slf4j;

import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.util.StringUtils;

import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;

/**
* This class provides configuration bean for customizing Jackson object mapper singleton instance configuration.
* This class provides configurations for customizing
* json bean Jackson object mapper singleton instance configuration
* and
* yaml bean Jackson object mapper singleton instance configuration
*/
@Slf4j
@Configuration
public class ObjectMapperConfiguration {
public static final String PREFIX = "Object mapper configuration started.";
private final Logger log = LoggerFactory.getLogger(this.getClass());
public static final String START_LOG_SUFIX = "ObjectMapper configuration started.";

public static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder builder) {
private static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) {

return builder
return jackson2ObjectMapperBuilder
.serializationInclusion(NON_NULL)
.failOnEmptyBeans(false)
.failOnUnknownProperties(false)
Expand All @@ -39,13 +46,36 @@ public static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder builder)
.build();
}

@Bean(name = {"objectMapper"})
@Primary
ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) {
/**
* @param jackson2ObjectMapperBuilder a builder used to create ObjectMapper instances with a fluent API.
* @return JsonMapper a JSON-format specific ObjectMapper implementation
*/
@Bean
@Qualifier("jsonObjectMapper")
ObjectMapper objectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) {

log.debug(PREFIX);
log.info("{}{}", StringUtils.capitalize(Extensions.JSON.name()), START_LOG_SUFIX);

return newObjectMapper(builder);
return newObjectMapper(jackson2ObjectMapperBuilder);
}
}

/**
* @return YAMLMapper a convenience version of ObjectMapper which is configured with YAMLFactory
*/
@Bean
@Qualifier("yamlObjectMapper")
public ObjectMapper yamlMapper() {

log.info("{}{}", StringUtils.capitalize(Extensions.YAML.name()), START_LOG_SUFIX);

return YAMLMapper.builder()
.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)
.disable(YAMLGenerator.Feature.ALLOW_LONG_KEYS)
.serializationInclusion(JsonInclude.Include.NON_NULL)
.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
.build();
}
}
35 changes: 35 additions & 0 deletions ...ng/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package org.odpi.openmetadata.serverchassis.springboot.constants;

import java.io.Serializable;
import java.util.stream.Stream;

/**
* Created by YM21WO on egeria
* on 17/08/2023 at 11:19
*/
public enum Extensions implements Serializable {

YAML(".yaml"),
YML(".yml"),
JSON(".json");


private final String value;

Extensions(String value) {
this.value = value;
}

public static String stream() {
StringBuffer stringBuffer = new StringBuffer();
Stream.of(Extensions.values())
.forEach(e -> stringBuffer.append(e.getValue()).append(", "));

return stringBuffer.toString();
}

private String getValue() {
return value;
}

}

0 comments on commit 7b9ce1c

Please sign in to comment.