From 194c9676c688ae8651609ff80392147b2efd6c4d Mon Sep 17 00:00:00 2001 From: sushi30 Date: Wed, 8 Jan 2025 16:05:12 +0200 Subject: [PATCH 1/8] feat(apps): support event subscriptions - added support for apps with event subscriptions. - added support for custom consumers on event subscriptions. - use native application methods instead of reflection in ApplicationHandler --- .../service/apps/ApplicationHandler.java | 128 +++++- .../scheduled/EventSubscriptionScheduler.java | 45 +- .../jdbi3/AppMarketPlaceRepository.java | 6 + .../service/jdbi3/AppRepository.java | 18 + .../service/resources/apps/AppMapper.java | 3 +- .../resources/apps/AppMarketPlaceMapper.java | 3 +- .../service/resources/apps/AppResource.java | 77 +++- .../subscription/EventSubscriptionMapper.java | 19 +- .../EventSubscriptionResource.java | 35 +- .../resources/apps/AppsResourceTest.java | 120 +++++ .../service/resources/apps/TestApp.java | 11 + .../events/BaseCallbackResource.java | 2 +- .../json/schema/entity/applications/app.json | 4 + .../marketplace/appMarketPlaceDefinition.json | 8 + .../createAppMarketPlaceDefinitionReq.json | 8 + .../events/api/createEventSubscription.json | 4 + .../json/schema/events/eventSubscription.json | 7 + .../resources/json/schema/type/basic.json | 6 + .../src/generated/entity/applications/app.ts | 29 +- .../marketplace/appMarketPlaceDefinition.ts | 421 +++++++++++++++++- .../createAppMarketPlaceDefinitionReq.ts | 421 +++++++++++++++++- .../events/api/createEventSubscription.ts | 13 +- .../src/generated/events/eventSubscription.ts | 13 +- 23 files changed, 1319 insertions(+), 82 deletions(-) create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index c62a0fb85e56..34fd70953492 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -1,5 +1,6 @@ package org.openmetadata.service.apps; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.service.apps.scheduler.AppScheduler.APPS_JOB_GROUP; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_INFO_KEY; import static org.openmetadata.service.apps.scheduler.AppScheduler.APP_NAME; @@ -9,16 +10,29 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.ws.rs.core.Response; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig; import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.events.EventSubscription; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.apps.scheduler.AppScheduler; +import org.openmetadata.service.events.scheduled.EventSubscriptionScheduler; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EventSubscriptionRepository; +import org.openmetadata.service.resources.events.subscription.EventSubscriptionMapper; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.OpenMetadataConnectionBuilder; @@ -34,11 +48,15 @@ public class ApplicationHandler { @Getter private static ApplicationHandler instance; private final OpenMetadataApplicationConfig config; private final AppRepository appRepository; + private final AppMarketPlaceRepository appMarketPlaceRepository; + private final EventSubscriptionRepository eventSubscriptionRepository; private final ConfigurationReader configReader = new ConfigurationReader(); private ApplicationHandler(OpenMetadataApplicationConfig config) { this.config = config; this.appRepository = new AppRepository(); + this.appMarketPlaceRepository = new AppMarketPlaceRepository(); + this.eventSubscriptionRepository = new EventSubscriptionRepository(); } public static void initialize(OpenMetadataApplicationConfig config) { @@ -84,21 +102,106 @@ public void triggerApplicationOnDemand( } public void installApplication( - App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "install"); + App app, CollectionDAO daoCollection, SearchRepository searchRepository, String installedBy) { + try { + runAppInit(app, daoCollection, searchRepository).install(); + installEventSubscriptions(app, installedBy); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException e) { + LOG.error("Failed to install application {}", app.getName(), e); + throw AppException.byMessage( + app.getName(), "install", e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + } + } + + private void installEventSubscriptions(App app, String installedBy) { + AppMarketPlaceDefinition definition = appMarketPlaceRepository.getDefinition(app); + Map eventSubscriptionsReferences = + listOrEmpty(app.getEventSubscriptions()).stream() + .collect(Collectors.toMap(EntityReference::getName, e -> e)); + definition.getEventSubscriptions().stream() + .map( + request -> + Optional.ofNullable(eventSubscriptionsReferences.get(request.getName())) + .flatMap( + sub -> + Optional.ofNullable( + eventSubscriptionRepository.findByNameOrNull( + sub.getName(), Include.ALL))) + .orElseGet( + () -> { + EventSubscription createdEventSub = + eventSubscriptionRepository.create( + null, + // TODO need to get the actual user + new EventSubscriptionMapper() + .createToEntity(request, installedBy)); + appRepository.addEventSubscription(app, createdEventSub); + return createdEventSub; + })) + .forEach( + eventSub -> { + try { + EventSubscriptionScheduler.getInstance().addSubscriptionPublisher(eventSub, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } public void configureApplication( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "configure"); + try { + runAppInit(app, daoCollection, searchRepository).configure(); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException e) { + LOG.error("Failed to configure application {}", app.getName(), e); + throw AppException.byMessage( + app.getName(), "configure", e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + } } public void performCleanup( - App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "cleanup"); + App app, CollectionDAO daoCollection, SearchRepository searchRepository, String deletedBy) { + try { + runAppInit(app, daoCollection, searchRepository).cleanup(); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException e) { + throw new RuntimeException(e); + } + deleteEventSubscriptions(app, deletedBy); } - public Object runAppInit(App app, CollectionDAO daoCollection, SearchRepository searchRepository) + private void deleteEventSubscriptions(App app, String deletedBy) { + listOrEmpty(app.getEventSubscriptions()) + .forEach( + eventSubscriptionReference -> { + try { + EventSubscription eventSub = + eventSubscriptionRepository.find( + eventSubscriptionReference.getId(), Include.ALL); + EventSubscriptionScheduler.getInstance().deleteEventSubscriptionPublisher(eventSub); + eventSubscriptionRepository.delete(deletedBy, eventSub.getId(), false, true); + + } catch (EntityNotFoundException e) { + LOG.debug("Event subscription {} not found", eventSubscriptionReference.getId()); + } catch (SchedulerException e) { + throw new RuntimeException(e); + } + }); + } + + public AbstractNativeApplication runAppInit( + App app, CollectionDAO daoCollection, SearchRepository searchRepository) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, @@ -106,20 +209,17 @@ public Object runAppInit(App app, CollectionDAO daoCollection, SearchRepository IllegalAccessException { // add private runtime properties setAppRuntimeProperties(app); - Class clz = Class.forName(app.getClassName()); - Object resource = + Class clz = + Class.forName(app.getClassName()).asSubclass(AbstractNativeApplication.class); + AbstractNativeApplication resource = clz.getDeclaredConstructor(CollectionDAO.class, SearchRepository.class) .newInstance(daoCollection, searchRepository); - // Raise preview message if the app is in Preview mode if (Boolean.TRUE.equals(app.getPreview())) { - Method preview = resource.getClass().getMethod("raisePreviewMessage", App.class); - preview.invoke(resource, app); + resource.raisePreviewMessage(app); } - // Call init Method - Method initMethod = resource.getClass().getMethod("init", App.class); - initMethod.invoke(resource, app); + resource.init(app); return resource; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java index d76bcafcebeb..b88c0823ec37 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/EventSubscriptionScheduler.java @@ -17,6 +17,7 @@ import static org.openmetadata.service.apps.bundles.changeEvent.AbstractEventConsumer.ALERT_OFFSET_KEY; import static org.openmetadata.service.events.subscription.AlertUtil.getStartingOffset; +import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -37,6 +38,7 @@ import org.openmetadata.schema.entity.events.SubscriptionStatus; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.service.Entity; +import org.openmetadata.service.apps.bundles.changeEvent.AbstractEventConsumer; import org.openmetadata.service.apps.bundles.changeEvent.AlertPublisher; import org.openmetadata.service.events.subscription.AlertUtil; import org.openmetadata.service.jdbi3.EntityRepository; @@ -87,9 +89,23 @@ private static void initialize() throws SchedulerException { } @Transaction - public void addSubscriptionPublisher(EventSubscription eventSubscription) - throws SchedulerException { - AlertPublisher alertPublisher = new AlertPublisher(); + public void addSubscriptionPublisher(EventSubscription eventSubscription, boolean reinstall) + throws SchedulerException, + ClassNotFoundException, + NoSuchMethodException, + InvocationTargetException, + InstantiationException, + IllegalAccessException { + Class defaultClass = AlertPublisher.class; + Class clazz = + Class.forName( + Optional.ofNullable(eventSubscription.getClassName()) + .orElse(defaultClass.getCanonicalName())) + .asSubclass(AbstractEventConsumer.class); + AbstractEventConsumer publisher = clazz.getDeclaredConstructor().newInstance(); + if (reinstall && isSubscriptionRegistered(eventSubscription)) { + deleteEventSubscriptionPublisher(eventSubscription); + } if (Boolean.FALSE.equals( eventSubscription.getEnabled())) { // Only add webhook that is enabled for publishing events eventSubscription @@ -111,7 +127,7 @@ public void addSubscriptionPublisher(EventSubscription eventSubscription) getSubscriptionStatusAtCurrentTime(SubscriptionStatus.Status.ACTIVE))); JobDetail jobDetail = jobBuilder( - alertPublisher, + publisher, eventSubscription, String.format("%s", eventSubscription.getId().toString())); Trigger trigger = trigger(eventSubscription); @@ -127,8 +143,17 @@ public void addSubscriptionPublisher(EventSubscription eventSubscription) } } + public boolean isSubscriptionRegistered(EventSubscription eventSubscription) { + try { + return alertsScheduler.checkExists(getJobKey(eventSubscription)); + } catch (SchedulerException e) { + LOG.error("Failed to check if subscription is registered: {}", eventSubscription.getId(), e); + return false; + } + } + private JobDetail jobBuilder( - AlertPublisher publisher, EventSubscription eventSubscription, String jobIdentity) { + AbstractEventConsumer publisher, EventSubscription eventSubscription, String jobIdentity) { JobDataMap dataMap = new JobDataMap(); dataMap.put(ALERT_INFO_KEY, eventSubscription); dataMap.put(ALERT_OFFSET_KEY, getStartingOffset(eventSubscription.getId())); @@ -158,7 +183,7 @@ public void updateEventSubscription(EventSubscription eventSubscription) { // Remove Existing Subscription Publisher deleteEventSubscriptionPublisher(eventSubscription); if (Boolean.TRUE.equals(eventSubscription.getEnabled())) { - addSubscriptionPublisher(eventSubscription); + addSubscriptionPublisher(eventSubscription, true); } } @@ -536,6 +561,14 @@ public boolean doesRecordExist(UUID id) { return Entity.getCollectionDAO().changeEventDAO().recordExists(id.toString()) > 0; } + public static JobKey getJobKey(EventSubscription eventSubscription) { + return getJobKey(eventSubscription.getId()); + } + + private static JobKey getJobKey(UUID subscriptionId) { + return new JobKey(subscriptionId.toString(), ALERT_JOB_GROUP); + } + public static void shutDown() throws SchedulerException { LOG.info("Shutting Down Event Subscription Scheduler"); if (instance != null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppMarketPlaceRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppMarketPlaceRepository.java index 4405ebbf64eb..066e1e48dd9c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppMarketPlaceRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppMarketPlaceRepository.java @@ -1,6 +1,8 @@ package org.openmetadata.service.jdbi3; +import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.apps.AppMarketPlaceResource; import org.openmetadata.service.util.EntityUtil; @@ -24,6 +26,10 @@ public void setFields(AppMarketPlaceDefinition entity, EntityUtil.Fields fields) /* Nothing to do */ } + public AppMarketPlaceDefinition getDefinition(App app) { + return findByName(app.getName(), Include.NON_DELETED); + } + @Override public void clearFields(AppMarketPlaceDefinition entity, EntityUtil.Fields fields) { /* Nothing to do */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java index 1452249f6673..1cd40337fd89 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AppRepository.java @@ -1,5 +1,6 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.util.UserUtil.getUser; @@ -14,6 +15,7 @@ import org.openmetadata.schema.entity.app.App; import org.openmetadata.schema.entity.app.AppExtension; import org.openmetadata.schema.entity.app.AppRunRecord; +import org.openmetadata.schema.entity.events.EventSubscription; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityReference; @@ -380,6 +382,20 @@ public EntityUpdater getUpdater(App original, App updated, Operation operation) return new AppRepository.AppUpdater(original, updated, operation); } + public void addEventSubscription(App app, EventSubscription eventSubscription) { + addRelationship( + app.getId(), + eventSubscription.getId(), + Entity.APPLICATION, + Entity.EVENT_SUBSCRIPTION, + Relationship.CONTAINS); + List newSubs = new ArrayList<>(listOrEmpty(app.getEventSubscriptions())); + newSubs.add(eventSubscription.getEntityReference()); + App updated = JsonUtils.deepCopy(app, App.class).withEventSubscriptions(newSubs); + updated.setOpenMetadataServerConnection(null); + getUpdater(app, updated, EntityRepository.Operation.PATCH).update(); + } + public class AppUpdater extends EntityUpdater { public AppUpdater(App original, App updated, Operation operation) { super(original, updated, operation); @@ -391,6 +407,8 @@ public void entitySpecificUpdate(boolean consolidatingChanges) { "appConfiguration", original.getAppConfiguration(), updated.getAppConfiguration()); recordChange("appSchedule", original.getAppSchedule(), updated.getAppSchedule()); recordChange("bot", original.getBot(), updated.getBot()); + recordChange( + "eventSubscriptions", original.getEventSubscriptions(), updated.getEventSubscriptions()); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java index 62b941626361..9c4ba01a00a5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java @@ -54,7 +54,8 @@ public App createToEntity(CreateApp createAppRequest, String updatedBy) { .withSourcePythonClass(marketPlaceDefinition.getSourcePythonClass()) .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()) .withSystem(marketPlaceDefinition.getSystem()) - .withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt()); + .withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt()) + .withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName()); // validate Bot if provided validateAndAddBot(app, createAppRequest.getBot()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java index 532cf057b101..f39b72e8081f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceMapper.java @@ -38,7 +38,8 @@ public AppMarketPlaceDefinition createToEntity( .withSourcePythonClass(create.getSourcePythonClass()) .withAllowConfiguration(create.getAllowConfiguration()) .withSystem(create.getSystem()) - .withSupportsInterrupt(create.getSupportsInterrupt()); + .withSupportsInterrupt(create.getSupportsInterrupt()) + .withEventSubscriptions(create.getEventSubscriptions()); // Validate App validateApplication(app); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 4edb03b13059..0bf712ce9f65 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -2,6 +2,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; import static org.openmetadata.service.Entity.APPLICATION; import static org.openmetadata.service.Entity.FIELD_OWNERS; import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; @@ -133,18 +134,19 @@ private void loadDefaultApplications(List defaultAppCreateRequests) { try { App app = getAppForInit(createApp.getName()); if (app == null) { - app = - mapper.createToEntity(createApp, "admin").withFullyQualifiedName(createApp.getName()); + app = mapper.createToEntity(createApp, ADMIN_USER_NAME); repository.initializeEntity(app); } // Schedule if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(app, Entity.getCollectionDAO(), searchRepository); + .installApplication( + app, Entity.getCollectionDAO(), searchRepository, ADMIN_USER_NAME); } } catch (Exception ex) { LOG.error("Failed in Creation/Initialization of Application : {}", createApp.getName(), ex); + repository.deleteByName("admin", createApp.getName(), false, true); } } } @@ -618,9 +620,11 @@ public Response create( new OperationContext(Entity.APPLICATION, MetadataOperation.CREATE)); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(app, Entity.getCollectionDAO(), searchRepository); - ApplicationHandler.getInstance() - .configureApplication(app, Entity.getCollectionDAO(), searchRepository); + .installApplication( + app, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); } // We don't want to store this information unsetAppRuntimeProperties(app); @@ -663,7 +667,11 @@ public Response patchApplication( App updatedApp = (App) response.getEntity(); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(updatedApp, Entity.getCollectionDAO(), searchRepository); + .installApplication( + updatedApp, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); } // We don't want to store this information unsetAppRuntimeProperties(updatedApp); @@ -707,7 +715,11 @@ public Response patchApplication( App updatedApp = (App) response.getEntity(); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(updatedApp, Entity.getCollectionDAO(), searchRepository); + .installApplication( + updatedApp, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); } // We don't want to store this information unsetAppRuntimeProperties(updatedApp); @@ -735,7 +747,11 @@ public Response createOrUpdate( AppScheduler.getInstance().deleteScheduledApplication(app); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(app, Entity.getCollectionDAO(), searchRepository); + .installApplication( + app, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); } // We don't want to store this information unsetAppRuntimeProperties(app); @@ -773,7 +789,11 @@ public Response delete( } ApplicationHandler.getInstance() - .performCleanup(app, Entity.getCollectionDAO(), searchRepository); + .performCleanup( + app, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); limits.invalidateCache(entityType); // Remove from Pipeline Service @@ -810,7 +830,11 @@ public Response delete( } ApplicationHandler.getInstance() - .performCleanup(app, Entity.getCollectionDAO(), searchRepository); + .performCleanup( + app, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); // Remove from Pipeline Service deleteApp(securityContext, app); @@ -842,7 +866,11 @@ public Response restoreApp( App app = (App) response.getEntity(); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(app, Entity.getCollectionDAO(), searchRepository); + .installApplication( + app, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); } // We don't want to store this information unsetAppRuntimeProperties(app); @@ -878,7 +906,12 @@ public Response scheduleApplication( repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() - .installApplication(app, repository.getDaoCollection(), searchRepository); + .installApplication( + app, + repository.getDaoCollection(), + searchRepository, + securityContext.getUserPrincipal().getName()); + return Response.status(Response.Status.OK).entity("App is Scheduled.").build(); } throw new IllegalArgumentException("App is not of schedule type Scheduled."); @@ -912,15 +945,9 @@ public Response configureApplication( repository.getByName(uriInfo, name, new EntityUtil.Fields(repository.getAllowedFields())); // The application will have the updated appConfiguration we can use to run the `configure` // logic - try { - ApplicationHandler.getInstance() - .configureApplication(app, repository.getDaoCollection(), searchRepository); - return Response.status(Response.Status.OK).entity("App has been configured.").build(); - } catch (RuntimeException e) { - return Response.status(Response.Status.INTERNAL_SERVER_ERROR) - .entity(String.format("Error configuring app [%s]", e.getMessage())) - .build(); - } + ApplicationHandler.getInstance() + .configureApplication(app, repository.getDaoCollection(), searchRepository); + return Response.status(Response.Status.OK).entity("App has been configured.").build(); } @POST @@ -1029,7 +1056,11 @@ public Response deployApplicationFlow( App app = repository.getByName(uriInfo, name, fields); if (app.getAppType().equals(AppType.Internal)) { ApplicationHandler.getInstance() - .installApplication(app, Entity.getCollectionDAO(), searchRepository); + .installApplication( + app, + Entity.getCollectionDAO(), + searchRepository, + securityContext.getUserPrincipal().getName()); return Response.status(Response.Status.OK).entity("Application Deployed").build(); } else { if (!app.getPipelines().isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java index 2892c2488030..df694a1ddd97 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionMapper.java @@ -6,10 +6,14 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; +import javax.ws.rs.BadRequestException; import org.openmetadata.schema.api.events.CreateEventSubscription; import org.openmetadata.schema.entity.events.EventSubscription; import org.openmetadata.schema.entity.events.SubscriptionDestination; +import org.openmetadata.service.apps.bundles.changeEvent.AbstractEventConsumer; +import org.openmetadata.service.apps.bundles.changeEvent.AlertPublisher; import org.openmetadata.service.mapper.EntityMapper; public class EventSubscriptionMapper @@ -28,7 +32,20 @@ public EventSubscription createToEntity(CreateEventSubscription create, String u .withProvider(create.getProvider()) .withRetries(create.getRetries()) .withPollInterval(create.getPollInterval()) - .withInput(create.getInput()); + .withInput(create.getInput()) + .withClassName( + validateConsumerClass( + Optional.ofNullable(create.getClassName()) + .orElse(AlertPublisher.class.getCanonicalName()))); + } + + private String validateConsumerClass(String className) { + try { + Class.forName(className).asSubclass(AbstractEventConsumer.class); + return className; + } catch (ClassNotFoundException e) { + throw new BadRequestException("Consumer class not found: " + className); + } } private List getSubscriptions( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java index d81538868c14..d575fbf01c3d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java @@ -27,6 +27,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -143,19 +144,18 @@ public void initialize(OpenMetadataApplicationConfig config) { } private void initializeEventSubscriptions() { - try { - CollectionDAO daoCollection = repository.getDaoCollection(); - List listAllEventsSubscriptions = - daoCollection.eventSubscriptionDAO().listAllEventsSubscriptions(); - List eventSubList = - JsonUtils.readObjects(listAllEventsSubscriptions, EventSubscription.class); - for (EventSubscription subscription : eventSubList) { - EventSubscriptionScheduler.getInstance().addSubscriptionPublisher(subscription); - } - } catch (Exception ex) { - // Starting application should not fail - LOG.warn("Exception during initializeEventSubscriptions", ex); - } + CollectionDAO daoCollection = repository.getDaoCollection(); + daoCollection.eventSubscriptionDAO().listAllEventsSubscriptions().stream() + .map(obj -> JsonUtils.readValue(obj, EventSubscription.class)) + .forEach( + subscription -> { + try { + EventSubscriptionScheduler.getInstance() + .addSubscriptionPublisher(subscription, true); + } catch (Exception ex) { + LOG.error("Failed to initialize subscription: {}", subscription.getId(), ex); + } + }); } @GET @@ -293,12 +293,17 @@ public Response createEventSubscription( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateEventSubscription request) - throws SchedulerException { + throws SchedulerException, + ClassNotFoundException, + InvocationTargetException, + NoSuchMethodException, + InstantiationException, + IllegalAccessException { EventSubscription eventSub = mapper.createToEntity(request, securityContext.getUserPrincipal().getName()); // Only one Creation is allowed Response response = create(uriInfo, securityContext, eventSub); - EventSubscriptionScheduler.getInstance().addSubscriptionPublisher(eventSub); + EventSubscriptionScheduler.getInstance().addSubscriptionPublisher(eventSub, false); return response; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java index e14532595c78..74a47646a44a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java @@ -1,7 +1,10 @@ package org.openmetadata.service.resources.apps; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; +import static org.junit.Assert.assertNotNull; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.schema.type.ColumnDataType.INT; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; @@ -18,6 +21,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -27,6 +31,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.apache.http.util.EntityUtils; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.analytics.PageViewData; @@ -34,23 +40,33 @@ import org.openmetadata.schema.analytics.WebAnalyticEventData; import org.openmetadata.schema.analytics.type.WebAnalyticEventType; import org.openmetadata.schema.api.data.CreateTableProfile; +import org.openmetadata.schema.api.events.CreateEventSubscription; import org.openmetadata.schema.api.services.CreateDatabaseService; import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppConfiguration; import org.openmetadata.schema.entity.app.AppExtension; import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; import org.openmetadata.schema.entity.app.AppRunRecord; import org.openmetadata.schema.entity.app.AppSchedule; +import org.openmetadata.schema.entity.app.AppType; import org.openmetadata.schema.entity.app.CreateApp; import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; +import org.openmetadata.schema.entity.app.NativeAppPermission; import org.openmetadata.schema.entity.app.ScheduleTimeline; +import org.openmetadata.schema.entity.app.ScheduleType; +import org.openmetadata.schema.entity.app.ScheduledExecutionContext; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.events.EventSubscription; import org.openmetadata.schema.entity.services.DatabaseService; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.AccessDetails; +import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.EventType; import org.openmetadata.schema.type.LifeCycle; +import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.TableProfile; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils; @@ -62,6 +78,8 @@ import org.openmetadata.service.resources.databases.DatabaseResourceTest; import org.openmetadata.service.resources.databases.DatabaseSchemaResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.events.BaseCallbackResource; +import org.openmetadata.service.resources.events.EventSubscriptionResourceTest; import org.openmetadata.service.resources.services.DatabaseServiceResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; import org.openmetadata.service.security.SecurityUtil; @@ -384,11 +402,113 @@ void post_trigger_no_trigger_app_400() { "App does not support manual trigger."); } + @SneakyThrows + @Test + void app_with_event_subscription() { + String subscriptionName = "TestEventSubscription"; + // register app in marketplace + EventSubscriptionResourceTest eventSubscriptionResourceTest = + new EventSubscriptionResourceTest(); + CreateAppMarketPlaceDefinitionReq createRequest = + new CreateAppMarketPlaceDefinitionReq() + .withName("TestAppEventSubscription") + .withDisplayName("Test App Event Subscription") + .withDescription("A Test application with event subscriptions.") + .withFeatures("nothing really") + .withDeveloper("Collate Inc.") + .withDeveloperUrl("https://www.example.com") + .withPrivacyPolicyUrl("https://www.example.com/privacy") + .withSupportEmail("support@example.com") + .withClassName("org.openmetadata.service.resources.apps.TestApp") + .withAppType(AppType.Internal) + .withScheduleType(ScheduleType.Scheduled) + .withRuntime(new ScheduledExecutionContext().withEnabled(true)) + .withAppConfiguration(new AppConfiguration()) + .withPermission(NativeAppPermission.All) + .withEventSubscriptions( + List.of( + new CreateEventSubscription() + .withName(subscriptionName) + .withDisplayName("Test Event Subscription") + .withDescription( + "Consume EntityChange Events in order to trigger reverse metadata changes.") + .withAlertType(CreateEventSubscription.AlertType.NOTIFICATION) + .withResources(List.of("all")) + .withProvider(ProviderType.USER) + .withPollInterval(5) + .withEnabled(true))); + String endpoint = + "http://localhost:" + APP.getLocalPort() + "/api/v1/test/webhook/" + subscriptionName; + createRequest + .getEventSubscriptions() + .get(0) + .setDestinations(eventSubscriptionResourceTest.getWebhook(endpoint)); + createAppMarketPlaceDefinition(createRequest, ADMIN_AUTH_HEADERS); + + // install app + CreateApp installApp = + new CreateApp() + .withName(createRequest.getName()) + .withAppConfiguration(new AppConfiguration()); + createEntity(installApp, ADMIN_AUTH_HEADERS); + TestUtils.get(getResource(String.format("events/subscriptions/name/%s", subscriptionName)), EventSubscription.class, ADMIN_AUTH_HEADERS); + + // make change in the system + TableResourceTest tableResourceTest = new TableResourceTest(); + Table table = + tableResourceTest.getEntityByName(TEST_TABLE1.getFullyQualifiedName(), ADMIN_AUTH_HEADERS); + Table updated = JsonUtils.deepCopy(table, Table.class); + updated.setDescription("Updated Description"); + tableResourceTest.patchEntity( + table.getId(), JsonUtils.pojoToJson(table), updated, ADMIN_AUTH_HEADERS); + // assert webhook was called + Awaitility.await() + .timeout( + Duration.ofSeconds(createRequest.getEventSubscriptions().get(0).getPollInterval() + 10)) + .untilAsserted( + () -> { + BaseCallbackResource.EventDetails result = + webhookCallbackResource.getEventDetails(subscriptionName); + Assertions.assertNotNull(result); + Assertions.assertTrue( + result.getEvents().stream() + .anyMatch( + e -> + e.getEventType().equals(EventType.ENTITY_UPDATED) + && e.getChangeDescription() + .getFieldsUpdated() + .get(0) + .getNewValue() + .equals("Updated Description"))); + }); + // uninstall app + deleteEntityByName(installApp.getName(), true, true, ADMIN_AUTH_HEADERS); + Table updated2 = JsonUtils.deepCopy(updated, Table.class); + updated2.setDescription("Updated Description 2"); + tableResourceTest.patchEntity( + table.getId(), JsonUtils.pojoToJson(table), updated2, ADMIN_AUTH_HEADERS); + + // assert event subscription was deleted + TestUtils.assertResponse( + () ->TestUtils.get(getResource(String.format("events/subscriptions/name/%s", subscriptionName)), EventSubscription.class, ADMIN_AUTH_HEADERS), + NOT_FOUND, + String.format("eventsubscription instance for %s not found", subscriptionName) + ); + } + @Override public void validateCreatedEntity( App createdEntity, CreateApp request, Map authHeaders) throws HttpResponseException {} + public void createAppMarketPlaceDefinition( + CreateAppMarketPlaceDefinitionReq create, Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("apps/marketplace"); + TestUtils.post( + target, create, AppMarketPlaceDefinition.class, CREATED.getStatusCode(), authHeaders); + } + @Override public void compareEntities(App expected, App updated, Map authHeaders) throws HttpResponseException {} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java new file mode 100644 index 000000000000..9fae7a62005f --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java @@ -0,0 +1,11 @@ +package org.openmetadata.service.resources.apps; + +import org.openmetadata.service.apps.AbstractNativeApplication; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.SearchRepository; + +public class TestApp extends AbstractNativeApplication { + public TestApp(CollectionDAO collectionDAO, SearchRepository searchRepository) { + super(collectionDAO, searchRepository); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/BaseCallbackResource.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/BaseCallbackResource.java index a20259aa2cf5..2ed6d3914d41 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/BaseCallbackResource.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/events/BaseCallbackResource.java @@ -141,7 +141,7 @@ public void clearEvents() { entityCallbackMap.clear(); } - static class EventDetails { + public static class EventDetails { @Getter @Setter long firstEventTime; @Getter @Setter long latestEventTime; @Getter final ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>(); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json index f4092bf15e27..191988cd5981 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/app.json @@ -255,6 +255,10 @@ "description": "If the app run can be interrupted as part of the execution.", "type": "boolean", "default": false + }, + "eventSubscriptions": { + "description": "Event Subscriptions for the Application.", + "$ref": "../../type/entityReferenceList.json" } }, "additionalProperties": false, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json index 12644408cd58..542f3ac710a2 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/appMarketPlaceDefinition.json @@ -149,6 +149,14 @@ "description": "If the app run can be interrupted as part of the execution.", "type": "boolean", "default": false + }, + "eventSubscriptions": { + "description": "Event subscriptions that will be created when the application is installed.", + "type": "array", + "default": [], + "items": { + "$ref": "../../../events/api/createEventSubscription.json" + } } }, "additionalProperties": false, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json index b5ebcbfc4e20..380466a610fa 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.json @@ -111,6 +111,14 @@ "description": "If the app run can be interrupted as part of the execution.", "type": "boolean", "default": false + }, + "eventSubscriptions": { + "description": "Event subscriptions that will be created when the application is installed.", + "type": "array", + "default": [], + "items": { + "$ref": "../../../events/api/createEventSubscription.json" + } } }, "additionalProperties": false, diff --git a/openmetadata-spec/src/main/resources/json/schema/events/api/createEventSubscription.json b/openmetadata-spec/src/main/resources/json/schema/events/api/createEventSubscription.json index 92f31aed1554..d21e07f750b5 100644 --- a/openmetadata-spec/src/main/resources/json/schema/events/api/createEventSubscription.json +++ b/openmetadata-spec/src/main/resources/json/schema/events/api/createEventSubscription.json @@ -11,6 +11,10 @@ "description": "Name that uniquely identifies this Alert.", "$ref": "../../type/basic.json#/definitions/entityName" }, + "className": { + "description": "Consumer Class for the Event Subscription. Will use 'AlertPublisher' if not provided.", + "type": "string" + }, "displayName": { "description": "Display name for this Alert.", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json b/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json index b295c158a7e0..596536916e50 100644 --- a/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json +++ b/openmetadata-spec/src/main/resources/json/schema/events/eventSubscription.json @@ -155,6 +155,9 @@ }, { "$ref": "./emailAlertConfig.json" + }, + { + "$ref": "../type/basic.json#/definitions/map" } ] } @@ -231,6 +234,10 @@ "description": "Unique identifier that identifies this Event Subscription.", "$ref": "../type/basic.json#/definitions/uuid" }, + "className": { + "description": "Java class for the Event Subscription.", + "type": "string" + }, "name": { "description": "Name that uniquely identifies this Event Subscription.", "$ref": "../type/basic.json#/definitions/entityName" diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index 314954800e07..fd732ad7e1c7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -173,6 +173,12 @@ ".{1,}": { "type": "string" } } }, + "map": { + "description": "A generic map that can be deserialized later.", + "existingJavaType" : "java.util.Map", + "type" : "object", + "additionalProperties": true + }, "status" : { "javaType": "org.openmetadata.schema.type.ApiStatus", "description": "State of an action over API.", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts index fc014f1bd1f7..47e915123f43 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This schema defines the applications for Open-Metadata. */ export interface App { @@ -78,6 +76,10 @@ export interface App { * it belongs to. */ domain?: EntityReference; + /** + * Event Subscriptions for the Application. + */ + eventSubscriptions?: EntityReference[]; /** * Features of the Application. */ @@ -283,8 +285,12 @@ export interface CollateAIAppConfig { * * Remove Owner Action Type * + * Add a Custom Property to the selected assets. + * * Add owners to the selected assets. * + * Remove Custom Properties Action Type + * * Propagate description, tags and glossary terms via lineage * * ML Tagging action configuration for external automator. @@ -314,6 +320,9 @@ export interface Action { * Update the description even if they are already defined in the asset. By default, we'll * only add the descriptions to assets without the description set. * + * Update the Custom Property even if it is defined in the asset. By default, we will only + * apply the owners to assets without the given Custom Property informed. + * * Update the tier even if it is defined in the asset. By default, we will only apply the * tier to assets without tier. * @@ -343,6 +352,12 @@ export interface Action { * Description to apply */ description?: string; + /** + * Owners to apply + * + * Custom Properties keys to remove + */ + customProperties?: any; /** * tier to apply */ @@ -546,6 +561,8 @@ export interface Style { * * Add Description Action Type. * + * Add Custom Properties Action Type. + * * Remove Description Action Type * * Add Tier Action Type. @@ -554,11 +571,14 @@ export interface Style { * * Remove Owner Action Type * + * Remove Custom Properties Action Type. + * * Lineage propagation action type. * * ML PII Tagging action type. */ export enum ActionType { + AddCustomPropertiesAction = "AddCustomPropertiesAction", AddDescriptionAction = "AddDescriptionAction", AddDomainAction = "AddDomainAction", AddOwnerAction = "AddOwnerAction", @@ -566,6 +586,7 @@ export enum ActionType { AddTierAction = "AddTierAction", LineagePropagationAction = "LineagePropagationAction", MLTaggingAction = "MLTaggingAction", + RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", RemoveDescriptionAction = "RemoveDescriptionAction", RemoveDomainAction = "RemoveDomainAction", RemoveOwnerAction = "RemoveOwnerAction", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts index 50aefad7e2ad..4b96f0b54107 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This schema defines the applications for Open-Metadata. */ export interface AppMarketPlaceDefinition { @@ -70,6 +68,10 @@ export interface AppMarketPlaceDefinition { * it belongs to. */ domain?: EntityReference; + /** + * Event subscriptions that will be created when the application is installed. + */ + eventSubscriptions?: CreateEventSubscription[]; /** * Features of the Application. */ @@ -268,8 +270,12 @@ export interface CollateAIAppConfig { * * Remove Owner Action Type * + * Add a Custom Property to the selected assets. + * * Add owners to the selected assets. * + * Remove Custom Properties Action Type + * * Propagate description, tags and glossary terms via lineage * * ML Tagging action configuration for external automator. @@ -299,6 +305,9 @@ export interface Action { * Update the description even if they are already defined in the asset. By default, we'll * only add the descriptions to assets without the description set. * + * Update the Custom Property even if it is defined in the asset. By default, we will only + * apply the owners to assets without the given Custom Property informed. + * * Update the tier even if it is defined in the asset. By default, we will only apply the * tier to assets without tier. * @@ -328,6 +337,12 @@ export interface Action { * Description to apply */ description?: string; + /** + * Owners to apply + * + * Custom Properties keys to remove + */ + customProperties?: any; /** * tier to apply */ @@ -529,6 +544,8 @@ export interface Style { * * Add Description Action Type. * + * Add Custom Properties Action Type. + * * Remove Description Action Type * * Add Tier Action Type. @@ -537,11 +554,14 @@ export interface Style { * * Remove Owner Action Type * + * Remove Custom Properties Action Type. + * * Lineage propagation action type. * * ML PII Tagging action type. */ export enum ActionType { + AddCustomPropertiesAction = "AddCustomPropertiesAction", AddDescriptionAction = "AddDescriptionAction", AddDomainAction = "AddDomainAction", AddOwnerAction = "AddOwnerAction", @@ -549,6 +569,7 @@ export enum ActionType { AddTierAction = "AddTierAction", LineagePropagationAction = "LineagePropagationAction", MLTaggingAction = "MLTaggingAction", + RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", RemoveDescriptionAction = "RemoveDescriptionAction", RemoveDomainAction = "RemoveDomainAction", RemoveOwnerAction = "RemoveOwnerAction", @@ -665,6 +686,398 @@ export interface FieldChange { oldValue?: any; } +/** + * This defines schema for sending alerts for OpenMetadata + */ +export interface CreateEventSubscription { + /** + * Type of Alert + */ + alertType: AlertType; + /** + * Maximum number of events sent in a batch (Default 10). + */ + batchSize?: number; + /** + * Consumer Class for the Event Subscription. Will use 'AlertPublisher' if not provided. + */ + className?: string; + /** + * A short description of the Alert, comprehensible to regular users. + */ + description?: string; + /** + * Subscription Config. + */ + destinations?: Destination[]; + /** + * Display name for this Alert. + */ + displayName?: string; + /** + * Fully qualified name of the domain the Table belongs to. + */ + domain?: string; + /** + * Is the alert enabled. + */ + enabled?: boolean; + /** + * Input for the Filters. + */ + input?: AlertFilteringInput; + /** + * Name that uniquely identifies this Alert. + */ + name: string; + /** + * Owners of this Alert. + */ + owners?: EntityReference[]; + /** + * Poll Interval in seconds. + */ + pollInterval?: number; + provider?: ProviderType; + /** + * Defines a list of resources that triggers the Event Subscription, Eg All, User, Teams etc. + */ + resources?: string[]; + /** + * Number of times to retry callback on failure. (Default 3). + */ + retries?: number; + trigger?: Trigger; +} + +/** + * Type of Alert + * + * Type of Alerts supported. + */ +export enum AlertType { + ActivityFeed = "ActivityFeed", + GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", + Notification = "Notification", + Observability = "Observability", +} + +/** + * Subscription which has a type and the config. + */ +export interface Destination { + category: SubscriptionCategory; + config?: Webhook; + /** + * Is the subscription enabled. + */ + enabled?: boolean; + /** + * Unique identifier that identifies this Event Subscription. + */ + id?: string; + /** + * Read timeout in seconds. (Default 12s). + */ + readTimeout?: number; + statusDetails?: TionStatus; + /** + * Connection timeout in seconds. (Default 10s). + */ + timeout?: number; + type: SubscriptionType; +} + +/** + * Subscription Endpoint Type. + */ +export enum SubscriptionCategory { + Admins = "Admins", + Assignees = "Assignees", + External = "External", + Followers = "Followers", + Mentions = "Mentions", + Owners = "Owners", + Teams = "Teams", + Users = "Users", +} + +/** + * This schema defines webhook for receiving events from OpenMetadata. + * + * This schema defines email config for receiving events from OpenMetadata. + * + * A generic map that can be deserialized later. + */ +export interface Webhook { + /** + * Endpoint to receive the webhook events over POST requests. + */ + endpoint?: string; + /** + * Custom headers to be sent with the webhook request. + */ + headers?: { [key: string]: any }; + /** + * HTTP operation to send the webhook request. Supports POST or PUT. + */ + httpMethod?: HTTPMethod; + /** + * List of receivers to send mail to + */ + receivers?: string[]; + /** + * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook + * payload and sent in `X-OM-Signature` header in POST requests to publish the events. + */ + secretKey?: string; + /** + * Send the Event to Admins + * + * Send the Mails to Admins + */ + sendToAdmins?: boolean; + /** + * Send the Event to Followers + * + * Send the Mails to Followers + */ + sendToFollowers?: boolean; + /** + * Send the Event to Owners + * + * Send the Mails to Owners + */ + sendToOwners?: boolean; + [property: string]: any; +} + +/** + * HTTP operation to send the webhook request. Supports POST or PUT. + */ +export enum HTTPMethod { + Post = "POST", + Put = "PUT", +} + +/** + * Current status of the subscription, including details on the last successful and failed + * attempts, and retry information. + * + * Detailed status of the destination during a test operation, including HTTP response + * information. + */ +export interface TionStatus { + /** + * Timestamp of the last failed callback in UNIX UTC epoch time in milliseconds. + */ + lastFailedAt?: number; + /** + * Detailed reason for the last failure received during callback. + */ + lastFailedReason?: string; + /** + * HTTP status code received during the last failed callback attempt. + */ + lastFailedStatusCode?: number; + /** + * Timestamp of the last successful callback in UNIX UTC epoch time in milliseconds. + */ + lastSuccessfulAt?: number; + /** + * Timestamp for the next retry attempt in UNIX epoch time in milliseconds. Only valid if + * `status` is `awaitingRetry`. + */ + nextAttempt?: number; + /** + * Status is `disabled` when the event subscription was created with `enabled` set to false + * and it never started publishing events. Status is `active` when the event subscription is + * functioning normally and a 200 OK response was received for the callback notification. + * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` + * response was received for the callback notification. Status is `awaitingRetry` when the + * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is + * `retryLimitReached` after all retries fail. + * + * Overall test status, indicating if the test operation succeeded or failed. + */ + status?: Status; + /** + * Current timestamp of this status in UNIX epoch time in milliseconds. + * + * Timestamp when the response was received, in UNIX epoch time milliseconds. + */ + timestamp?: number; + /** + * Body of the HTTP response, if any, returned by the server. + */ + entity?: string; + /** + * HTTP headers returned in the response as a map of header names to values. + */ + headers?: any; + /** + * URL location if the response indicates a redirect or newly created resource. + */ + location?: string; + /** + * Media type of the response entity, if specified (e.g., application/json). + */ + mediaType?: string; + /** + * Detailed reason for failure if the test did not succeed. + */ + reason?: string; + /** + * HTTP status code of the response (e.g., 200 for OK, 404 for Not Found). + */ + statusCode?: number; + /** + * HTTP status reason phrase associated with the status code (e.g., 'Not Found'). + */ + statusInfo?: string; +} + +/** + * Status is `disabled` when the event subscription was created with `enabled` set to false + * and it never started publishing events. Status is `active` when the event subscription is + * functioning normally and a 200 OK response was received for the callback notification. + * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` + * response was received for the callback notification. Status is `awaitingRetry` when the + * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is + * `retryLimitReached` after all retries fail. + * + * Overall test status, indicating if the test operation succeeded or failed. + */ +export enum Status { + Active = "active", + AwaitingRetry = "awaitingRetry", + Disabled = "disabled", + Failed = "failed", + RetryLimitReached = "retryLimitReached", + StatusFailed = "Failed", + Success = "Success", +} + +/** + * Subscription Endpoint Type. + */ +export enum SubscriptionType { + ActivityFeed = "ActivityFeed", + Email = "Email", + GChat = "GChat", + GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", + MSTeams = "MsTeams", + Slack = "Slack", + Webhook = "Webhook", +} + +/** + * Input for the Filters. + * + * Observability of the event subscription. + */ +export interface AlertFilteringInput { + /** + * List of filters for the event subscription. + */ + actions?: ArgumentsInput[]; + /** + * List of filters for the event subscription. + */ + filters?: ArgumentsInput[]; +} + +/** + * Observability Filters for Event Subscription. + */ +export interface ArgumentsInput { + /** + * Arguments List + */ + arguments?: Argument[]; + effect?: Effect; + /** + * Name of the filter + */ + name?: string; + /** + * Prefix Condition for the filter. + */ + prefixCondition?: PrefixCondition; +} + +/** + * Argument for the filter. + */ +export interface Argument { + /** + * Value of the Argument + */ + input?: string[]; + /** + * Name of the Argument + */ + name?: string; +} + +export enum Effect { + Exclude = "exclude", + Include = "include", +} + +/** + * Prefix Condition for the filter. + * + * Prefix Condition to be applied to the Condition. + */ +export enum PrefixCondition { + And = "AND", + Or = "OR", +} + +/** + * Type of provider of an entity. Some entities are provided by the `system`. Some are + * entities created and provided by the `user`. Typically `system` provide entities can't be + * deleted and can only be disabled. + */ +export enum ProviderType { + System = "system", + User = "user", +} + +/** + * Trigger Configuration for Alerts. + */ +export interface Trigger { + /** + * Cron Expression in case of Custom scheduled Trigger + */ + cronExpression?: string; + /** + * Schedule Info + */ + scheduleInfo?: ScheduleInfo; + triggerType: TriggerType; +} + +/** + * Schedule Info + */ +export enum ScheduleInfo { + Custom = "Custom", + Daily = "Daily", + Monthly = "Monthly", + Weekly = "Weekly", +} + +/** + * Trigger Configuration for Alerts. + */ +export enum TriggerType { + RealTime = "RealTime", + Scheduled = "Scheduled", +} + /** * Permission used by Native Applications. * diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts index 2d86c739d4cd..90c023ea2204 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This schema defines the applications for Open-Metadata. */ export interface CreateAppMarketPlaceDefinitionReq { @@ -61,6 +59,10 @@ export interface CreateAppMarketPlaceDefinitionReq { * Fully qualified name of the domain the Table belongs to. */ domain?: string; + /** + * Event subscriptions that will be created when the application is installed. + */ + eventSubscriptions?: CreateEventSubscription[]; /** * Features of the Application. */ @@ -229,8 +231,12 @@ export interface CollateAIAppConfig { * * Remove Owner Action Type * + * Add a Custom Property to the selected assets. + * * Add owners to the selected assets. * + * Remove Custom Properties Action Type + * * Propagate description, tags and glossary terms via lineage * * ML Tagging action configuration for external automator. @@ -260,6 +266,9 @@ export interface Action { * Update the description even if they are already defined in the asset. By default, we'll * only add the descriptions to assets without the description set. * + * Update the Custom Property even if it is defined in the asset. By default, we will only + * apply the owners to assets without the given Custom Property informed. + * * Update the tier even if it is defined in the asset. By default, we will only apply the * tier to assets without tier. * @@ -289,6 +298,12 @@ export interface Action { * Description to apply */ description?: string; + /** + * Owners to apply + * + * Custom Properties keys to remove + */ + customProperties?: any; /** * tier to apply */ @@ -487,6 +502,8 @@ export interface Style { * * Add Description Action Type. * + * Add Custom Properties Action Type. + * * Remove Description Action Type * * Add Tier Action Type. @@ -495,11 +512,14 @@ export interface Style { * * Remove Owner Action Type * + * Remove Custom Properties Action Type. + * * Lineage propagation action type. * * ML PII Tagging action type. */ export enum ActionType { + AddCustomPropertiesAction = "AddCustomPropertiesAction", AddDescriptionAction = "AddDescriptionAction", AddDomainAction = "AddDomainAction", AddOwnerAction = "AddOwnerAction", @@ -507,6 +527,7 @@ export enum ActionType { AddTierAction = "AddTierAction", LineagePropagationAction = "LineagePropagationAction", MLTaggingAction = "MLTaggingAction", + RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", RemoveDescriptionAction = "RemoveDescriptionAction", RemoveDomainAction = "RemoveDomainAction", RemoveOwnerAction = "RemoveOwnerAction", @@ -582,6 +603,398 @@ export enum AppType { Internal = "internal", } +/** + * This defines schema for sending alerts for OpenMetadata + */ +export interface CreateEventSubscription { + /** + * Type of Alert + */ + alertType: AlertType; + /** + * Maximum number of events sent in a batch (Default 10). + */ + batchSize?: number; + /** + * Consumer Class for the Event Subscription. Will use 'AlertPublisher' if not provided. + */ + className?: string; + /** + * A short description of the Alert, comprehensible to regular users. + */ + description?: string; + /** + * Subscription Config. + */ + destinations?: Destination[]; + /** + * Display name for this Alert. + */ + displayName?: string; + /** + * Fully qualified name of the domain the Table belongs to. + */ + domain?: string; + /** + * Is the alert enabled. + */ + enabled?: boolean; + /** + * Input for the Filters. + */ + input?: AlertFilteringInput; + /** + * Name that uniquely identifies this Alert. + */ + name: string; + /** + * Owners of this Alert. + */ + owners?: EntityReference[]; + /** + * Poll Interval in seconds. + */ + pollInterval?: number; + provider?: ProviderType; + /** + * Defines a list of resources that triggers the Event Subscription, Eg All, User, Teams etc. + */ + resources?: string[]; + /** + * Number of times to retry callback on failure. (Default 3). + */ + retries?: number; + trigger?: Trigger; +} + +/** + * Type of Alert + * + * Type of Alerts supported. + */ +export enum AlertType { + ActivityFeed = "ActivityFeed", + GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", + Notification = "Notification", + Observability = "Observability", +} + +/** + * Subscription which has a type and the config. + */ +export interface Destination { + category: SubscriptionCategory; + config?: Webhook; + /** + * Is the subscription enabled. + */ + enabled?: boolean; + /** + * Unique identifier that identifies this Event Subscription. + */ + id?: string; + /** + * Read timeout in seconds. (Default 12s). + */ + readTimeout?: number; + statusDetails?: TionStatus; + /** + * Connection timeout in seconds. (Default 10s). + */ + timeout?: number; + type: SubscriptionType; +} + +/** + * Subscription Endpoint Type. + */ +export enum SubscriptionCategory { + Admins = "Admins", + Assignees = "Assignees", + External = "External", + Followers = "Followers", + Mentions = "Mentions", + Owners = "Owners", + Teams = "Teams", + Users = "Users", +} + +/** + * This schema defines webhook for receiving events from OpenMetadata. + * + * This schema defines email config for receiving events from OpenMetadata. + * + * A generic map that can be deserialized later. + */ +export interface Webhook { + /** + * Endpoint to receive the webhook events over POST requests. + */ + endpoint?: string; + /** + * Custom headers to be sent with the webhook request. + */ + headers?: { [key: string]: any }; + /** + * HTTP operation to send the webhook request. Supports POST or PUT. + */ + httpMethod?: HTTPMethod; + /** + * List of receivers to send mail to + */ + receivers?: string[]; + /** + * Secret set by the webhook client used for computing HMAC SHA256 signature of webhook + * payload and sent in `X-OM-Signature` header in POST requests to publish the events. + */ + secretKey?: string; + /** + * Send the Event to Admins + * + * Send the Mails to Admins + */ + sendToAdmins?: boolean; + /** + * Send the Event to Followers + * + * Send the Mails to Followers + */ + sendToFollowers?: boolean; + /** + * Send the Event to Owners + * + * Send the Mails to Owners + */ + sendToOwners?: boolean; + [property: string]: any; +} + +/** + * HTTP operation to send the webhook request. Supports POST or PUT. + */ +export enum HTTPMethod { + Post = "POST", + Put = "PUT", +} + +/** + * Current status of the subscription, including details on the last successful and failed + * attempts, and retry information. + * + * Detailed status of the destination during a test operation, including HTTP response + * information. + */ +export interface TionStatus { + /** + * Timestamp of the last failed callback in UNIX UTC epoch time in milliseconds. + */ + lastFailedAt?: number; + /** + * Detailed reason for the last failure received during callback. + */ + lastFailedReason?: string; + /** + * HTTP status code received during the last failed callback attempt. + */ + lastFailedStatusCode?: number; + /** + * Timestamp of the last successful callback in UNIX UTC epoch time in milliseconds. + */ + lastSuccessfulAt?: number; + /** + * Timestamp for the next retry attempt in UNIX epoch time in milliseconds. Only valid if + * `status` is `awaitingRetry`. + */ + nextAttempt?: number; + /** + * Status is `disabled` when the event subscription was created with `enabled` set to false + * and it never started publishing events. Status is `active` when the event subscription is + * functioning normally and a 200 OK response was received for the callback notification. + * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` + * response was received for the callback notification. Status is `awaitingRetry` when the + * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is + * `retryLimitReached` after all retries fail. + * + * Overall test status, indicating if the test operation succeeded or failed. + */ + status?: Status; + /** + * Current timestamp of this status in UNIX epoch time in milliseconds. + * + * Timestamp when the response was received, in UNIX epoch time milliseconds. + */ + timestamp?: number; + /** + * Body of the HTTP response, if any, returned by the server. + */ + entity?: string; + /** + * HTTP headers returned in the response as a map of header names to values. + */ + headers?: any; + /** + * URL location if the response indicates a redirect or newly created resource. + */ + location?: string; + /** + * Media type of the response entity, if specified (e.g., application/json). + */ + mediaType?: string; + /** + * Detailed reason for failure if the test did not succeed. + */ + reason?: string; + /** + * HTTP status code of the response (e.g., 200 for OK, 404 for Not Found). + */ + statusCode?: number; + /** + * HTTP status reason phrase associated with the status code (e.g., 'Not Found'). + */ + statusInfo?: string; +} + +/** + * Status is `disabled` when the event subscription was created with `enabled` set to false + * and it never started publishing events. Status is `active` when the event subscription is + * functioning normally and a 200 OK response was received for the callback notification. + * Status is `failed` when a bad callback URL, connection failures, or `1xx` or `3xx` + * response was received for the callback notification. Status is `awaitingRetry` when the + * previous attempt at callback timed out or received a `4xx` or `5xx` response. Status is + * `retryLimitReached` after all retries fail. + * + * Overall test status, indicating if the test operation succeeded or failed. + */ +export enum Status { + Active = "active", + AwaitingRetry = "awaitingRetry", + Disabled = "disabled", + Failed = "failed", + RetryLimitReached = "retryLimitReached", + StatusFailed = "Failed", + Success = "Success", +} + +/** + * Subscription Endpoint Type. + */ +export enum SubscriptionType { + ActivityFeed = "ActivityFeed", + Email = "Email", + GChat = "GChat", + GovernanceWorkflowChangeEvent = "GovernanceWorkflowChangeEvent", + MSTeams = "MsTeams", + Slack = "Slack", + Webhook = "Webhook", +} + +/** + * Input for the Filters. + * + * Observability of the event subscription. + */ +export interface AlertFilteringInput { + /** + * List of filters for the event subscription. + */ + actions?: ArgumentsInput[]; + /** + * List of filters for the event subscription. + */ + filters?: ArgumentsInput[]; +} + +/** + * Observability Filters for Event Subscription. + */ +export interface ArgumentsInput { + /** + * Arguments List + */ + arguments?: Argument[]; + effect?: Effect; + /** + * Name of the filter + */ + name?: string; + /** + * Prefix Condition for the filter. + */ + prefixCondition?: PrefixCondition; +} + +/** + * Argument for the filter. + */ +export interface Argument { + /** + * Value of the Argument + */ + input?: string[]; + /** + * Name of the Argument + */ + name?: string; +} + +export enum Effect { + Exclude = "exclude", + Include = "include", +} + +/** + * Prefix Condition for the filter. + * + * Prefix Condition to be applied to the Condition. + */ +export enum PrefixCondition { + And = "AND", + Or = "OR", +} + +/** + * Type of provider of an entity. Some entities are provided by the `system`. Some are + * entities created and provided by the `user`. Typically `system` provide entities can't be + * deleted and can only be disabled. + */ +export enum ProviderType { + System = "system", + User = "user", +} + +/** + * Trigger Configuration for Alerts. + */ +export interface Trigger { + /** + * Cron Expression in case of Custom scheduled Trigger + */ + cronExpression?: string; + /** + * Schedule Info + */ + scheduleInfo?: ScheduleInfo; + triggerType: TriggerType; +} + +/** + * Schedule Info + */ +export enum ScheduleInfo { + Custom = "Custom", + Daily = "Daily", + Monthly = "Monthly", + Weekly = "Weekly", +} + +/** + * Trigger Configuration for Alerts. + */ +export enum TriggerType { + RealTime = "RealTime", + Scheduled = "Scheduled", +} + /** * Permission used by Native Applications. * diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts b/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts index 21a6ccd700e8..d7a75280f013 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/events/api/createEventSubscription.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This defines schema for sending alerts for OpenMetadata */ export interface CreateEventSubscription { @@ -24,6 +22,10 @@ export interface CreateEventSubscription { * Maximum number of events sent in a batch (Default 10). */ batchSize?: number; + /** + * Consumer Class for the Event Subscription. Will use 'AlertPublisher' if not provided. + */ + className?: string; /** * A short description of the Alert, comprehensible to regular users. */ @@ -128,6 +130,8 @@ export enum SubscriptionCategory { * This schema defines webhook for receiving events from OpenMetadata. * * This schema defines email config for receiving events from OpenMetadata. + * + * A generic map that can be deserialized later. */ export interface Webhook { /** @@ -169,6 +173,7 @@ export interface Webhook { * Send the Mails to Owners */ sendToOwners?: boolean; + [property: string]: any; } /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts b/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts index 01dbd8d71edc..6f658bb19ff4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/events/eventSubscription.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * This schema defines the EventSubscription entity. An Event Subscription has trigger, * filters and Subscription */ @@ -29,6 +27,10 @@ export interface EventSubscription { * Change that led to this version of the Event Subscription. */ changeDescription?: ChangeDescription; + /** + * Java class for the Event Subscription. + */ + className?: string; /** * A short description of the Event Subscription, comprehensible to regular users. */ @@ -204,6 +206,8 @@ export enum SubscriptionCategory { * This schema defines webhook for receiving events from OpenMetadata. * * This schema defines email config for receiving events from OpenMetadata. + * + * A generic map that can be deserialized later. */ export interface Webhook { /** @@ -245,6 +249,7 @@ export interface Webhook { * Send the Mails to Owners */ sendToOwners?: boolean; + [property: string]: any; } /** From 787a36791dfdb6b1c7a043e90f30fe47a7db1f1d Mon Sep 17 00:00:00 2001 From: sushi30 Date: Mon, 13 Jan 2025 17:41:20 +0200 Subject: [PATCH 2/8] removed runMethodFromApplication reflection and use concrete methods from AbstractNativeApplication --- .../service/apps/ApplicationHandler.java | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 34fd70953492..1877dff73599 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -98,7 +98,17 @@ public Boolean isPreview(String appName) { public void triggerApplicationOnDemand( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - runMethodFromApplication(app, daoCollection, searchRepository, "triggerOnDemand"); + try { + runAppInit(app, daoCollection, searchRepository).triggerOnDemand(); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException e) { + LOG.error("Failed to install application {}", app.getName(), e); + throw AppException.byMessage( + app.getName(), "triggerOnDemand", e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + } } public void installApplication( @@ -224,29 +234,6 @@ public AbstractNativeApplication runAppInit( return resource; } - /** - * Load an App from its className and call its methods dynamically - */ - public void runMethodFromApplication( - App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) { - // Native Application - setAppRuntimeProperties(app); - try { - Object resource = runAppInit(app, daoCollection, searchRepository); - // Call method on demand - Method scheduleMethod = resource.getClass().getMethod(methodName); - scheduleMethod.invoke(resource); - - } catch (NoSuchMethodException | InstantiationException | IllegalAccessException e) { - LOG.error("Exception encountered", e); - throw new UnhandledServerException(e.getMessage()); - } catch (ClassNotFoundException e) { - throw new UnhandledServerException(e.getMessage()); - } catch (InvocationTargetException e) { - throw AppException.byMessage(app.getName(), methodName, e.getTargetException().getMessage()); - } - } - public void migrateQuartzConfig(App application) throws SchedulerException { JobDetail jobDetails = AppScheduler.getInstance() From 567ae6d751275109f10f5ace33e94c1df687148c Mon Sep 17 00:00:00 2001 From: sushi30 Date: Mon, 13 Jan 2025 17:48:16 +0200 Subject: [PATCH 3/8] format --- .../service/apps/ApplicationHandler.java | 24 +++++++++---------- .../service/resources/apps/AppMapper.java | 2 +- .../resources/apps/AppsResourceTest.java | 17 ++++++++----- .../service/resources/apps/TestApp.java | 6 ++--- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java index 1877dff73599..8dba089bbb47 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java @@ -8,7 +8,6 @@ import io.dropwizard.configuration.ConfigurationException; import java.io.IOException; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Collection; import java.util.Map; import java.util.Optional; @@ -26,7 +25,6 @@ import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.events.scheduled.EventSubscriptionScheduler; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; @@ -98,17 +96,17 @@ public Boolean isPreview(String appName) { public void triggerApplicationOnDemand( App app, CollectionDAO daoCollection, SearchRepository searchRepository) { - try { - runAppInit(app, daoCollection, searchRepository).triggerOnDemand(); - } catch (ClassNotFoundException - | NoSuchMethodException - | InvocationTargetException - | InstantiationException - | IllegalAccessException e) { - LOG.error("Failed to install application {}", app.getName(), e); - throw AppException.byMessage( - app.getName(), "triggerOnDemand", e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); - } + try { + runAppInit(app, daoCollection, searchRepository).triggerOnDemand(); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException e) { + LOG.error("Failed to install application {}", app.getName(), e); + throw AppException.byMessage( + app.getName(), "triggerOnDemand", e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + } } public void installApplication( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java index 9c4ba01a00a5..14c873ab8d63 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMapper.java @@ -55,7 +55,7 @@ public App createToEntity(CreateApp createAppRequest, String updatedBy) { .withAllowConfiguration(marketPlaceDefinition.getAllowConfiguration()) .withSystem(marketPlaceDefinition.getSystem()) .withSupportsInterrupt(marketPlaceDefinition.getSupportsInterrupt()) - .withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName()); + .withFullyQualifiedName(marketPlaceDefinition.getFullyQualifiedName()); // validate Bot if provided validateAndAddBot(app, createAppRequest.getBot()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java index 74a47646a44a..f4dcb06a6d37 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/AppsResourceTest.java @@ -4,7 +4,6 @@ import static javax.ws.rs.core.Response.Status.CREATED; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; -import static org.junit.Assert.assertNotNull; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.schema.type.ColumnDataType.INT; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; @@ -451,7 +450,10 @@ void app_with_event_subscription() { .withName(createRequest.getName()) .withAppConfiguration(new AppConfiguration()); createEntity(installApp, ADMIN_AUTH_HEADERS); - TestUtils.get(getResource(String.format("events/subscriptions/name/%s", subscriptionName)), EventSubscription.class, ADMIN_AUTH_HEADERS); + TestUtils.get( + getResource(String.format("events/subscriptions/name/%s", subscriptionName)), + EventSubscription.class, + ADMIN_AUTH_HEADERS); // make change in the system TableResourceTest tableResourceTest = new TableResourceTest(); @@ -490,10 +492,13 @@ void app_with_event_subscription() { // assert event subscription was deleted TestUtils.assertResponse( - () ->TestUtils.get(getResource(String.format("events/subscriptions/name/%s", subscriptionName)), EventSubscription.class, ADMIN_AUTH_HEADERS), - NOT_FOUND, - String.format("eventsubscription instance for %s not found", subscriptionName) - ); + () -> + TestUtils.get( + getResource(String.format("events/subscriptions/name/%s", subscriptionName)), + EventSubscription.class, + ADMIN_AUTH_HEADERS), + NOT_FOUND, + String.format("eventsubscription instance for %s not found", subscriptionName)); } @Override diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java index 9fae7a62005f..b6d091c205a2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/apps/TestApp.java @@ -5,7 +5,7 @@ import org.openmetadata.service.search.SearchRepository; public class TestApp extends AbstractNativeApplication { - public TestApp(CollectionDAO collectionDAO, SearchRepository searchRepository) { - super(collectionDAO, searchRepository); - } + public TestApp(CollectionDAO collectionDAO, SearchRepository searchRepository) { + super(collectionDAO, searchRepository); + } } From e9771bddbe7c7d55817c68336ae59820fe94471e Mon Sep 17 00:00:00 2001 From: sushi30 Date: Wed, 15 Jan 2025 16:01:27 +0200 Subject: [PATCH 4/8] fix(governance-workflows): defined interface in workflow schema - defined types for workflow nodes - fixed updateEdge method in WorkflowDefinitionRepository --- .../jdbi3/WorkflowDefinitionRepository.java | 24 +++++++++---------- .../governance/createWorkflowDefinition.json | 1 + .../workflows/workflowDefinition.json | 7 +++++- .../workflows/workflowDefinition.ts | 8 +++---- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java index aff180015d82..4397aa4c5fff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java @@ -1,7 +1,5 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.service.util.EntityUtil.objectMatch; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -10,6 +8,8 @@ import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.governance.workflows.WorkflowDefinition; +import org.openmetadata.schema.governance.workflows.elements.EdgeDefinition; +import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.governance.workflows.Workflow; @@ -95,27 +95,27 @@ private void updateTrigger() { } private void updateNodes() { - List addedNodes = new ArrayList<>(); - List deletedNodes = new ArrayList<>(); + List addedNodes = new ArrayList<>(); + List deletedNodes = new ArrayList<>(); recordListChange( "nodes", - (List) original.getNodes(), - (List) updated.getNodes(), + original.getNodes(), + updated.getNodes(), addedNodes, deletedNodes, - objectMatch); + WorkflowNodeDefinitionInterface::equals); } private void updateEdges() { - List addedEdges = new ArrayList<>(); - List deletedEdges = new ArrayList<>(); + List addedEdges = new ArrayList<>(); + List deletedEdges = new ArrayList<>(); recordListChange( "nodes", - (List) original.getNodes(), - (List) updated.getNodes(), + original.getEdges(), + updated.getEdges(), addedEdges, deletedEdges, - objectMatch); + EdgeDefinition::equals); } } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json b/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json index 348190675e6c..be8c4c312c43 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json @@ -41,6 +41,7 @@ "description": "List of processes used on the workflow.", "type": "array", "items": { + "existingJavaType": "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface", "oneOf": [ { "$ref": "../../governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json index d7e6e4740436..4f16dceb82f9 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json @@ -73,7 +73,12 @@ "description": "Workflow Trigger." }, "nodes": { - "description": "List of nodes used on the workflow." + "description": "List of nodes used on the workflow.", + "type": "array", + "items": { + "type": "object", + "existingJavaType": "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" + } }, "edges": { "description": "List of edges that connect the workflow elements and guide its flow.", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts index 5877619314f0..9b08cbc9f322 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Defines a workflow, having all the different pieces and attributes. */ export interface WorkflowDefinition { @@ -55,7 +53,7 @@ export interface WorkflowDefinition { /** * List of nodes used on the workflow. */ - nodes?: any; + nodes?: { [key: string]: any }[]; /** * Owners of this workflow definition. */ From f749de196812f06f2235b997921f032cbb585ebd Mon Sep 17 00:00:00 2001 From: sushi30 Date: Thu, 16 Jan 2025 08:59:01 +0200 Subject: [PATCH 5/8] ref(governance-workflows): use explicit types in schema use explicit interface type in json schemas and use annotations to infer the concrete classes when deserializing --- .../workflows/elements/NodeFactory.java | 24 ++++--- .../nodes/automatedTask/NoOpTask.java | 66 +++++++++++++++++++ .../nodes/automatedTask/impl/NoOpTaskImp.java | 16 +++++ .../workflows/flowable/MainWorkflow.java | 8 +-- .../WorkflowNodeDefinitionInterface.java | 29 +++++++- .../workflows/elements/nodeSubType.json | 3 +- .../pythonWorkflowAutomationTask.json | 37 +++++++++++ .../workflows/elements/nodeSubType.ts | 7 +- .../pythonWorkflowAutomationTask.ts | 33 ++++++++++ 9 files changed, 198 insertions(+), 25 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.ts diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java index 9ea1d81cc0d5..b2611a971422 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java @@ -1,7 +1,7 @@ package org.openmetadata.service.governance.workflows.elements; -import java.util.Map; import org.openmetadata.schema.governance.workflows.elements.NodeSubType; +import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTaskDefinition; @@ -9,28 +9,26 @@ import org.openmetadata.schema.governance.workflows.elements.nodes.startEvent.StartEventDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.userTask.UserApprovalTaskDefinition; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTask; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.NoOpTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTask; import org.openmetadata.service.governance.workflows.elements.nodes.endEvent.EndEvent; import org.openmetadata.service.governance.workflows.elements.nodes.startEvent.StartEvent; import org.openmetadata.service.governance.workflows.elements.nodes.userTask.UserApprovalTask; -import org.openmetadata.service.util.JsonUtils; public class NodeFactory { - public static NodeInterface createNode(Map nodeDefinition) { - return switch (NodeSubType.fromValue((String) nodeDefinition.get("subType"))) { - case START_EVENT -> new StartEvent( - JsonUtils.readOrConvertValue(nodeDefinition, StartEventDefinition.class)); - case END_EVENT -> new EndEvent( - JsonUtils.readOrConvertValue(nodeDefinition, EndEventDefinition.class)); + public static NodeInterface createNode(WorkflowNodeDefinitionInterface nodeDefinition) { + return switch (NodeSubType.fromValue(nodeDefinition.getSubType())) { + case START_EVENT -> new StartEvent((StartEventDefinition) nodeDefinition); + case END_EVENT -> new EndEvent((EndEventDefinition) nodeDefinition); case CHECK_ENTITY_ATTRIBUTES_TASK -> new CheckEntityAttributesTask( - JsonUtils.readOrConvertValue(nodeDefinition, CheckEntityAttributesTaskDefinition.class)); + (CheckEntityAttributesTaskDefinition) nodeDefinition); case SET_ENTITY_CERTIFICATION_TASK -> new SetEntityCertificationTask( - JsonUtils.readOrConvertValue(nodeDefinition, SetEntityCertificationTaskDefinition.class)); + (SetEntityCertificationTaskDefinition) nodeDefinition); case SET_GLOSSARY_TERM_STATUS_TASK -> new SetGlossaryTermStatusTask( - JsonUtils.readOrConvertValue(nodeDefinition, SetGlossaryTermStatusTaskDefinition.class)); - case USER_APPROVAL_TASK -> new UserApprovalTask( - JsonUtils.readOrConvertValue(nodeDefinition, UserApprovalTaskDefinition.class)); + (SetGlossaryTermStatusTaskDefinition) nodeDefinition); + case USER_APPROVAL_TASK -> new UserApprovalTask((UserApprovalTaskDefinition) nodeDefinition); + case PYTHON_WORKFLOW_AUTOMATION_TASK -> new NoOpTask(nodeDefinition); }; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java new file mode 100644 index 000000000000..b33d11d0cbe4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java @@ -0,0 +1,66 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask; + +import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; + +import org.flowable.bpmn.model.BoundaryEvent; +import org.flowable.bpmn.model.BpmnModel; +import org.flowable.bpmn.model.EndEvent; +import org.flowable.bpmn.model.Process; +import org.flowable.bpmn.model.SequenceFlow; +import org.flowable.bpmn.model.ServiceTask; +import org.flowable.bpmn.model.StartEvent; +import org.flowable.bpmn.model.SubProcess; +import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; +import org.openmetadata.service.governance.workflows.elements.NodeInterface; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl.NoOpTaskImp; +import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; + +public class NoOpTask implements NodeInterface { + private final SubProcess subProcess; + private final BoundaryEvent runtimeExceptionBoundaryEvent; + + public NoOpTask(WorkflowNodeDefinitionInterface nodeDefinition) { + String subProcessId = nodeDefinition.getName(); + + SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); + + StartEvent startEvent = + new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); + + ServiceTask noOpTask = setNoOpTask(subProcessId); + + EndEvent endEvent = + new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); + + subProcess.addFlowElement(startEvent); + subProcess.addFlowElement(noOpTask); + subProcess.addFlowElement(endEvent); + + subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), noOpTask.getId())); + subProcess.addFlowElement(new SequenceFlow(noOpTask.getId(), endEvent.getId())); + + this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.subProcess = subProcess; + } + + @Override + public BoundaryEvent getRuntimeExceptionBoundaryEvent() { + return runtimeExceptionBoundaryEvent; + } + + private ServiceTask setNoOpTask(String subProcessId) { + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "printHelloTask")) + .implementation(NoOpTaskImp.class.getName()) + .build(); + } + + public void addToWorkflow(BpmnModel model, Process process) { + process.addFlowElement(subProcess); + process.addFlowElement(runtimeExceptionBoundaryEvent); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java new file mode 100644 index 000000000000..c6087614c9f9 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java @@ -0,0 +1,16 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; + +import lombok.extern.slf4j.Slf4j; +import org.flowable.common.engine.api.delegate.Expression; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.engine.delegate.JavaDelegate; + +@Slf4j +public class NoOpTaskImp implements JavaDelegate { + private Expression statusExpr; + + @Override + public void execute(DelegateExecution execution) { + System.out.println("hello"); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java index 4fc00c29b660..53acaa0d8e0e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Optional; import lombok.Getter; import org.flowable.bpmn.model.BoundaryEvent; @@ -16,7 +15,6 @@ import org.openmetadata.service.governance.workflows.elements.NodeFactory; import org.openmetadata.service.governance.workflows.elements.NodeInterface; import org.openmetadata.service.governance.workflows.elements.nodes.endEvent.EndEvent; -import org.openmetadata.service.util.JsonUtils; @Getter public class MainWorkflow { @@ -37,10 +35,8 @@ public MainWorkflow(WorkflowDefinition workflowDefinition) { model.addProcess(process); // Add Nodes - for (Object nodeDefinitionObj : - (List) workflowDefinition.getNodes()) { - NodeInterface node = - NodeFactory.createNode(JsonUtils.readOrConvertValue(nodeDefinitionObj, Map.class)); + for (WorkflowNodeDefinitionInterface nodeDefinitionObj : workflowDefinition.getNodes()) { + NodeInterface node = NodeFactory.createNode(nodeDefinitionObj); node.addToWorkflow(model, process); Optional.ofNullable(node.getRuntimeExceptionBoundaryEvent()) diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java index e8d062b4cae3..afb86aedc317 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java @@ -1,10 +1,37 @@ package org.openmetadata.schema.governance.workflows.elements; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.util.List; import java.util.Map; import org.openmetadata.common.utils.CommonUtil; - +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.PythonWorkflowAutomationTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.endEvent.EndEventDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.startEvent.StartEventDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.userTask.UserApprovalTaskDefinition; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "subType") +@JsonSubTypes({ + @JsonSubTypes.Type( + value = CheckEntityAttributesTaskDefinition.class, + name = "checkEntityAttributesTask"), + @JsonSubTypes.Type( + value = SetEntityCertificationTaskDefinition.class, + name = "setEntityCertificationTask"), + @JsonSubTypes.Type(value = StartEventDefinition.class, name = "startEvent"), + @JsonSubTypes.Type(value = EndEventDefinition.class, name = "endEvent"), + @JsonSubTypes.Type( + value = SetGlossaryTermStatusTaskDefinition.class, + name = "setGlossaryTermStatusTask"), + @JsonSubTypes.Type(value = UserApprovalTaskDefinition.class, name = "userApprovalTask"), + @JsonSubTypes.Type( + value = PythonWorkflowAutomationTaskDefinition.class, + name = "pythonWorkflowAutomationTask") +}) public interface WorkflowNodeDefinitionInterface { String getType(); diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json index cea4c491a388..90fe13650d87 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json @@ -11,6 +11,7 @@ "setEntityCertificationTask", "setGlossaryTermStatusTask", "endEvent", - "startEvent" + "startEvent", + "pythonWorkflowAutomationTask" ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json new file mode 100644 index 000000000000..5812e6cbb59e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json @@ -0,0 +1,37 @@ +{ + "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PythonWorkflowAutomationTaskDefinition", + "description": "Sets the GlossaryTerm Status to the configured value.", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" + ], + "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.PythonWorkflowAutomationTaskDefinition", + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "automatedTask" + }, + "subType": { + "type": "string", + "default": "pythonWorkflowAutomationTask" + }, + "name": { + "description": "Name that identifies this Node.", + "$ref": "../../../../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this Node.", + "type": "string" + }, + "description": { + "description": "Description of the Node.", + "$ref": "../../../../../type/basic.json#/definitions/markdown" + }, + "config": { + "type": "object", + "existingJavaType": "java.util.Map" + } + } +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts index b197909a190d..6d8489bce735 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,14 +10,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * SubType of the Node. */ export enum NodeSubType { CheckEntityAttributesTask = "checkEntityAttributesTask", EndEvent = "endEvent", + PythonWorkflowAutomationTask = "pythonWorkflowAutomationTask", SetEntityCertificationTask = "setEntityCertificationTask", SetGlossaryTermStatusTask = "setGlossaryTermStatusTask", StartEvent = "startEvent", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.ts new file mode 100644 index 000000000000..fdcbc415f412 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Sets the GlossaryTerm Status to the configured value. + */ +export interface PythonWorkflowAutomationTask { + config?: { [key: string]: any }; + /** + * Description of the Node. + */ + description?: string; + /** + * Display Name that identifies this Node. + */ + displayName?: string; + /** + * Name that identifies this Node. + */ + name?: string; + subType?: string; + type?: string; + [property: string]: any; +} From b8fcea3cb427ad722280dbdb1af76e6448a6e5b1 Mon Sep 17 00:00:00 2001 From: sushi30 Date: Fri, 17 Jan 2025 12:23:27 +0200 Subject: [PATCH 6/8] - implemented CustomSignal trigger - use JsonTypeInfo and JsonSubTypes to infer types in jsonschema for nodes and triggers - Implemented JsonLogicFilter task - implemented placeholder noop task --- .../governance/workflows/Workflow.java | 1 + .../workflows/elements/NodeFactory.java | 4 + .../workflows/elements/TriggerFactory.java | 24 ++-- .../automatedTask/JsonLogicFilterTask.java | 74 ++++++++++ .../impl/JsonLogicFilterImpl.java | 59 ++++++++ .../nodes/automatedTask/impl/NoOpTaskImp.java | 5 +- .../triggers/CustomSignalTrigger.java | 130 ++++++++++++++++++ .../governance/WorkflowDefinitionMapper.java | 2 +- .../WorkflowDefinitionResource.java | 35 +++++ .../WorkflowNodeDefinitionInterface.java | 4 +- .../elements/WorkflowTriggerInterface.java | 26 ++++ .../governance/createWorkflowDefinition.json | 3 +- .../workflows/elements/nodeSubType.json | 3 +- .../nodes/automatedTask/jsonLogicTask.json | 44 ++++++ .../triggers/customSignalTrigger.json | 33 +++++ .../triggers/eventBasedEntityTrigger.json | 14 +- .../triggers/periodicBatchEntityTrigger.json | 15 +- .../workflows/workflowDefinition.json | 10 +- .../workflows/elements/nodeSubType.ts | 1 + .../nodes/automatedTask/jsonLogicTask.ts | 37 +++++ .../elements/triggers/customSignalTrigger.ts | 29 ++++ .../triggers/eventBasedEntityTrigger.ts | 6 +- .../triggers/periodicBatchEntityTrigger.ts | 6 +- .../workflows/workflowDefinition.ts | 8 +- 24 files changed, 541 insertions(+), 32 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java create mode 100644 openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.ts create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/customSignalTrigger.ts diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java index 3b9b437e3611..7b6bb3111590 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java @@ -8,6 +8,7 @@ @Getter public class Workflow { public static final String RELATED_ENTITY_VARIABLE = "relatedEntity"; + public static final String PAYLOAD = "payload"; public static final String RESULT_VARIABLE = "result"; public static final String RESOLVED_BY_VARIABLE = "resolvedBy"; public static final String STAGE_INSTANCE_STATE_ID_VARIABLE = "stageInstanceStateId"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java index b2611a971422..0b81b3477bd9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java @@ -1,14 +1,17 @@ package org.openmetadata.service.governance.workflows.elements; +import java.lang.reflect.InvocationTargetException; import org.openmetadata.schema.governance.workflows.elements.NodeSubType; import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.endEvent.EndEventDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.startEvent.StartEventDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.userTask.UserApprovalTaskDefinition; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTask; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.JsonLogicFilterTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.NoOpTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTask; @@ -29,6 +32,7 @@ public static NodeInterface createNode(WorkflowNodeDefinitionInterface nodeDefin (SetGlossaryTermStatusTaskDefinition) nodeDefinition); case USER_APPROVAL_TASK -> new UserApprovalTask((UserApprovalTaskDefinition) nodeDefinition); case PYTHON_WORKFLOW_AUTOMATION_TASK -> new NoOpTask(nodeDefinition); + case JSON_LOGIC_TASK -> new JsonLogicFilterTask((JsonLogicTaskDefinition) nodeDefinition); }; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java index 49d60adcd0bc..3230e8982156 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java @@ -2,27 +2,29 @@ import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.trigger.PeriodicBatchEntityTriggerDefinition; +import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; import org.openmetadata.schema.governance.workflows.elements.triggers.EventBasedEntityTriggerDefinition; +import org.openmetadata.service.governance.workflows.elements.triggers.CustomSignalTrigger; import org.openmetadata.service.governance.workflows.elements.triggers.EventBasedEntityTrigger; import org.openmetadata.service.governance.workflows.elements.triggers.PeriodicBatchEntityTrigger; -import org.openmetadata.service.util.JsonUtils; public class TriggerFactory { - public static TriggerInterface createTrigger(WorkflowDefinition workflowDefinition) { - String mainWorkflowName = workflowDefinition.getFullyQualifiedName(); - String triggerWorkflowId = getTriggerWorkflowId(mainWorkflowName); + public static TriggerInterface createTrigger(WorkflowDefinition workflow) { + String triggerWorkflowId = getTriggerWorkflowId(workflow.getFullyQualifiedName()); - return switch (workflowDefinition.getType()) { + return switch (workflow.getType()) { case EVENT_BASED_ENTITY_WORKFLOW -> new EventBasedEntityTrigger( - mainWorkflowName, + workflow.getName(), triggerWorkflowId, - JsonUtils.readOrConvertValue( - workflowDefinition.getTrigger(), EventBasedEntityTriggerDefinition.class)); + (EventBasedEntityTriggerDefinition) workflow.getTrigger()); + case CUSTOM_SIGNAL_WORKFLOW -> new CustomSignalTrigger( + workflow.getName(), + triggerWorkflowId, + (CustomSignalTriggerDefinition) workflow.getTrigger()); case PERIODIC_BATCH_ENTITY_WORKFLOW -> new PeriodicBatchEntityTrigger( - mainWorkflowName, + workflow.getName(), triggerWorkflowId, - JsonUtils.readOrConvertValue( - workflowDefinition.getTrigger(), PeriodicBatchEntityTriggerDefinition.class)); + (PeriodicBatchEntityTriggerDefinition) workflow.getTrigger()); }; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java new file mode 100644 index 000000000000..f73589a467c3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java @@ -0,0 +1,74 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask; + +import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; + +import org.flowable.bpmn.model.BoundaryEvent; +import org.flowable.bpmn.model.BpmnModel; +import org.flowable.bpmn.model.EndEvent; +import org.flowable.bpmn.model.FieldExtension; +import org.flowable.bpmn.model.Process; +import org.flowable.bpmn.model.SequenceFlow; +import org.flowable.bpmn.model.ServiceTask; +import org.flowable.bpmn.model.StartEvent; +import org.flowable.bpmn.model.SubProcess; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition; +import org.openmetadata.service.governance.workflows.elements.NodeInterface; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl.JsonLogicFilterImpl; +import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.FieldExtensionBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; + +public class JsonLogicFilterTask implements NodeInterface { + private final SubProcess subProcess; + private final BoundaryEvent runtimeExceptionBoundaryEvent; + + public JsonLogicFilterTask(JsonLogicTaskDefinition nodeDefinition) { + String subProcessId = nodeDefinition.getName(); + + SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); + + StartEvent startEvent = + new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); + + ServiceTask checkEntityAttributes = + getCheckEntityAttributesServiceTask(subProcessId, nodeDefinition.getConfig().getRules()); + + EndEvent endEvent = + new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); + + subProcess.addFlowElement(startEvent); + subProcess.addFlowElement(checkEntityAttributes); + subProcess.addFlowElement(endEvent); + + subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), checkEntityAttributes.getId())); + subProcess.addFlowElement(new SequenceFlow(checkEntityAttributes.getId(), endEvent.getId())); + + this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.subProcess = subProcess; + } + + @Override + public BoundaryEvent getRuntimeExceptionBoundaryEvent() { + return runtimeExceptionBoundaryEvent; + } + + private ServiceTask getCheckEntityAttributesServiceTask(String subProcessId, String rules) { + FieldExtension rulesExpr = + new FieldExtensionBuilder().fieldName("rulesExpr").fieldValue(rules).build(); + + ServiceTask serviceTask = + new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "jsonLogic")) + .implementation(JsonLogicFilterImpl.class.getName()) + .build(); + serviceTask.getFieldExtensions().add(rulesExpr); + return serviceTask; + } + + public void addToWorkflow(BpmnModel model, Process process) { + process.addFlowElement(subProcess); + process.addFlowElement(runtimeExceptionBoundaryEvent); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java new file mode 100644 index 000000000000..51e04b4b1d5e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java @@ -0,0 +1,59 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; + +import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.PAYLOAD; +import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; +import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; + +import io.github.jamsesso.jsonlogic.JsonLogic; +import io.github.jamsesso.jsonlogic.JsonLogicException; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.flowable.common.engine.api.delegate.Expression; +import org.flowable.engine.delegate.BpmnError; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.engine.delegate.JavaDelegate; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class JsonLogicFilterImpl implements JavaDelegate { + private Expression rulesExpr; + + @Override + public void execute(DelegateExecution execution) { + try { + // TODO why is 'rulesExpr' not passed as a variable? + String rules = (String) rulesExpr.getValue(execution); + String payload = (String) execution.getVariable("payload"); + List filtered = + JsonUtils.readObjects(payload, ChangeEvent.class).stream() + .filter( + ce -> { + EntityInterface entity = + Entity.getEntity(ce.getEntityType(), ce.getEntityId(), "*", Include.ALL); + return checkAttributes(rules, entity); + }) + .toList(); + + execution.setVariable(PAYLOAD, JsonUtils.pojoToJson(filtered)); + } catch (Exception exc) { + LOG.error( + "[{}] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId()), exc); + execution.setVariable(EXCEPTION_VARIABLE, exc.toString()); + throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); + } + } + + private Boolean checkAttributes(String rules, EntityInterface entity) { + JsonLogic jsonLogic = new JsonLogic(); + try { + return (boolean) jsonLogic.apply(rules, JsonUtils.getMap(entity)); + } catch (JsonLogicException e) { + throw new RuntimeException(e); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java index c6087614c9f9..5480acc42320 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java @@ -1,5 +1,7 @@ package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; +import static org.openmetadata.service.governance.workflows.Workflow.PAYLOAD; + import lombok.extern.slf4j.Slf4j; import org.flowable.common.engine.api.delegate.Expression; import org.flowable.engine.delegate.DelegateExecution; @@ -11,6 +13,7 @@ public class NoOpTaskImp implements JavaDelegate { @Override public void execute(DelegateExecution execution) { - System.out.println("hello"); + String payload = (String) execution.getVariable(PAYLOAD); + System.out.println("NoOpTaskImp: " + payload); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java new file mode 100644 index 000000000000..107fd2eb6377 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java @@ -0,0 +1,130 @@ +package org.openmetadata.service.governance.workflows.elements.triggers; + +import static org.openmetadata.service.governance.workflows.Workflow.PAYLOAD; +import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; +import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import org.flowable.bpmn.model.BoundaryEvent; +import org.flowable.bpmn.model.BpmnModel; +import org.flowable.bpmn.model.CallActivity; +import org.flowable.bpmn.model.EndEvent; +import org.flowable.bpmn.model.ErrorEventDefinition; +import org.flowable.bpmn.model.IOParameter; +import org.flowable.bpmn.model.Process; +import org.flowable.bpmn.model.SequenceFlow; +import org.flowable.bpmn.model.Signal; +import org.flowable.bpmn.model.SignalEventDefinition; +import org.flowable.bpmn.model.StartEvent; +import org.jetbrains.annotations.NotNull; +import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; +import org.openmetadata.service.governance.workflows.elements.TriggerInterface; +import org.openmetadata.service.governance.workflows.flowable.builders.CallActivityBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.SignalBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; + +public class CustomSignalTrigger implements TriggerInterface { + private final Process process; + @Getter private final String triggerWorkflowId; + private StartEvent startEvent = null; + private final List signals = new ArrayList<>(); + + public CustomSignalTrigger( + String mainWorkflowName, + String triggerWorkflowId, + CustomSignalTriggerDefinition triggerDefinition) { + Process process = new Process(); + process.setId(triggerWorkflowId); + process.setName(triggerWorkflowId); + attachWorkflowInstanceListeners(process); + + setStartEvent(triggerWorkflowId, triggerDefinition); + + CallActivity workflowTrigger = getWorkflowTrigger(triggerWorkflowId, mainWorkflowName); + process.addFlowElement(workflowTrigger); + + BoundaryEvent runtimeExceptionBoundaryEvent = getBoundaryEvent(workflowTrigger); + process.addFlowElement(runtimeExceptionBoundaryEvent); + + EndEvent errorEndEvent = + new EndEventBuilder().id(getFlowableElementId(triggerWorkflowId, "errorEndEvent")).build(); + process.addFlowElement(errorEndEvent); + + EndEvent endEvent = + new EndEventBuilder().id(getFlowableElementId(triggerWorkflowId, "endEvent")).build(); + process.addFlowElement(endEvent); + + // Start Events -> FilterTask + process.addFlowElement(startEvent); + process.addFlowElement(new SequenceFlow(startEvent.getId(), workflowTrigger.getId())); + process.addFlowElement(new SequenceFlow(workflowTrigger.getId(), endEvent.getId())); + + // WorkflowTrigger -> End + process.addFlowElement( + new SequenceFlow(runtimeExceptionBoundaryEvent.getId(), errorEndEvent.getId())); + + this.process = process; + this.triggerWorkflowId = triggerWorkflowId; + } + + private static @NotNull BoundaryEvent getBoundaryEvent(CallActivity workflowTrigger) { + ErrorEventDefinition runtimeExceptionDefinition = new ErrorEventDefinition(); + runtimeExceptionDefinition.setErrorCode(WORKFLOW_RUNTIME_EXCEPTION); + + BoundaryEvent runtimeExceptionBoundaryEvent = new BoundaryEvent(); + runtimeExceptionBoundaryEvent.setId( + getFlowableElementId(workflowTrigger.getId(), "runtimeExceptionBoundaryEvent")); + runtimeExceptionBoundaryEvent.addEventDefinition(runtimeExceptionDefinition); + + runtimeExceptionBoundaryEvent.setAttachedToRef(workflowTrigger); + return runtimeExceptionBoundaryEvent; + } + + private void setStartEvent( + String workflowTriggerId, CustomSignalTriggerDefinition triggerDefinition) { + Signal signal = new SignalBuilder().id(triggerDefinition.getConfig().getSignal()).build(); + + SignalEventDefinition signalEventDefinition = new SignalEventDefinition(); + signalEventDefinition.setSignalRef(signal.getId()); + + StartEvent startEvent = + new StartEventBuilder().id(getFlowableElementId(workflowTriggerId, "customSignal")).build(); + startEvent.getEventDefinitions().add(signalEventDefinition); + + this.startEvent = startEvent; + this.signals.add(signal); + } + + private CallActivity getWorkflowTrigger(String triggerWorkflowId, String mainWorkflowName) { + CallActivity workflowTrigger = + new CallActivityBuilder() + .id(getFlowableElementId(triggerWorkflowId, "workflowTrigger")) + .calledElement(mainWorkflowName) + .inheritBusinessKey(true) + .build(); + + IOParameter inputParameter = new IOParameter(); + inputParameter.setSource(PAYLOAD); + inputParameter.setTarget(PAYLOAD); + + IOParameter outputParameter = new IOParameter(); + outputParameter.setSource(PAYLOAD); + outputParameter.setTarget(PAYLOAD); + + workflowTrigger.setInParameters(List.of(inputParameter)); + workflowTrigger.setOutParameters(List.of(outputParameter)); + + return workflowTrigger; + } + + @Override + public void addToWorkflow(BpmnModel model) { + model.addProcess(process); + for (Signal signal : signals) { + model.addSignal(signal); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java index 452ca49309f4..89c25ac90fed 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java @@ -10,7 +10,7 @@ public class WorkflowDefinitionMapper public WorkflowDefinition createToEntity(CreateWorkflowDefinition create, String user) { return copy(new WorkflowDefinition(), create, user) .withFullyQualifiedName(create.getName()) - .withType(WorkflowDefinition.Type.fromValue(create.getType().toString())) + .withType(create.getType()) .withTrigger(create.getTrigger()) .withNodes(create.getNodes()) .withEdges(create.getEdges()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java index 73832c1313e9..ee8badb23366 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java @@ -38,6 +38,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.governance.workflows.Workflow; import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.WorkflowDefinitionRepository; @@ -45,6 +46,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/governance/workflowDefinitions") @@ -185,6 +187,39 @@ public WorkflowDefinition get( return getInternal(uriInfo, securityContext, id, fieldsParam, include); } + @POST + @Path("/{id}/redeploy") + @Operation( + operationId = "getWorkflowDefinitionByID", + summary = "Get a Workflow Definition by Id", + description = "Get a Workflow Definition by `Id`.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The Workflow Definition", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = WorkflowDefinition.class))) + }) + public Response redeploy( + @Context UriInfo uriInfo, + @Parameter(description = "Id of the Workflow Definition", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id, + @Context SecurityContext securityContext) { + WorkflowDefinition wd = + repository.get( + uriInfo, + id, + new EntityUtil.Fields(repository.getAllowedFields()), + Include.NON_DELETED, + false); + WorkflowHandler.getInstance().deleteWorkflowDefinition(wd.getName()); + WorkflowHandler.getInstance().deploy(new Workflow(wd)); + return Response.status(Response.Status.OK).entity("Workflow Redeployed").build(); + } + @GET @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java index afb86aedc317..304f5eec867b 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java @@ -7,6 +7,7 @@ import java.util.Map; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.PythonWorkflowAutomationTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTaskDefinition; @@ -30,7 +31,8 @@ @JsonSubTypes.Type(value = UserApprovalTaskDefinition.class, name = "userApprovalTask"), @JsonSubTypes.Type( value = PythonWorkflowAutomationTaskDefinition.class, - name = "pythonWorkflowAutomationTask") + name = "pythonWorkflowAutomationTask"), + @JsonSubTypes.Type(value = JsonLogicTaskDefinition.class, name = "jsonLogicTask"), }) public interface WorkflowNodeDefinitionInterface { String getType(); diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java new file mode 100644 index 000000000000..06efb88c6705 --- /dev/null +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java @@ -0,0 +1,26 @@ +package org.openmetadata.schema.governance.workflows.elements; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.openmetadata.schema.governance.workflows.elements.nodes.trigger.PeriodicBatchEntityTriggerDefinition; +import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; +import org.openmetadata.schema.governance.workflows.elements.triggers.EventBasedEntityTriggerDefinition; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type( + value = EventBasedEntityTriggerDefinition.class, + name = "eventBasedEntityWorkflow"), + @JsonSubTypes.Type(value = CustomSignalTriggerDefinition.class, name = "customSignalWorkflow"), + @JsonSubTypes.Type( + value = PeriodicBatchEntityTriggerDefinition.class, + name = "periodicBatchEntityWorkflow"), +}) +public interface WorkflowTriggerInterface { + // TODO If set as enum, it results in null when the JSON is deserialized. Maybe there can be + // another + // way to validate it on deserialization. + String getType(); + + Object getConfig(); +} diff --git a/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json b/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json index be8c4c312c43..2ff20a9ca7e0 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/governance/createWorkflowDefinition.json @@ -25,9 +25,10 @@ "default": null }, "type": { - "$ref": "../../governance/workflows/workflowDefinition.json#definitions/type" + "$ref": "../../governance/workflows/workflowDefinition.json#/definitions/type" }, "trigger": { + "existingJavaType": "org.openmetadata.schema.governance.workflows.elements.WorkflowTriggerInterface", "oneOf": [ { "$ref": "../../governance/workflows/elements/triggers/eventBasedEntityTrigger.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json index 90fe13650d87..8ce4b811a63c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json @@ -12,6 +12,7 @@ "setGlossaryTermStatusTask", "endEvent", "startEvent", - "pythonWorkflowAutomationTask" + "pythonWorkflowAutomationTask", + "jsonLogicTask" ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json new file mode 100644 index 000000000000..512e1f21e05e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json @@ -0,0 +1,44 @@ +{ + "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "JsonLogicTaskDefinition", + "description": "Checks if an Entity attributes fit given rules.", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" + ], + "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition", + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "automatedTask" + }, + "subType": { + "type": "string", + "default": "jsonLogicTask" + }, + "name": { + "description": "Name that identifies this Node.", + "$ref": "../../../../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "description": "Display Name that identifies this Node.", + "type": "string" + }, + "description": { + "description": "Description of the Node.", + "$ref": "../../../../../type/basic.json#/definitions/markdown" + }, + "config": { + "type": "object", + "properties": { + "rules": { + "type": "string", + "outputType": "jsonlogic", + "format": "queryBuilder" + } + }, + "additionalProperties": false + } + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json new file mode 100644 index 000000000000..2e776f679ff9 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json @@ -0,0 +1,33 @@ +{ + "$id": "https://open-metadata.org/schema/governance/workflows/elements/triggers/customSignalTrigger.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CustomSignalTriggerDefinition", + "description": "Event Based Entity Trigger.", + "javaType": "org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition", + "javaInterfaces" : ["org.openmetadata.schema.governance.workflows.elements.WorkflowTriggerInterface"], + "type": "object", + "definitions": { + "config": { + "description": "Entity Event Trigger Configuration.", + "type": "object", + "properties": { + "signal": { + "description": "The signal to be listened to.", + "type": "string" + } + }, + "required": ["signal"], + "additionalProperties": false + } + }, + "properties": { + "type": { + "type": "string", + "default": "customSignalWorkflow" + }, + "config": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json index eaa3b716c756..8627c163dae6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json @@ -4,6 +4,9 @@ "title": "EventBasedEntityTriggerDefinition", "description": "Event Based Entity Trigger.", "javaType": "org.openmetadata.schema.governance.workflows.elements.triggers.EventBasedEntityTriggerDefinition", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowTriggerInterface" + ], "type": "object", "definitions": { "event": { @@ -37,14 +40,17 @@ } } }, - "required": ["entityType", "events"], + "required": [ + "entityType", + "events" + ], "additionalProperties": false } }, "properties": { "type": { "type": "string", - "default": "eventBasedEntityTrigger" + "default": "eventBasedEntityWorkflow" }, "config": { "$ref": "#/definitions/config" @@ -54,7 +60,9 @@ "items": { "type": "string" }, - "default": ["relatedEntity"], + "default": [ + "relatedEntity" + ], "additionalItems": false, "minItems": 1, "maxItems": 1, diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json index 43bbc31af1b0..d4759df311c7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json @@ -4,6 +4,9 @@ "title": "PeriodicBatchEntityTriggerDefinition", "description": "Periodic Batch Entity Trigger.", "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.trigger.PeriodicBatchEntityTriggerDefinition", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowTriggerInterface" + ], "type": "object", "definitions": { "config": { @@ -28,14 +31,18 @@ "default": 500 } }, - "required": ["schedule", "entityType", "filters"], + "required": [ + "schedule", + "entityType", + "filters" + ], "additionalProperties": false } }, "properties": { "type": { "type": "string", - "default": "periodicBatchEntityTrigger" + "default": "periodicBatchEntityWorkflow" }, "config": { "$ref": "#/definitions/config" @@ -45,7 +52,9 @@ "items": { "type": "string" }, - "default": ["relatedEntity"], + "default": [ + "relatedEntity" + ], "additionalItems": false, "minItems": 1, "maxItems": 1, diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json index 4f16dceb82f9..17c16c6c715c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json @@ -9,8 +9,10 @@ "definitions": { "type": { "type": "string", + "javaType": "org.openmetadata.schema.governance.workflows.TriggerType", "enum": [ "eventBasedEntityWorkflow", + "customSignalWorkflow", "periodicBatchEntityWorkflow" ] } @@ -70,7 +72,13 @@ "$ref": "#/definitions/type" }, "trigger": { - "description": "Workflow Trigger." + "description": "Workflow Trigger.", + "existingJavaType": "org.openmetadata.schema.governance.workflows.elements.WorkflowTriggerInterface", + "properties": { + "type": { + "$ref": "#/definitions/type" + } + } }, "nodes": { "description": "List of nodes used on the workflow.", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts index 6d8489bce735..4436eb492c32 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodeSubType.ts @@ -16,6 +16,7 @@ export enum NodeSubType { CheckEntityAttributesTask = "checkEntityAttributesTask", EndEvent = "endEvent", + JSONLogicTask = "jsonLogicTask", PythonWorkflowAutomationTask = "pythonWorkflowAutomationTask", SetEntityCertificationTask = "setEntityCertificationTask", SetGlossaryTermStatusTask = "setGlossaryTermStatusTask", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.ts new file mode 100644 index 000000000000..0b1eb49a07c9 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Checks if an Entity attributes fit given rules. + */ +export interface JSONLogicTask { + config?: Config; + /** + * Description of the Node. + */ + description?: string; + /** + * Display Name that identifies this Node. + */ + displayName?: string; + /** + * Name that identifies this Node. + */ + name?: string; + subType?: string; + type?: string; + [property: string]: any; +} + +export interface Config { + rules?: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/customSignalTrigger.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/customSignalTrigger.ts new file mode 100644 index 000000000000..3630133cde00 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/customSignalTrigger.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Event Based Entity Trigger. + */ +export interface CustomSignalTrigger { + config?: Config; + type?: string; +} + +/** + * Entity Event Trigger Configuration. + */ +export interface Config { + /** + * The signal to be listened to. + */ + signal: string; +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/eventBasedEntityTrigger.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/eventBasedEntityTrigger.ts index 187c728aea9d..bba1461d61ef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/eventBasedEntityTrigger.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/eventBasedEntityTrigger.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Event Based Entity Trigger. */ export interface EventBasedEntityTrigger { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/periodicBatchEntityTrigger.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/periodicBatchEntityTrigger.ts index 6780fa7b4618..e538874252fb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/periodicBatchEntityTrigger.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/elements/triggers/periodicBatchEntityTrigger.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Periodic Batch Entity Trigger. */ export interface PeriodicBatchEntityTrigger { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts index 9b08cbc9f322..6457988ab8d7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts @@ -61,7 +61,7 @@ export interface WorkflowDefinition { /** * Workflow Trigger. */ - trigger?: any; + trigger?: any[] | boolean | number | number | null | TriggerObject | string; type?: Type; /** * Last update time corresponding to the new version of the entity in Unix epoch time @@ -193,7 +193,13 @@ export interface EntityReference { type: string; } +export interface TriggerObject { + type?: Type; + [property: string]: any; +} + export enum Type { + CustomSignalWorkflow = "customSignalWorkflow", EventBasedEntityWorkflow = "eventBasedEntityWorkflow", PeriodicBatchEntityWorkflow = "periodicBatchEntityWorkflow", } From 1a9030da9bde5cbb4325f32d43a2614eec6506e8 Mon Sep 17 00:00:00 2001 From: sushi30 Date: Mon, 20 Jan 2025 16:31:22 +0100 Subject: [PATCH 7/8] - simplified trigger type names - removed governance workflow root "type" - added "deployed" field to governance workflow - applied changes to existing gov-workflows - migrations --- .../1.7.0/mysql/postDataMigrationSQLScript.sql | 7 +++++++ .../native/1.7.0/mysql/schemaChanges.sql | 3 +++ .../postgres/postDataMigrationSQLScript.sql | 7 +++++++ .../native/1.7.0/postgres/schemaChanges.sql | 3 +++ .../governance/workflows/WorkflowHandler.java | 16 ++++++++++------ .../workflows/elements/NodeFactory.java | 1 - .../workflows/elements/TriggerFactory.java | 9 +++++---- .../jdbi3/WorkflowDefinitionRepository.java | 14 ++++---------- .../governance/WorkflowDefinitionMapper.java | 1 - .../governance/WorkflowDefinitionResource.java | 2 +- .../workflows/GlossaryApprovalWorkflow.json | 3 +-- .../elements/WorkflowTriggerInterface.java | 13 +++++-------- .../elements/triggers/customSignalTrigger.json | 2 +- .../triggers/eventBasedEntityTrigger.json | 2 +- .../triggers/periodicBatchEntityTrigger.json | 2 +- .../governance/workflows/workflowDefinition.json | 11 ++++++----- .../governance/workflows/workflowDefinition.ts | 11 +++++++---- 17 files changed, 62 insertions(+), 45 deletions(-) create mode 100644 bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.7.0/mysql/schemaChanges.sql create mode 100644 bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql create mode 100644 bootstrap/sql/migrations/native/1.7.0/postgres/schemaChanges.sql diff --git a/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..078f4495ff92 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql @@ -0,0 +1,7 @@ +UPDATE workflow_definition_entity +SET json = JSON_SET(json, '$.trigger.type', 'eventBasedEntity') +WHERE JSON_EXTRACT(json, '$.trigger.type') = 'eventBasedEntityWorkflow'; + +UPDATE your_table_name +SET json = JSON_SET(json, '$.trigger.type', 'periodicBatchEntity') +WHERE JSON_EXTRACT(json, '$.trigger.type') = 'periodicBatchEntityWorkflow'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.7.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.7.0/mysql/schemaChanges.sql new file mode 100644 index 000000000000..e163f28c94f1 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.7.0/mysql/schemaChanges.sql @@ -0,0 +1,3 @@ +UPDATE workflow_definition_entity +SET json = JSON_REMOVE(json, '$.type') +WHERE JSON_EXTRACT(json, '$.type') IS NOT NULL; diff --git a/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql new file mode 100644 index 000000000000..8950262f5199 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.7.0/postgres/postDataMigrationSQLScript.sql @@ -0,0 +1,7 @@ +UPDATE workflow_definition_entity +SET json = jsonb_set(json, '{trigger,type}', '"eventBasedEntity"') +WHERE json->'trigger'->>'type' = 'eventBasedEntityWorkflow'; + +UPDATE workflow_definition_entity +SET json = jsonb_set(json, '{trigger,type}', '"periodicBatchEntity"') +WHERE json->'trigger'->>'type' = 'periodicBatchEntityWorkflow'; \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.7.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.7.0/postgres/schemaChanges.sql new file mode 100644 index 000000000000..e45493d32caa --- /dev/null +++ b/bootstrap/sql/migrations/native/1.7.0/postgres/schemaChanges.sql @@ -0,0 +1,3 @@ +UPDATE workflow_definition_entity +SET json = jsonb - 'type' +WHERE json->>'type' IS NOT NULL; \ No newline at end of file diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java index 3e28d2d39885..16287cd21fda 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java @@ -26,6 +26,7 @@ import org.flowable.engine.runtime.ProcessInstance; import org.flowable.task.api.Task; import org.openmetadata.schema.configuration.WorkflowSettings; +import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.UnhandledServerException; @@ -156,12 +157,15 @@ public void deploy(Workflow workflow) { .deploy(); } - public void deleteWorkflowDefinition(String processDefinitionKey) { + public boolean isDeployed(WorkflowDefinition wf) { List processDefinitions = - repositoryService - .createProcessDefinitionQuery() - .processDefinitionKey(processDefinitionKey) - .list(); + repositoryService.createProcessDefinitionQuery().processDefinitionKey(wf.getName()).list(); + return !processDefinitions.isEmpty(); + } + + public void deleteWorkflowDefinition(WorkflowDefinition wf) { + List processDefinitions = + repositoryService.createProcessDefinitionQuery().processDefinitionKey(wf.getName()).list(); for (ProcessDefinition processDefinition : processDefinitions) { String deploymentId = processDefinition.getDeploymentId(); @@ -172,7 +176,7 @@ public void deleteWorkflowDefinition(String processDefinitionKey) { List triggerProcessDefinition = repositoryService .createProcessDefinitionQuery() - .processDefinitionKey(getTriggerWorkflowId(processDefinitionKey)) + .processDefinitionKey(getTriggerWorkflowId(wf.getName())) .list(); for (ProcessDefinition processDefinition : triggerProcessDefinition) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java index 0b81b3477bd9..60e17fa138a3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java @@ -1,6 +1,5 @@ package org.openmetadata.service.governance.workflows.elements; -import java.lang.reflect.InvocationTargetException; import org.openmetadata.schema.governance.workflows.elements.NodeSubType; import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java index 3230e8982156..27ae31d267ec 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java @@ -1,5 +1,6 @@ package org.openmetadata.service.governance.workflows.elements; +import org.openmetadata.schema.governance.workflows.TriggerType; import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.trigger.PeriodicBatchEntityTriggerDefinition; import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; @@ -12,16 +13,16 @@ public class TriggerFactory { public static TriggerInterface createTrigger(WorkflowDefinition workflow) { String triggerWorkflowId = getTriggerWorkflowId(workflow.getFullyQualifiedName()); - return switch (workflow.getType()) { - case EVENT_BASED_ENTITY_WORKFLOW -> new EventBasedEntityTrigger( + return switch (TriggerType.fromValue(workflow.getTrigger().getType())) { + case EVENT_BASED_ENTITY -> new EventBasedEntityTrigger( workflow.getName(), triggerWorkflowId, (EventBasedEntityTriggerDefinition) workflow.getTrigger()); - case CUSTOM_SIGNAL_WORKFLOW -> new CustomSignalTrigger( + case CUSTOM_SIGNAL -> new CustomSignalTrigger( workflow.getName(), triggerWorkflowId, (CustomSignalTriggerDefinition) workflow.getTrigger()); - case PERIODIC_BATCH_ENTITY_WORKFLOW -> new PeriodicBatchEntityTrigger( + case PERIODIC_BATCH_ENTITY -> new PeriodicBatchEntityTrigger( workflow.getName(), triggerWorkflowId, (PeriodicBatchEntityTriggerDefinition) workflow.getTrigger()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java index 4397aa4c5fff..0e7e89fb5353 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java @@ -47,11 +47,13 @@ protected void postUpdate(WorkflowDefinition original, WorkflowDefinition update @Override protected void postDelete(WorkflowDefinition entity) { - WorkflowHandler.getInstance().deleteWorkflowDefinition(entity.getName()); + WorkflowHandler.getInstance().deleteWorkflowDefinition(entity); } @Override - protected void setFields(WorkflowDefinition entity, EntityUtil.Fields fields) {} + protected void setFields(WorkflowDefinition entity, EntityUtil.Fields fields) { + entity.withDeployed(WorkflowHandler.getInstance().isDeployed(entity)); + } @Override protected void clearFields(WorkflowDefinition entity, EntityUtil.Fields fields) {} @@ -74,19 +76,11 @@ public WorkflowDefinitionUpdater( @Transaction @Override public void entitySpecificUpdate(boolean consolidatingChanges) { - updateType(); updateTrigger(); updateNodes(); updateEdges(); } - private void updateType() { - if (original.getType() == updated.getType()) { - return; - } - recordChange("type", original.getType(), updated.getType()); - } - private void updateTrigger() { if (original.getTrigger() == updated.getTrigger()) { return; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java index 89c25ac90fed..0391fd005216 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionMapper.java @@ -10,7 +10,6 @@ public class WorkflowDefinitionMapper public WorkflowDefinition createToEntity(CreateWorkflowDefinition create, String user) { return copy(new WorkflowDefinition(), create, user) .withFullyQualifiedName(create.getName()) - .withType(create.getType()) .withTrigger(create.getTrigger()) .withNodes(create.getNodes()) .withEdges(create.getEdges()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java index ee8badb23366..3b4e64951d65 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java @@ -215,7 +215,7 @@ public Response redeploy( new EntityUtil.Fields(repository.getAllowedFields()), Include.NON_DELETED, false); - WorkflowHandler.getInstance().deleteWorkflowDefinition(wd.getName()); + WorkflowHandler.getInstance().deleteWorkflowDefinition(wd); WorkflowHandler.getInstance().deploy(new Workflow(wd)); return Response.status(Response.Status.OK).entity("Workflow Redeployed").build(); } diff --git a/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json b/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json index 84a6d293d769..f14c8acc5152 100644 --- a/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json +++ b/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json @@ -3,9 +3,8 @@ "fullyQualifiedName": "GlossaryTermApprovalWorkflow", "displayName": "Glossary Approval Workflow", "description": "When a Glossary Term is Created or Updated, this Workflow will be triggered for the Term to be Approved.", - "type": "eventBasedEntityWorkflow", "trigger": { - "type": "eventBasedEntityWorkflow", + "type": "eventBasedEntity", "config": { "entityType": "glossaryTerm", "events": ["Created", "Updated"], diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java index 06efb88c6705..ebaa8688ef46 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java @@ -8,18 +8,15 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") @JsonSubTypes({ - @JsonSubTypes.Type( - value = EventBasedEntityTriggerDefinition.class, - name = "eventBasedEntityWorkflow"), - @JsonSubTypes.Type(value = CustomSignalTriggerDefinition.class, name = "customSignalWorkflow"), + @JsonSubTypes.Type(value = EventBasedEntityTriggerDefinition.class, name = "eventBasedEntity"), + @JsonSubTypes.Type(value = CustomSignalTriggerDefinition.class, name = "customSignal"), @JsonSubTypes.Type( value = PeriodicBatchEntityTriggerDefinition.class, - name = "periodicBatchEntityWorkflow"), + name = "periodicBatchEntity"), }) public interface WorkflowTriggerInterface { - // TODO If set as enum, it results in null when the JSON is deserialized. Maybe there can be - // another - // way to validate it on deserialization. + // TODO If set as enum, it results in null when the JSON is deserialized. + // Maybe there can be another way to validate it on deserialization. String getType(); Object getConfig(); diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json index 2e776f679ff9..a2effcf96e9f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json @@ -23,7 +23,7 @@ "properties": { "type": { "type": "string", - "default": "customSignalWorkflow" + "default": "customSignal" }, "config": { "$ref": "#/definitions/config" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json index 8627c163dae6..65bf7f099b18 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json @@ -50,7 +50,7 @@ "properties": { "type": { "type": "string", - "default": "eventBasedEntityWorkflow" + "default": "eventBasedEntity" }, "config": { "$ref": "#/definitions/config" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json index d4759df311c7..e0817dfc8beb 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json @@ -42,7 +42,7 @@ "properties": { "type": { "type": "string", - "default": "periodicBatchEntityWorkflow" + "default": "periodicBatchEntity" }, "config": { "$ref": "#/definitions/config" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json index 17c16c6c715c..4b3ae4cef8ac 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json @@ -11,9 +11,9 @@ "type": "string", "javaType": "org.openmetadata.schema.governance.workflows.TriggerType", "enum": [ - "eventBasedEntityWorkflow", - "customSignalWorkflow", - "periodicBatchEntityWorkflow" + "eventBasedEntity", + "customSignal", + "periodicBatchEntity" ] } }, @@ -68,8 +68,9 @@ "type": "boolean", "default": false }, - "type": { - "$ref": "#/definitions/type" + "deployed": { + "description": "When `true` indicates the workflow is deployed.", + "type": "boolean" }, "trigger": { "description": "Workflow Trigger.", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts index 6457988ab8d7..f4f0418b4f14 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/governance/workflows/workflowDefinition.ts @@ -22,6 +22,10 @@ export interface WorkflowDefinition { * When `true` indicates the entity has been soft deleted. */ deleted?: boolean; + /** + * When `true` indicates the workflow is deployed. + */ + deployed?: boolean; /** * Description of the workflow definition. */ @@ -62,7 +66,6 @@ export interface WorkflowDefinition { * Workflow Trigger. */ trigger?: any[] | boolean | number | number | null | TriggerObject | string; - type?: Type; /** * Last update time corresponding to the new version of the entity in Unix epoch time * milliseconds. @@ -199,7 +202,7 @@ export interface TriggerObject { } export enum Type { - CustomSignalWorkflow = "customSignalWorkflow", - EventBasedEntityWorkflow = "eventBasedEntityWorkflow", - PeriodicBatchEntityWorkflow = "periodicBatchEntityWorkflow", + CustomSignal = "customSignal", + EventBasedEntity = "eventBasedEntity", + PeriodicBatchEntity = "periodicBatchEntity", } From 0d45d4dd8ef0f47b251e927eecea0ce5c56e40d7 Mon Sep 17 00:00:00 2001 From: sushi30 Date: Tue, 21 Jan 2025 16:15:36 +0100 Subject: [PATCH 8/8] fixed migrations --- .../native/1.7.0/mysql/postDataMigrationSQLScript.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql index 078f4495ff92..db7398c184fb 100644 --- a/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql +++ b/bootstrap/sql/migrations/native/1.7.0/mysql/postDataMigrationSQLScript.sql @@ -2,6 +2,6 @@ UPDATE workflow_definition_entity SET json = JSON_SET(json, '$.trigger.type', 'eventBasedEntity') WHERE JSON_EXTRACT(json, '$.trigger.type') = 'eventBasedEntityWorkflow'; -UPDATE your_table_name +UPDATE workflow_definition_entity SET json = JSON_SET(json, '$.trigger.type', 'periodicBatchEntity') WHERE JSON_EXTRACT(json, '$.trigger.type') = 'periodicBatchEntityWorkflow'; \ No newline at end of file