serialized as JSON';
-COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON';
-
-CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex
- ON edc_policydefinitions (policy_id);
-
---
--- Copyright (c) 2023 sovity GmbH
---
--- This program and the accompanying materials are made available under the
--- terms of the Apache License, Version 2.0 which is available at
--- https://www.apache.org/licenses/LICENSE-2.0
---
--- SPDX-License-Identifier: Apache-2.0
---
--- Contributors:
--- sovity GmbH - Update Tables to Milestone-7 EDC
---
---
-ALTER TABLE edc_policydefinitions
- ADD created_at BIGINT;
-
-UPDATE edc_policydefinitions
-SET created_at=EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000;
-
-ALTER TABLE edc_policydefinitions
- ALTER COLUMN created_at SET NOT NULL;
-
--- Statements are designed for and tested with Postgres only!
-
-CREATE TABLE IF NOT EXISTS edc_lease
-(
- leased_by VARCHAR NOT NULL,
- leased_at BIGINT,
- lease_duration INTEGER NOT NULL,
- lease_id VARCHAR NOT NULL
- CONSTRAINT lease_pk
- PRIMARY KEY
-);
-
-COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease';
-
-COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds';
-
-CREATE TABLE IF NOT EXISTS edc_transfer_process
-(
- transferprocess_id VARCHAR NOT NULL
- CONSTRAINT transfer_process_pk
- PRIMARY KEY,
- type VARCHAR NOT NULL,
- state INTEGER NOT NULL,
- state_count INTEGER DEFAULT 0 NOT NULL,
- state_time_stamp BIGINT,
- created_time_stamp BIGINT,
- trace_context JSON,
- error_detail VARCHAR,
- resource_manifest JSON,
- provisioned_resource_set JSON,
- content_data_address JSON,
- deprovisioned_resources JSON,
- lease_id VARCHAR
- CONSTRAINT transfer_process_lease_lease_id_fk
- REFERENCES edc_lease
- ON DELETE SET NULL
-);
-
-COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON';
-
-COMMENT ON COLUMN edc_transfer_process.resource_manifest IS 'java ResourceManifest serialized as JSON';
-
-COMMENT ON COLUMN edc_transfer_process.provisioned_resource_set IS 'ProvisionedResourceSet serialized as JSON';
-
-COMMENT ON COLUMN edc_transfer_process.content_data_address IS 'DataAddress serialized as JSON';
-
-COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of deprovisioned resources, serialized as JSON';
-
-
-CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex
- ON edc_transfer_process (transferprocess_id);
-
-CREATE TABLE IF NOT EXISTS edc_data_request
-(
- datarequest_id VARCHAR NOT NULL
- CONSTRAINT data_request_pk
- PRIMARY KEY,
- process_id VARCHAR NOT NULL,
- connector_address VARCHAR NOT NULL,
- protocol VARCHAR NOT NULL,
- connector_id VARCHAR,
- asset_id VARCHAR NOT NULL,
- contract_id VARCHAR NOT NULL,
- data_destination JSON NOT NULL,
- managed_resources BOOLEAN DEFAULT TRUE,
- properties JSON,
- transfer_type JSON,
- transfer_process_id VARCHAR NOT NULL
- CONSTRAINT data_request_transfer_process_id_fk
- REFERENCES edc_transfer_process
- ON UPDATE RESTRICT ON DELETE CASCADE
-);
-
-COMMENT ON COLUMN edc_data_request.data_destination IS 'DataAddress serialized as JSON';
-
-COMMENT ON COLUMN edc_data_request.properties IS 'java Map serialized as JSON';
-
-COMMENT ON COLUMN edc_data_request.transfer_type IS 'TransferType serialized as JSON';
-
-
-CREATE UNIQUE INDEX IF NOT EXISTS data_request_id_uindex
- ON edc_data_request (datarequest_id);
-
-CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex
- ON edc_lease (lease_id);
-
---
--- Copyright (c) 2023 sovity GmbH
---
--- This program and the accompanying materials are made available under the
--- terms of the Apache License, Version 2.0 which is available at
--- https://www.apache.org/licenses/LICENSE-2.0
---
--- SPDX-License-Identifier: Apache-2.0
---
--- Contributors:
--- sovity GmbH - Update Tables to Milestone-7 EDC
---
---
-
-ALTER TABLE edc_transfer_process
- RENAME COLUMN created_time_stamp TO created_at;
-
-UPDATE edc_transfer_process
-SET created_at = EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000
-WHERE created_at = NULL;
-
-ALTER TABLE edc_transfer_process
- ADD updated_at BIGINT;
-
-UPDATE edc_transfer_process
-SET updated_at=created_at;
-
-ALTER TABLE edc_transfer_process
- ALTER COLUMN updated_at SET NOT NULL;
-ALTER TABLE edc_transfer_process
- ALTER COLUMN created_at SET NOT NULL;
-
---
--- Copyright (c) 2023 sovity GmbH
---
--- This program and the accompanying materials are made available under the
--- terms of the Apache License, Version 2.0 which is available at
--- https://www.apache.org/licenses/LICENSE-2.0
---
--- SPDX-License-Identifier: Apache-2.0
---
--- Contributors:
--- sovity GmbH - Update Tables to Milestone-8 EDC
---
---
-ALTER TABLE edc_transfer_process
- ADD transferprocess_properties JSON;
-
---
--- Copyright (c) 2023 sovity GmbH
---
--- This program and the accompanying materials are made available under the
--- terms of the Apache License, Version 2.0 which is available at
--- https://www.apache.org/licenses/LICENSE-2.0
---
--- SPDX-License-Identifier: Apache-2.0
---
--- Contributors:
--- sovity GmbH - Update Tables to Milestone-8 EDC
---
---
-UPDATE edc_transfer_process
-SET transferprocess_properties = '{}'::json
-WHERE transferprocess_properties IS NULL;
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V2__PoC.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V2__PoC.sql
deleted file mode 100644
index 45481e629..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V2__PoC.sql
+++ /dev/null
@@ -1,94 +0,0 @@
-create type connector_online_status as enum ('ONLINE', 'OFFLINE');
-create type measurement_type as enum ('CONNECTOR_REFRESH');
-create type measurement_error_status as enum ('ERROR', 'OK');
-
-create table connector
-(
- endpoint text not null,
- connector_id text not null,
- created_at timestamp with time zone not null,
- last_refresh_attempt_at timestamp with time zone,
- last_successful_refresh_at timestamp with time zone,
- online_status connector_online_status not null,
-
- PRIMARY KEY (endpoint)
-);
-
-create table data_offer
-(
- connector_endpoint text not null,
- asset_id text not null,
- asset_properties jsonb not null,
- created_at timestamp with time zone not null,
- updated_at timestamp with time zone,
-
- PRIMARY KEY (connector_endpoint, asset_id),
- FOREIGN KEY (connector_endpoint) REFERENCES connector (endpoint)
-);
-
-create table data_offer_contract_offer
-(
- contract_offer_id text not null,
- connector_endpoint text not null,
- asset_id text not null,
- policy jsonb not null,
- created_at timestamp with time zone not null,
- updated_at timestamp with time zone,
-
- PRIMARY KEY (contract_offer_id),
- FOREIGN KEY (connector_endpoint, asset_id) REFERENCES data_offer (connector_endpoint, asset_id),
- FOREIGN KEY (connector_endpoint) REFERENCES connector (endpoint)
-);
-
-create type broker_event_type as enum (
- --Connector was successfully updated, and changes were incorporated
- 'CONNECTOR_UPDATED',
-
- --Connector went online
- 'CONNECTOR_STATUS_CHANGE_ONLINE',
-
- --Connector went offline
- 'CONNECTOR_STATUS_CHANGE_OFFLINE',
-
- --Connector was "force deleted"
- 'CONNECTOR_STATUS_CHANGE_FORCE_DELETED',
-
- --Contract Offer was updated
- 'CONTRACT_OFFER_UPDATED',
-
- --Contract Offer was clicked
- 'CONTRACT_OFFER_CLICK'
-);
-
-create type broker_event_status as enum (
- -- Default
- 'OK',
-
- -- Failures
- 'ERROR'
-);
-
-create table broker_event_log
-(
- id serial primary key,
- created_at timestamp with time zone not null,
- user_message text not null,
- event broker_event_type not null,
- event_status broker_event_status not null,
- connector_endpoint text,
- asset_id text,
- error_stack text,
- duration_in_ms bigint
-);
-
-create table broker_execution_time_measurement
-(
- id serial primary key,
- created_at timestamp with time zone not null,
- connector_endpoint text not null,
- duration_in_ms bigint not null,
- type measurement_type not null,
- error_status measurement_error_status not null
-);
-
-create index speedup on broker_event_log (connector_endpoint, asset_id, event_status);
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V3_1__MvP.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V3_1__MvP.sql
deleted file mode 100644
index ecffc4e51..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V3_1__MvP.sql
+++ /dev/null
@@ -1,36 +0,0 @@
-create collation if not exists alphanumeric_with_natural_sort (provider = icu, locale = 'en-u-kn-true');
-
-create type connector_data_offers_exceeded as enum ('OK', 'EXCEEDED');
-create type connector_contract_offers_exceeded as enum ('OK', 'EXCEEDED');
-
-alter table broker_event_log
- drop column duration_in_ms;
-
-alter table connector
- alter column endpoint type text collate alphanumeric_with_natural_sort,
- add column data_offers_exceeded connector_data_offers_exceeded,
- add column contract_offers_exceeded connector_contract_offers_exceeded;
-
-update connector
-set data_offers_exceeded = 'OK',
- contract_offers_exceeded = 'OK';
-
-alter table connector
- alter column data_offers_exceeded set not null,
- alter column contract_offers_exceeded set not null;
-
-alter table data_offer
- alter column asset_id type text collate alphanumeric_with_natural_sort,
- add column asset_name text collate alphanumeric_with_natural_sort;
-
-update data_offer
-set asset_name = coalesce(asset_properties ->> 'asset:prop:name', asset_id);
-
-alter table data_offer
- alter column asset_name set not null;
-
--- update contract offer table's primary key
-alter table data_offer_contract_offer
- drop constraint data_offer_contract_offer_pkey;
-alter table data_offer_contract_offer
- add primary key (connector_endpoint, asset_id, contract_offer_id);
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V3_2__MvP_Non_Transactional.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V3_2__MvP_Non_Transactional.sql
deleted file mode 100644
index c39525a59..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V3_2__MvP_Non_Transactional.sql
+++ /dev/null
@@ -1,10 +0,0 @@
--- Changes to Enums are non-transactional and must be supplied in a separate migration script for flyway
-
--- Connector Data Offer Limit was exceeded
-alter type broker_event_type add value 'CONNECTOR_DATA_OFFER_LIMIT_EXCEEDED';
--- Connector Data Offer Limit was not exceeded
-alter type broker_event_type add value 'CONNECTOR_DATA_OFFER_LIMIT_OK';
--- Connector Contract Offer Limit was exceeded
-alter type broker_event_type add value 'CONNECTOR_CONTRACT_OFFER_LIMIT_EXCEEDED';
--- Connector Contract Offer Limit was not exceeded
-alter type broker_event_type add value 'CONNECTOR_CONTRACT_OFFER_LIMIT_OK';
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V4_1__MvP_Bugfixes_1_1_0.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V4_1__MvP_Bugfixes_1_1_0.sql
deleted file mode 100644
index da600f9e5..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V4_1__MvP_Bugfixes_1_1_0.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-create table data_offer_view_count (
- id serial primary key,
- connector_endpoint text not null,
- asset_id text not null,
- date timestamp with time zone not null
-);
-
-create index data_offer_view_count_speedup on data_offer_view_count (connector_endpoint, asset_id);
-
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V4_2__MvP_Bugfixes_Non_Transactional.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V4_2__MvP_Bugfixes_Non_Transactional.sql
deleted file mode 100644
index 986e58cfd..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V4_2__MvP_Bugfixes_Non_Transactional.sql
+++ /dev/null
@@ -1,5 +0,0 @@
--- Changes to Enums are non-transactional and must be supplied in a separate migration script for flyway
-
--- Connector deleted due to being offline for too long
-alter type broker_event_type add value 'CONNECTOR_KILLED_DUE_TO_OFFLINE_FOR_TOO_LONG';
-alter type connector_online_status add value 'DEAD';
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V5_1__MvP_Fix_Asset_JSON_Properties.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V5_1__MvP_Fix_Asset_JSON_Properties.sql
deleted file mode 100644
index fea8c8173..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V5_1__MvP_Fix_Asset_JSON_Properties.sql
+++ /dev/null
@@ -1,18 +0,0 @@
--- Maps JSON Asset Properties to String
--- '{"a": "b", "c": [1, 2], "d": true}'::jsonb becomes '{"a": "b", "c": "[1, 2]", "d": "true"}'::jsonb
-create
-or replace function pg_temp.migrate_asset_properties(asset_properties jsonb) returns jsonb as
-$$
-begin
-return (select jsonb_object_agg(key, case when jsonb_typeof(value) = 'string' then value #>> '{}' else value::text end)
- from jsonb_each(asset_properties));
-end;
-$$
-language plpgsql;
-
--- Fix existing data offer asssets
-update data_offer
-set asset_properties = pg_temp.migrate_asset_properties(asset_properties);
-
--- Add new Event Log Status
-alter type broker_event_type add value 'CONNECTOR_DELETED';
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V6__EDC0.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V6__EDC0.sql
deleted file mode 100644
index c8edbe6fc..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V6__EDC0.sql
+++ /dev/null
@@ -1,166 +0,0 @@
--- Migration Script for Broker from MS8 to EDC 0
-
--- Migrates an Asset ID
-create
- or replace function pg_temp.migrate_asset_id(asset_id text) returns text as
-$$
-begin
- return replace(replace(asset_id::text, 'urn:artifact:', ''), ':', '-');
-end;
-$$
- language plpgsql;
-
--- Migrates a Connector Endpoint to EDC 0
-create
- or replace function pg_temp.migrate_connector_endpoint(endpoint text) returns text as
-$$
-begin
- return pg_temp.replace_suffix(endpoint, '/api/v1/ids/data', '/api/dsp');
-end;
-$$
- language plpgsql;
-
--- Creates a valid Asset JSON-LD from an Asset ID and Asset Title
-create
- or replace function pg_temp.build_asset_json_ld(asset_id text, asset_title text) returns jsonb as
-$$
-begin
- return jsonb_build_object(
- '@id', asset_id,
- 'https://w3id.org/edc/v0.0.1/ns/properties', jsonb_build_object(
- 'https://w3id.org/edc/v0.0.1/ns/id', asset_id,
- 'http://purl.org/dc/terms/title', asset_title
- )
- );
-end;
-$$
- language plpgsql;
-
--- Utility Function: replaceSuffix
-create
- or replace function pg_temp.replace_suffix(str text, old_suffix text, new_suffix text) returns text as
-$$
-begin
- return case
- when pg_temp.ends_with(str, old_suffix) then
- left(str, length(str) - length(old_suffix)) || new_suffix
- else
- str
- end;
-end;
-$$
- language plpgsql;
-
--- Utility Function: endsWith
-create or replace function pg_temp.ends_with(str text, suffix text)
- returns boolean as
-$$
-begin
- return right(str, length(suffix)) = suffix;
-end;
-$$ language plpgsql;
-
--- Utility Function: Drops fkey constraints that have auto-generated names. Different Postgresql versions generated different names.
-create or replace function pg_temp.drop_constraints_containing_fkey(table_name text)
- returns void as
-$$
-declare
- i record;
-begin
- for i in (select conname
- from pg_catalog.pg_constraint con
- inner join pg_catalog.pg_class rel on rel.oid = con.conrelid
- inner join pg_catalog.pg_namespace nsp on nsp.oid = connamespace
- where rel.relname = table_name
- and conname like '%fkey%')
- loop
- execute format('alter table %s drop constraint %s', table_name, i.conname);
- end loop;
-end;
-$$ language plpgsql;
-
-
--- Remove Connector Tables
--- All connector tables should be empty
--- There should be no references from broker tables to connector tables
-drop table edc_asset cascade;
-drop table edc_asset_dataaddress cascade;
-drop table edc_asset_property cascade;
-drop table edc_contract_agreement cascade;
-drop table edc_contract_definitions cascade;
-drop table edc_contract_negotiation cascade;
-drop table edc_data_plane_instance cascade;
-drop table edc_data_request cascade;
-drop table edc_lease cascade;
-drop table edc_policydefinitions cascade;
-drop table edc_transfer_process cascade;
-
-
--- Drop constraints
-select pg_temp.drop_constraints_containing_fkey('data_offer');
-select pg_temp.drop_constraints_containing_fkey('data_offer_contract_offer');
-
--- Migrate Connector Endpoints
-update broker_event_log
-set connector_endpoint = pg_temp.migrate_connector_endpoint(connector_endpoint);
-update broker_execution_time_measurement
-set connector_endpoint = pg_temp.migrate_connector_endpoint(connector_endpoint);
-update connector
-set endpoint = pg_temp.migrate_connector_endpoint(endpoint);
-update data_offer
-set connector_endpoint = pg_temp.migrate_connector_endpoint(connector_endpoint);
-update data_offer_contract_offer
-set connector_endpoint = pg_temp.migrate_connector_endpoint(connector_endpoint);
-update data_offer_view_count
-set connector_endpoint = pg_temp.migrate_connector_endpoint(connector_endpoint);
-
-
--- Migrate Asset IDs
-update broker_event_log
-set asset_id = pg_temp.migrate_asset_id(asset_id);
-update data_offer
-set asset_id = pg_temp.migrate_asset_id(asset_id);
-update data_offer_contract_offer
-set asset_id = pg_temp.migrate_asset_id(asset_id);
-update data_offer_view_count
-set asset_id = pg_temp.migrate_asset_id(asset_id);
-
--- Rename data_offer_contract_offer to contract_offer
-alter table data_offer_contract_offer
- rename to contract_offer;
-
--- Rename Connector ID to Participant ID
-alter table connector
- rename column connector_id to participant_id;
-
--- Add constraints
-alter table data_offer
- add constraint data_offer_connector_endpoint_fkey
- foreign key (connector_endpoint) references connector (endpoint);
-alter table contract_offer
- add constraint contract_offer_data_offer_fkey
- foreign key (connector_endpoint, asset_id) references data_offer (connector_endpoint, asset_id);
-alter table contract_offer
- add constraint contract_offer_connector_fkey
- foreign key (connector_endpoint) references connector (endpoint);
-
--- Migrate to Asset JSON-LD
-alter table data_offer
- rename column asset_properties to asset_json_ld;
-alter table data_offer
- rename column asset_name to asset_title;
-update data_offer
-set asset_json_ld = pg_temp.build_asset_json_ld(asset_id, asset_title);
-
--- Extracted Asset Metadata from the JSON-LD for Search / Filtering
-alter table data_offer
- add column description text not null default '',
- add column curator_organization_name text not null default '',
- add column data_category text not null default '',
- add column data_subcategory text not null default '',
- add column data_model text not null default '',
- add column transport_mode text not null default '',
- add column geo_reference_method text not null default '',
- add column keywords text[] not null default '{}',
- -- comma joined keywords for easier search
- add column keywords_comma_joined text not null default '';
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V7__Organization_Metadata.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V7__Organization_Metadata.sql
deleted file mode 100644
index d3e19bca0..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/migration/V7__Organization_Metadata.sql
+++ /dev/null
@@ -1,11 +0,0 @@
--- Create table for organization metadata
-create table organization_metadata
-(
- mds_id text not null primary key,
- name text not null
-);
-
--- Add MDS-ID column to organization table
-alter table connector add column mds_id text;
-update connector set mds_id = split_part(participant_id, '.', 1)
-where participant_id ~ '^MDSL[A-Za-z0-9]+\.C[A-Za-z0-9]+$';
diff --git a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/testdata/V2_1__PoC_Test_Data.sql b/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/testdata/V2_1__PoC_Test_Data.sql
deleted file mode 100644
index fd42a12dd..000000000
--- a/extensions/broker-server-postgres-flyway-jooq/src/main/resources/db/testdata/V2_1__PoC_Test_Data.sql
+++ /dev/null
@@ -1,69 +0,0 @@
--- Test Data to be added after V2 so we can test subsequent migrations
-
-insert into connector (endpoint, connector_id, created_at, last_refresh_attempt_at, last_successful_refresh_at,
- online_status)
-values ('https://my-connector.com/api/v1/ids/data', 'test-connector-1', '2019-01-01 00:00:00',
- '2019-01-01 00:00:00', '2019-01-01 00:00:00', 'ONLINE');
-insert into data_offer (connector_endpoint, asset_id, asset_properties, created_at, updated_at)
-values ('https://my-connector.com/api/v1/ids/data',
- 'test-asset-1',
- '{
- "asset:prop:id": "test-asset-1"
- }',
- '2019-01-01 00:00:00',
- '2019-01-01 00:00:00'),
- ('https://my-connector.com/api/v1/ids/data',
- 'test-asset-2',
- '{
- "asset:prop:id": "urn:artifact:db-rail-network-2023-jan",
- "asset:prop:name": "Rail Network DB 2023 January",
- "asset:prop:version": "1.1",
- "asset:prop:originator": "https://example-connector.rail-mgmt.bahn.de/api/v1/api/v1/ids/data",
- "asset:prop:originatorOrganization": "Deutsche Bahn AG",
- "asset:prop:keywords": "db, bahn, rail, Rail-Designer",
- "asset:prop:contenttype": "application/json",
- "asset:prop:description": "Train Network Map released on 10.01.2023, valid until 31.02.2023. \nFile format is xyz as exported by Rail-Designer.",
- "asset:prop:language": "https://w3id.org/idsa/code/EN",
- "asset:prop:publisher": "https://my.cool-api.gg/about",
- "asset:prop:standardLicense": "https://my.cool-api.gg/license",
- "asset:prop:endpointDocumentation": "https://my.cool-api.gg/docs",
- "http://w3id.org/mds#dataCategory": "Infrastructure and Logistics",
- "http://w3id.org/mds#dataSubcategory": "General Information About Planning Of Routes",
- "http://w3id.org/mds#dataModel": "my-data-model-001",
- "http://w3id.org/mds#geoReferenceMethod": "my-geo-reference-method",
- "http://w3id.org/mds#transportMode": "Rail"
- }',
- '2019-01-01 00:00:00',
- '2019-01-01 00:00:00');
-
-insert into data_offer_contract_offer (contract_offer_id, connector_endpoint, asset_id, policy, created_at, updated_at)
-values ('test-contract-offer-1',
- 'https://my-connector.com/api/v1/ids/data',
- 'test-asset-1',
- '"test-policy-1"',
- '2019-01-01 00:00:00',
- '2019-01-01 00:00:00'),
- ('test-contract-offer-2',
- 'https://my-connector.com/api/v1/ids/data',
- 'test-asset-2',
- '"test-policy-2"',
- '2019-01-01 00:00:00',
- '2019-01-01 00:00:00');
-
-insert into broker_event_log (created_at, user_message, event, event_status, connector_endpoint, asset_id, error_stack,
- duration_in_ms)
-values ('2019-01-01 00:00:00',
- 'Connector was successfully updated, and changes were incorporated',
- 'CONNECTOR_UPDATED',
- 'OK',
- 'https://my-connector.com/api/v1/ids/data',
- 'test-asset-1',
- null,
- 100);
-
-insert into broker_execution_time_measurement (connector_endpoint, created_at, type, error_status, duration_in_ms)
-values ('https://my-connector.com/api/v1/ids/data',
- '2019-01-01 00:00:00',
- 'CONNECTOR_REFRESH',
- 'OK',
- 100);
diff --git a/extensions/broker-server/README.md b/extensions/broker-server/README.md
deleted file mode 100644
index 103af330b..000000000
--- a/extensions/broker-server/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-
-
-
-
-## About this Extension
-
-Implementation of an EDC Broker backend as an EDC Extension.
-
-This extension does multiple things:
-
-- Storage of Connectors and Data Offers
-- Connector Crawling
-- Connector Discovery
-- API implementations for our the full management capabilities of our Broker UI
-
-## Why does this extension exist?
-
-To let the broker easily be a part of a data space we are implementing it on an EDC basis.
-
-## License
-
-Apache License 2.0 - see [LICENSE](../../LICENSE)
-
-## Contact
-
-sovity GmbH - contact@sovity.de
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtension.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtension.java
deleted file mode 100644
index 3b288ea13..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtension.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration;
-import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry;
-import org.eclipse.edc.connector.spi.catalog.CatalogService;
-import org.eclipse.edc.jsonld.spi.JsonLd;
-import org.eclipse.edc.runtime.metamodel.annotation.Inject;
-import org.eclipse.edc.runtime.metamodel.annotation.Setting;
-import org.eclipse.edc.spi.system.ServiceExtension;
-import org.eclipse.edc.spi.system.ServiceExtensionContext;
-import org.eclipse.edc.spi.types.TypeManager;
-import org.eclipse.edc.web.spi.WebService;
-
-import static de.sovity.edc.ext.brokerserver.services.config.EdcConfigPropertyUtils.toEdcProp;
-
-public class BrokerServerExtension implements ServiceExtension {
-
- public static final String EXTENSION_NAME = "BrokerServerExtension";
-
- @Setting
- public static final String ADMIN_API_KEY = toEdcProp("EDC_BROKER_SERVER_ADMIN_API_KEY");
-
- @Setting
- public static final String KNOWN_CONNECTORS = toEdcProp("EDC_BROKER_SERVER_KNOWN_CONNECTORS");
-
- @Setting
- public static final String CRON_ONLINE_CONNECTOR_REFRESH = toEdcProp("EDC_BROKER_SERVER_CRON_ONLINE_CONNECTOR_REFRESH");
-
- @Setting
- public static final String CRON_OFFLINE_CONNECTOR_REFRESH = toEdcProp("EDC_BROKER_SERVER_CRON_OFFLINE_CONNECTOR_REFRESH");
-
- @Setting
- public static final String CRON_DEAD_CONNECTOR_REFRESH = toEdcProp("EDC_BROKER_SERVER_CRON_DEAD_CONNECTOR_REFRESH");
-
- @Setting
- public static final String NUM_THREADS = toEdcProp("EDC_BROKER_SERVER_NUM_THREADS");
-
- @Setting
- public static final String HIDE_OFFLINE_DATA_OFFERS_AFTER = toEdcProp("EDC_BROKER_SERVER_HIDE_OFFLINE_DATA_OFFERS_AFTER");
-
- @Setting
- public static final String MAX_DATA_OFFERS_PER_CONNECTOR = toEdcProp("EDC_BROKER_SERVER_MAX_DATA_OFFERS_PER_CONNECTOR");
-
- @Setting
- public static final String MAX_CONTRACT_OFFERS_PER_DATA_OFFER = toEdcProp("EDC_BROKER_SERVER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER");
-
- @Setting
- public static final String CATALOG_PAGE_PAGE_SIZE = toEdcProp("EDC_BROKER_SERVER_CATALOG_PAGE_PAGE_SIZE");
-
- @Setting
- public static final String DEFAULT_CONNECTOR_DATASPACE = toEdcProp("EDC_BROKER_SERVER_DEFAULT_DATASPACE");
-
- @Setting
- public static final String KNOWN_DATASPACE_CONNECTORS = toEdcProp("EDC_BROKER_SERVER_KNOWN_DATASPACE_CONNECTORS");
-
- @Setting
- public static final String KILL_OFFLINE_CONNECTORS_AFTER = toEdcProp("EDC_BROKER_SERVER_KILL_OFFLINE_CONNECTORS_AFTER");
-
- @Setting
- public static final String SCHEDULED_KILL_OFFLINE_CONNECTORS = toEdcProp("EDC_BROKER_SERVER_SCHEDULED_KILL_OFFLINE_CONNECTORS");
-
- @Inject
- private ManagementApiConfiguration managementApiConfiguration;
-
- @Inject
- private WebService webService;
-
- @Inject
- private TypeManager typeManager;
-
- @Inject
- private ManagementApiTypeTransformerRegistry typeTransformerRegistry;
-
- @Inject
- private JsonLd jsonLd;
-
- @Inject
- private CatalogService catalogService;
-
- /**
- * Manual Dependency Injection Result
- */
- private BrokerServerExtensionContext services;
-
- @Override
- public String name() {
- return EXTENSION_NAME;
- }
-
- @Override
- public void initialize(ServiceExtensionContext context) {
- services = BrokerServerExtensionContextBuilder.buildContext(
- context.getConfig(),
- context.getMonitor(),
- typeManager,
- typeTransformerRegistry,
- jsonLd,
- catalogService
- );
-
- // This is a hack for tests, so we can access the running context from tests.
- BrokerServerExtensionContext.instance = services;
-
- var managementApiGroup = managementApiConfiguration.getContextAlias();
- webService.registerResource(managementApiGroup, services.brokerServerResource());
- }
-
- @Override
- public void start() {
- services.brokerServerInitializer().onStartup();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContext.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContext.java
deleted file mode 100644
index 3dd8bd2fa..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContext.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import de.sovity.edc.ext.brokerserver.api.BrokerServerResource;
-import de.sovity.edc.ext.brokerserver.services.BrokerServerInitializer;
-import de.sovity.edc.ext.brokerserver.services.ConnectorCreator;
-import de.sovity.edc.ext.brokerserver.services.refreshing.ConnectorUpdater;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferRecordUpdater;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.FetchedCatalogBuilder;
-import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
-
-
-/**
- * Manual Dependency Injection result
- *
- * @param brokerServerResource REST Resource with API Endpoint implementations
- * @param brokerServerInitializer Startup Logic
- */
-public record BrokerServerExtensionContext(
- BrokerServerResource brokerServerResource,
- BrokerServerInitializer brokerServerInitializer,
-
- // Required for Integration Tests
- ConnectorUpdater connectorUpdater,
- ConnectorCreator connectorCreator,
- PolicyMapper policyMapper,
- FetchedCatalogBuilder fetchedCatalogBuilder,
- DataOfferRecordUpdater dataOfferRecordUpdater
-) {
- /**
- * This is a hack for our tests.
- *
- * Right now we have no good way to access the context from tests.
- */
- public static BrokerServerExtensionContext instance;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContextBuilder.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContextBuilder.java
deleted file mode 100644
index 8f78ffa67..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerExtensionContextBuilder.java
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import de.sovity.edc.ext.brokerserver.dao.ConnectorQueries;
-import de.sovity.edc.ext.brokerserver.dao.ContractOfferQueries;
-import de.sovity.edc.ext.brokerserver.dao.DataOfferQueries;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryAvailableFilterFetcher;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryContractOfferFetcher;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryDataOfferFetcher;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFilterService;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQuerySortingService;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.ConnectorDetailQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.ConnectorListQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.DataOfferDetailPageQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.ViewCountLogger;
-import de.sovity.edc.ext.brokerserver.db.DataSourceFactory;
-import de.sovity.edc.ext.brokerserver.db.DslContextFactory;
-import de.sovity.edc.ext.brokerserver.services.BrokerServerInitializer;
-import de.sovity.edc.ext.brokerserver.services.ConnectorCleaner;
-import de.sovity.edc.ext.brokerserver.services.ConnectorCreator;
-import de.sovity.edc.ext.brokerserver.services.ConnectorKiller;
-import de.sovity.edc.ext.brokerserver.services.KnownConnectorsInitializer;
-import de.sovity.edc.ext.brokerserver.services.OfflineConnectorKiller;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorDataOfferApiService;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorMetadataApiService;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorQueryService;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalOrganizationMetadataApiService;
-import de.sovity.edc.ext.brokerserver.services.api.CatalogApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorDetailApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorListApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorOnlineStatusMapper;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorService;
-import de.sovity.edc.ext.brokerserver.services.api.DataOfferDetailApiService;
-import de.sovity.edc.ext.brokerserver.services.api.DataOfferMappingUtils;
-import de.sovity.edc.ext.brokerserver.services.api.PaginationMetadataUtils;
-import de.sovity.edc.ext.brokerserver.services.api.filtering.CatalogFilterAttributeDefinitionService;
-import de.sovity.edc.ext.brokerserver.services.api.filtering.CatalogFilterService;
-import de.sovity.edc.ext.brokerserver.services.api.filtering.CatalogSearchService;
-import de.sovity.edc.ext.brokerserver.services.config.AdminApiKeyValidator;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettingsFactory;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerExecutionTimeLogger;
-import de.sovity.edc.ext.brokerserver.services.queue.ConnectorQueue;
-import de.sovity.edc.ext.brokerserver.services.queue.ConnectorQueueFiller;
-import de.sovity.edc.ext.brokerserver.services.queue.ThreadPool;
-import de.sovity.edc.ext.brokerserver.services.queue.ThreadPoolTaskQueue;
-import de.sovity.edc.ext.brokerserver.services.refreshing.ConnectorUpdateFailureWriter;
-import de.sovity.edc.ext.brokerserver.services.refreshing.ConnectorUpdateSuccessWriter;
-import de.sovity.edc.ext.brokerserver.services.refreshing.ConnectorUpdater;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.CatalogFetcher;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.ContractOfferRecordUpdater;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferLimitsEnforcer;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferPatchApplier;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferPatchBuilder;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferRecordUpdater;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferWriter;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.FetchedCatalogBuilder;
-import de.sovity.edc.ext.brokerserver.services.schedules.DeadConnectorRefreshJob;
-import de.sovity.edc.ext.brokerserver.services.schedules.OfflineConnectorKillerJob;
-import de.sovity.edc.ext.brokerserver.services.schedules.OfflineConnectorRefreshJob;
-import de.sovity.edc.ext.brokerserver.services.schedules.OnlineConnectorRefreshJob;
-import de.sovity.edc.ext.brokerserver.services.schedules.QuartzScheduleInitializer;
-import de.sovity.edc.ext.brokerserver.services.schedules.utils.CronJobRef;
-import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdParser;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.AssetJsonLdUtils;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.EdcPropertyUtils;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder;
-import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.DataSourceMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.policy.AtomicConstraintMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.policy.ConstraintExtractor;
-import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator;
-import de.sovity.edc.utils.catalog.DspCatalogService;
-import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder;
-import lombok.NoArgsConstructor;
-import org.eclipse.edc.connector.spi.catalog.CatalogService;
-import org.eclipse.edc.jsonld.spi.JsonLd;
-import org.eclipse.edc.runtime.metamodel.annotation.Inject;
-import org.eclipse.edc.spi.CoreConstants;
-import org.eclipse.edc.spi.monitor.Monitor;
-import org.eclipse.edc.spi.system.configuration.Config;
-import org.eclipse.edc.spi.types.TypeManager;
-import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.List;
-
-
-/**
- * Manual Dependency Injection (DYDI).
- *
- * We want to develop as Java Backend Development is done, but we have
- * no CDI / DI Framework to rely on.
- *
- * EDC {@link Inject} only works in {@link BrokerServerExtension}.
- */
-@NoArgsConstructor(access = lombok.AccessLevel.PRIVATE)
-public class BrokerServerExtensionContextBuilder {
-
- public static BrokerServerExtensionContext buildContext(
- Config config,
- Monitor monitor,
- TypeManager typeManager,
- TypeTransformerRegistry typeTransformerRegistry,
- JsonLd jsonLd,
- CatalogService catalogService
- ) {
- var brokerServerSettingsFactory = new BrokerServerSettingsFactory(config, monitor);
- var brokerServerSettings = brokerServerSettingsFactory.buildBrokerServerSettings();
- var adminApiKeyValidator = new AdminApiKeyValidator(brokerServerSettings);
-
- // Dao
- var dataOfferQueries = new DataOfferQueries();
- var dataSourceFactory = new DataSourceFactory(config);
- var dataSource = dataSourceFactory.newDataSource();
- var dslContextFactory = new DslContextFactory(dataSource);
- var connectorQueries = new ConnectorQueries();
- var catalogQuerySortingService = new CatalogQuerySortingService();
- var catalogSearchService = new CatalogSearchService();
- var catalogQueryFilterService = new CatalogQueryFilterService(brokerServerSettings, catalogSearchService);
- var catalogQueryContractOfferFetcher = new CatalogQueryContractOfferFetcher();
- var catalogQueryDataOfferFetcher = new CatalogQueryDataOfferFetcher(
- catalogQuerySortingService,
- catalogQueryFilterService,
- catalogQueryContractOfferFetcher
- );
- var catalogQueryAvailableFilterFetcher = new CatalogQueryAvailableFilterFetcher(catalogQueryFilterService);
- var catalogQueryService = new CatalogQueryService(
- catalogQueryDataOfferFetcher,
- catalogQueryAvailableFilterFetcher,
- brokerServerSettings
- );
- var connectorListQueryService = new ConnectorListQueryService();
- var connectorDetailQueryService = new ConnectorDetailQueryService();
- var dataOfferDetailPageQueryService = new DataOfferDetailPageQueryService(
- catalogQueryContractOfferFetcher, brokerServerSettings);
-
-
- // Services
- var objectMapperJsonLd = getJsonLdObjectMapper(typeManager);
- var brokerEventLogger = new BrokerEventLogger();
- var brokerExecutionTimeLogger = new BrokerExecutionTimeLogger();
- var contractOfferRecordUpdater = new ContractOfferRecordUpdater();
- var dataOfferRecordUpdater = new DataOfferRecordUpdater();
- var contractOfferQueries = new ContractOfferQueries();
- var dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer(brokerServerSettings, brokerEventLogger);
- var dataOfferPatchBuilder = new DataOfferPatchBuilder(
- contractOfferQueries,
- dataOfferQueries,
- dataOfferRecordUpdater,
- contractOfferRecordUpdater
- );
- var dataOfferPatchApplier = new DataOfferPatchApplier();
- var dataOfferWriter = new DataOfferWriter(dataOfferPatchBuilder, dataOfferPatchApplier);
- var connectorUpdateSuccessWriter = new ConnectorUpdateSuccessWriter(
- brokerEventLogger,
- dataOfferWriter,
- dataOfferLimitsEnforcer
- );
- var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd);
- var fetchedDataOfferBuilder = new FetchedCatalogBuilder(assetMapper);
- var dspDataOfferBuilder = new DspDataOfferBuilder(jsonLd);
- var dspCatalogService = new DspCatalogService(
- catalogService,
- dspDataOfferBuilder
- );
- var dataOfferFetcher = new CatalogFetcher(dspCatalogService, fetchedDataOfferBuilder);
- var connectorUpdateFailureWriter = new ConnectorUpdateFailureWriter(brokerEventLogger, monitor);
- var connectorUpdater = new ConnectorUpdater(
- dataOfferFetcher,
- connectorUpdateSuccessWriter,
- connectorUpdateFailureWriter,
- connectorQueries,
- dslContextFactory,
- monitor,
- brokerExecutionTimeLogger
- );
- var paginationMetadataUtils = new PaginationMetadataUtils();
- var threadPoolTaskQueue = new ThreadPoolTaskQueue();
- var threadPool = new ThreadPool(threadPoolTaskQueue, brokerServerSettings, monitor);
- var connectorQueue = new ConnectorQueue(connectorUpdater, threadPool);
- var connectorQueueFiller = new ConnectorQueueFiller(connectorQueue, connectorQueries);
- var connectorCreator = new ConnectorCreator(connectorQueries);
- var knownConnectorsInitializer = new KnownConnectorsInitializer(
- config,
- connectorQueue,
- connectorCreator
- );
- var catalogFilterAttributeDefinitionService = new CatalogFilterAttributeDefinitionService();
- var catalogFilterService = new CatalogFilterService(catalogFilterAttributeDefinitionService);
- var viewCountLogger = new ViewCountLogger();
- var connectorService = new ConnectorService(connectorCreator, connectorQueue);
- var connectorKiller = new ConnectorKiller();
- var connectorClearer = new ConnectorCleaner();
- var offlineConnectorKiller = new OfflineConnectorKiller(
- brokerServerSettings,
- connectorQueries,
- brokerEventLogger,
- connectorKiller,
- connectorClearer
- );
- var operatorMapper = new OperatorMapper();
- var literalMapper = new LiteralMapper(
- objectMapperJsonLd
- );
- var atomicConstraintMapper = new AtomicConstraintMapper(
- literalMapper,
- operatorMapper
- );
- var policyValidator = new PolicyValidator();
- var constraintExtractor = new ConstraintExtractor(
- policyValidator,
- atomicConstraintMapper
- );
- var policyMapper = new PolicyMapper(
- constraintExtractor,
- atomicConstraintMapper,
- typeTransformerRegistry
- );
- var dataOfferMappingUtils = new DataOfferMappingUtils(
- policyMapper,
- assetMapper
- );
- var connectorOnlineStatusMapper = new ConnectorOnlineStatusMapper();
-
- // Schedules
- List> jobs = List.of(
- getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller),
- getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller),
- getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller),
- getOfflineConnectorKillerCronJob(dslContextFactory, offlineConnectorKiller)
- );
-
- // Startup
- var quartzScheduleInitializer = new QuartzScheduleInitializer(config, monitor, jobs);
- var brokerServerInitializer = new BrokerServerInitializer(
- dslContextFactory,
- knownConnectorsInitializer,
- quartzScheduleInitializer
- );
-
- // UI Capabilities
- var catalogApiService = new CatalogApiService(
- paginationMetadataUtils,
- catalogQueryService,
- dataOfferMappingUtils,
- catalogFilterService,
- brokerServerSettings
- );
- var connectorApiService = new ConnectorApiService(
- connectorService,
- brokerEventLogger,
- connectorQueries
- );
- var dataOfferDetailApiService = new DataOfferDetailApiService(
- dataOfferDetailPageQueryService,
- viewCountLogger,
- dataOfferMappingUtils
- );
- var connectorQueryService = new AuthorityPortalConnectorQueryService();
- var dataOfferCountApiService = new AuthorityPortalConnectorMetadataApiService(
- connectorQueryService,
- connectorOnlineStatusMapper
- );
- var connectorDetailApiService = new ConnectorDetailApiService(connectorDetailQueryService, connectorOnlineStatusMapper);
- var connectorListApiService = new ConnectorListApiService(connectorListQueryService, connectorOnlineStatusMapper, paginationMetadataUtils);
- var authorityPortalOrganizationMetadataApiService = new AuthorityPortalOrganizationMetadataApiService();
- var authorityPortalDataOfferApiService = new AuthorityPortalConnectorDataOfferApiService(connectorQueryService, connectorOnlineStatusMapper);
- var brokerServerResource = new BrokerServerResourceImpl(
- dslContextFactory,
- connectorApiService,
- connectorListApiService,
- connectorDetailApiService,
- catalogApiService,
- dataOfferDetailApiService,
- adminApiKeyValidator,
- dataOfferCountApiService,
- authorityPortalDataOfferApiService,
- authorityPortalOrganizationMetadataApiService
- );
-
- return new BrokerServerExtensionContext(
- brokerServerResource,
- brokerServerInitializer,
- connectorUpdater,
- connectorCreator,
- policyMapper,
- fetchedDataOfferBuilder,
- dataOfferRecordUpdater
- );
- }
-
- @NotNull
- private static CronJobRef getOfflineConnectorKillerCronJob(DslContextFactory dslContextFactory, OfflineConnectorKiller offlineConnectorKiller) {
- return new CronJobRef<>(
- BrokerServerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS,
- OfflineConnectorKillerJob.class,
- () -> new OfflineConnectorKillerJob(dslContextFactory, offlineConnectorKiller)
- );
- }
-
- @NotNull
- private static CronJobRef getOnlineConnectorRefreshCronJob(DslContextFactory dslContextFactory, ConnectorQueueFiller connectorQueueFiller) {
- return new CronJobRef<>(
- BrokerServerExtension.CRON_ONLINE_CONNECTOR_REFRESH,
- OnlineConnectorRefreshJob.class,
- () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller)
- );
- }
-
- @NotNull
- private static CronJobRef getOfflineConnectorRefreshCronJob(DslContextFactory dslContextFactory, ConnectorQueueFiller connectorQueueFiller) {
- return new CronJobRef<>(
- BrokerServerExtension.CRON_OFFLINE_CONNECTOR_REFRESH,
- OfflineConnectorRefreshJob.class,
- () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller)
- );
- }
-
- @NotNull
- private static CronJobRef getDeadConnectorRefreshCronJob(DslContextFactory dslContextFactory, ConnectorQueueFiller connectorQueueFiller) {
- return new CronJobRef<>(
- BrokerServerExtension.CRON_DEAD_CONNECTOR_REFRESH,
- DeadConnectorRefreshJob.class,
- () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller)
- );
- }
-
- private static ObjectMapper getJsonLdObjectMapper(TypeManager typeManager) {
- var objectMapper = typeManager.getMapper(CoreConstants.JSON_LD);
-
- // Fixes Dates in JSON-LD Object Mapper
- // The Core EDC uses longs over OffsetDateTime, so they never fixed the date format
- objectMapper.registerModule(new JavaTimeModule());
- objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
-
- return objectMapper;
- }
-
- @NotNull
- private static AssetMapper newAssetMapper(TypeTransformerRegistry typeTransformerRegistry, JsonLd jsonLd) {
- var edcPropertyUtils = new EdcPropertyUtils();
- var assetJsonLdUtils = new AssetJsonLdUtils();
- var assetEditRequestMapper = new AssetEditRequestMapper();
- var shortDescriptionBuilder = new ShortDescriptionBuilder();
- var assetJsonLdParser = new AssetJsonLdParser(
- assetJsonLdUtils,
- shortDescriptionBuilder,
- endpoint -> false
- );
- var httpHeaderMapper = new HttpHeaderMapper();
- var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper);
- var dataSourceMapper = new DataSourceMapper(
- edcPropertyUtils,
- httpDataSourceMapper
- );
- var assetJsonLdBuilder = new AssetJsonLdBuilder(
- dataSourceMapper,
- assetJsonLdParser,
- assetEditRequestMapper
- );
- return new AssetMapper(
- typeTransformerRegistry,
- assetJsonLdBuilder,
- assetJsonLdParser,
- jsonLd
- );
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerResourceImpl.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerResourceImpl.java
deleted file mode 100644
index 7f38ea0d1..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/BrokerServerResourceImpl.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import de.sovity.edc.ext.brokerserver.api.BrokerServerResource;
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorDataOfferInfo;
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorInfo;
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalOrganizationMetadataRequest;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageResult;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorCreationRequest;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageResult;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageResult;
-import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageResult;
-import de.sovity.edc.ext.brokerserver.db.DslContextFactory;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorDataOfferApiService;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalConnectorMetadataApiService;
-import de.sovity.edc.ext.brokerserver.services.api.AuthorityPortalOrganizationMetadataApiService;
-import de.sovity.edc.ext.brokerserver.services.api.CatalogApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorDetailApiService;
-import de.sovity.edc.ext.brokerserver.services.api.ConnectorListApiService;
-import de.sovity.edc.ext.brokerserver.services.api.DataOfferDetailApiService;
-import de.sovity.edc.ext.brokerserver.services.config.AdminApiKeyValidator;
-import lombok.RequiredArgsConstructor;
-
-import java.util.List;
-
-
-/**
- * Implementation of {@link BrokerServerResource}
- */
-@RequiredArgsConstructor
-public class BrokerServerResourceImpl implements BrokerServerResource {
- private final DslContextFactory dslContextFactory;
- private final ConnectorApiService connectorApiService;
- private final ConnectorListApiService connectorListApiService;
- private final ConnectorDetailApiService connectorDetailApiService;
- private final CatalogApiService catalogApiService;
- private final DataOfferDetailApiService dataOfferDetailApiService;
- private final AdminApiKeyValidator adminApiKeyValidator;
- private final AuthorityPortalConnectorMetadataApiService authorityPortalConnectorMetadataApiService;
- private final AuthorityPortalConnectorDataOfferApiService authorityPortalConnectorDataOffersApiService;
- private final AuthorityPortalOrganizationMetadataApiService authorityPortalOrganizationMetadataApiService;
-
- @Override
- public CatalogPageResult catalogPage(CatalogPageQuery query) {
- return dslContextFactory.transactionResult(dsl -> catalogApiService.catalogPage(dsl, query));
- }
-
- @Override
- public ConnectorPageResult connectorPage(ConnectorPageQuery query) {
- return dslContextFactory.transactionResult(dsl -> connectorListApiService.connectorListPage(dsl, query));
- }
-
- @Override
- public DataOfferDetailPageResult dataOfferDetailPage(DataOfferDetailPageQuery query) {
- return dslContextFactory.transactionResult(dsl -> dataOfferDetailApiService.dataOfferDetailPage(dsl, query));
- }
-
- @Override
- public ConnectorDetailPageResult connectorDetailPage(ConnectorDetailPageQuery query) {
- return dslContextFactory.transactionResult(dsl -> connectorDetailApiService.connectorDetailPage(dsl, query));
- }
-
- @Override
- public void addConnectors(List endpoints, String adminApiKey) {
- adminApiKeyValidator.validateAdminApiKey(adminApiKey);
- dslContextFactory.transaction(dsl -> connectorApiService.addConnectors(dsl, endpoints));
- }
-
- @Override
- public void addConnectorsWithMdsIds(ConnectorCreationRequest connectors, String adminApiKey) {
- adminApiKeyValidator.validateAdminApiKey(adminApiKey);
- dslContextFactory.transaction(dsl -> connectorApiService.addConnectorsWithMdsIds(dsl, connectors));
- }
-
- @Override
- public void deleteConnectors(List endpoints, String adminApiKey) {
- adminApiKeyValidator.validateAdminApiKey(adminApiKey);
- dslContextFactory.transaction(dsl -> connectorApiService.deleteConnectors(dsl, endpoints));
- }
-
- @Override
- public List getConnectorMetadata(List endpoints, String adminApiKey) {
- adminApiKeyValidator.validateAdminApiKey(adminApiKey);
- return dslContextFactory.transactionResult(dsl -> authorityPortalConnectorMetadataApiService.getMetadataByEndpoints(dsl, endpoints));
- }
-
- @Override
- public void setOrganizationMetadata(AuthorityPortalOrganizationMetadataRequest organizationMetadataRequest, String adminApiKey) {
- adminApiKeyValidator.validateAdminApiKey(adminApiKey);
- dslContextFactory.transaction(dsl -> authorityPortalOrganizationMetadataApiService.setOrganizationMetadata(dsl, organizationMetadataRequest.getOrganizations()));
- }
-
- @Override
- public List getConnectorDataOffers(List endpoints, String adminApiKey) {
- adminApiKeyValidator.validateAdminApiKey(adminApiKey);
- return dslContextFactory.transactionResult(dsl -> authorityPortalConnectorDataOffersApiService.getConnectorDataOffersByEndpoints(dsl, endpoints));
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/ConnectorQueries.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/ConnectorQueries.java
deleted file mode 100644
index 31fd44a7f..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/ConnectorQueries.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao;
-
-import de.sovity.edc.ext.brokerserver.dao.utils.PostgresqlUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import org.jooq.DSLContext;
-
-import java.time.Duration;
-import java.time.OffsetDateTime;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-
-public class ConnectorQueries {
-
- public ConnectorRecord findByEndpoint(DSLContext dsl, String endpoint) {
- var c = Tables.CONNECTOR;
- return dsl.selectFrom(c).where(c.ENDPOINT.eq(endpoint)).fetchOne();
- }
-
- public Set findConnectorsForScheduledRefresh(DSLContext dsl, ConnectorOnlineStatus onlineStatus) {
- var c = Tables.CONNECTOR;
- return dsl.select(c.ENDPOINT).from(c).where(c.ONLINE_STATUS.eq(onlineStatus)).fetchSet(c.ENDPOINT);
- }
-
- public Set findExistingConnectors(DSLContext dsl, Collection connectorEndpoints) {
- var c = Tables.CONNECTOR;
- return dsl.select(c.ENDPOINT).from(c)
- .where(PostgresqlUtils.in(c.ENDPOINT, connectorEndpoints))
- .fetchSet(c.ENDPOINT);
- }
-
- public List findAllConnectorsForKilling(DSLContext dsl, Duration deleteOfflineConnectorsAfter) {
- var c = Tables.CONNECTOR;
- return dsl.select(c.ENDPOINT).from(c)
- .where(c.LAST_SUCCESSFUL_REFRESH_AT.lt(OffsetDateTime.now().minus(deleteOfflineConnectorsAfter)))
- .fetch(c.ENDPOINT);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryAvailableFilterFetcher.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryAvailableFilterFetcher.java
deleted file mode 100644
index ee3fdd7aa..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryAvailableFilterFetcher.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogQueryFilter;
-import de.sovity.edc.ext.brokerserver.utils.CollectionUtils2;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Field;
-import org.jooq.JSON;
-import org.jooq.impl.DSL;
-import org.jooq.impl.SQLDataType;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@RequiredArgsConstructor
-public class CatalogQueryAvailableFilterFetcher {
- private final CatalogQueryFilterService catalogQueryFilterService;
-
- /**
- * Query available filter values.
- *
- * @param fields query fields
- * @param searchQuery search query
- * @param filters filters (values + filter clauses)
- * @return {@link Field} with field[iFilter][iValue]
- */
- public Field queryAvailableFilterValues(
- CatalogQueryFields fields,
- String searchQuery,
- List filters
- ) {
- List> resultFields = new ArrayList<>();
- for (int i = 0; i < filters.size(); i++) {
- // When querying a filter's values we apply all filters except for the current filter's values
- var currentFilter = filters.get(i);
- var otherFilters = CollectionUtils2.allElementsExceptForIndex(filters, i);
- var resultField = queryFilterValues(fields, currentFilter, searchQuery, otherFilters);
- resultFields.add(resultField);
- }
- return DSL.select(DSL.jsonArray(resultFields)).asField();
- }
-
- private Field queryFilterValues(
- CatalogQueryFields parentQueryFields,
- CatalogQueryFilter currentFilter,
- String searchQuery,
- List otherFilters
- ) {
- var fields = parentQueryFields.withSuffix("filter_" + currentFilter.name());
- var c = fields.getConnectorTable();
- var d = fields.getDataOfferTable();
-
- var value = currentFilter.valueQuery().getAttributeValueField(fields);
-
- return DSL.select(DSL.coalesce(DSL.arrayAggDistinct(value), DSL.array().cast(SQLDataType.VARCHAR.array())))
- .from(d)
- .leftJoin(c).on(c.ENDPOINT.eq(d.CONNECTOR_ENDPOINT))
- .where(catalogQueryFilterService.filterDbQuery(fields, searchQuery, otherFilters))
- .asField();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryContractOfferFetcher.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryContractOfferFetcher.java
deleted file mode 100644
index d2e370c5c..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryContractOfferFetcher.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model.ContractOfferRs;
-import de.sovity.edc.ext.brokerserver.dao.utils.MultisetUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.DataOffer;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Field;
-import org.jooq.impl.DSL;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class CatalogQueryContractOfferFetcher {
-
- /**
- * Query a data offer's contract offers.
- *
- * @param d Data offer table
- * @return {@link Field} of {@link ContractOfferRs}s
- */
- public Field> getContractOffers(DataOffer d) {
- var co = Tables.CONTRACT_OFFER;
-
- var query = DSL.select(
- co.CONTRACT_OFFER_ID,
- co.POLICY.cast(String.class).as("policyJson"),
- co.CREATED_AT,
- co.UPDATED_AT
- ).from(co).where(
- co.CONNECTOR_ENDPOINT.eq(d.CONNECTOR_ENDPOINT),
- co.ASSET_ID.eq(d.ASSET_ID)).orderBy(co.CREATED_AT.desc()
- );
-
- return MultisetUtils.multiset(query, ContractOfferRs.class);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryDataOfferFetcher.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryDataOfferFetcher.java
deleted file mode 100644
index 24cbb3282..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryDataOfferFetcher.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageSortingType;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogQueryFilter;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.DataOfferListEntryRs;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.PageQuery;
-import de.sovity.edc.ext.brokerserver.dao.utils.MultisetUtils;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Field;
-import org.jooq.Record;
-import org.jooq.SelectOnConditionStep;
-import org.jooq.SelectSelectStep;
-import org.jooq.impl.DSL;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class CatalogQueryDataOfferFetcher {
- private final CatalogQuerySortingService catalogQuerySortingService;
- private final CatalogQueryFilterService catalogQueryFilterService;
- private final CatalogQueryContractOfferFetcher catalogQueryContractOfferFetcher;
-
- /**
- * Query data offers
- *
- * @param fields query fields
- * @param searchQuery search query
- * @param filters filters (queries + filter clauses)
- * @param sorting sorting
- * @param pageQuery pagination
- * @return {@link Field} of {@link DataOfferListEntryRs}s
- */
- public Field> queryDataOffers(
- CatalogQueryFields fields,
- String searchQuery,
- List filters,
- CatalogPageSortingType sorting,
- PageQuery pageQuery
- ) {
- var c = fields.getConnectorTable();
- var d = fields.getDataOfferTable();
-
- var select = DSL.select(
- d.ASSET_ID.as("assetId"),
- d.ASSET_JSON_LD.cast(String.class).as("assetJsonLd"),
- d.CREATED_AT,
- d.UPDATED_AT,
- catalogQueryContractOfferFetcher.getContractOffers(d).as("contractOffers"),
- c.ENDPOINT.as("connectorEndpoint"),
- c.ONLINE_STATUS.as("connectorOnlineStatus"),
- c.PARTICIPANT_ID.as("connectorParticipantId"),
- fields.getOrganizationName().as("organizationName"),
- fields.getOfflineSinceOrLastUpdatedAt().as("connectorOfflineSinceOrLastUpdatedAt")
- );
-
- var query = from(select, fields)
- .where(catalogQueryFilterService.filterDbQuery(fields, searchQuery, filters))
- .orderBy(catalogQuerySortingService.getOrderBy(fields, sorting))
- .limit(pageQuery.offset(), pageQuery.limit());
-
- return MultisetUtils.multiset(query, DataOfferListEntryRs.class);
- }
-
- /**
- * Query number of data offers
- *
- * @param fields query fields
- * @param searchQuery search query
- * @param filters filters (queries + filter clauses)
- * @return {@link Field} with number of data offers
- */
- public Field queryNumDataOffers(CatalogQueryFields fields, String searchQuery, List filters) {
- var query = from(DSL.select(DSL.count()), fields)
- .where(catalogQueryFilterService.filterDbQuery(fields, searchQuery, filters));
- return DSL.field(query);
- }
-
- private SelectOnConditionStep from(SelectSelectStep select, CatalogQueryFields fields) {
- var c = fields.getConnectorTable();
- var d = fields.getDataOfferTable();
- return select.from(d).leftJoin(c).on(c.ENDPOINT.eq(d.CONNECTOR_ENDPOINT));
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFields.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFields.java
deleted file mode 100644
index 747f0707d..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFields.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.Connector;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.DataOffer;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.DataOfferViewCount;
-import de.sovity.edc.ext.brokerserver.services.config.DataSpaceConfig;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-import org.jooq.Field;
-import org.jooq.Table;
-import org.jooq.impl.DSL;
-
-import java.time.OffsetDateTime;
-
-import static org.jooq.impl.DSL.coalesce;
-
-/**
- * Tables and fields used in the catalog page query.
- *
- * Having this as a class makes access to computed fields (e.g. asset properties) easier.
- */
-@Getter
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
-public class CatalogQueryFields {
- Connector connectorTable;
- DataOffer dataOfferTable;
- DataOfferViewCount dataOfferViewCountTable;
-
- // Asset Properties from JSON to be used in sorting / filtering
- Field dataSpace;
-
- // This date should always be non-null
- // It's used in the UI to display the last relevant change date of a connector
- Field offlineSinceOrLastUpdatedAt;
-
- DataSpaceConfig dataSpaceConfig;
-
- public CatalogQueryFields(
- Connector connectorTable,
- DataOffer dataOfferTable,
- DataOfferViewCount dataOfferViewCountTable,
- DataSpaceConfig dataSpaceConfig
- ) {
- this.connectorTable = connectorTable;
- this.dataOfferTable = dataOfferTable;
- this.dataOfferViewCountTable = dataOfferViewCountTable;
- this.dataSpaceConfig = dataSpaceConfig;
- offlineSinceOrLastUpdatedAt = offlineSinceOrLastUpdatedAt(connectorTable);
-
- dataSpace = buildDataSpaceField(connectorTable, dataSpaceConfig);
- }
-
- private Field buildDataSpaceField(Connector connectorTable, DataSpaceConfig dataSpaceConfig) {
- var endpoint = connectorTable.ENDPOINT;
-
- var connectors = dataSpaceConfig.dataSpaceConnectors();
- if (connectors.isEmpty()) {
- return DSL.val(dataSpaceConfig.defaultDataSpace());
- }
-
- var first = connectors.get(0);
- var dspCase = DSL.case_(endpoint).when(first.endpoint(), first.dataSpaceName());
-
- for (var dsp : connectors.subList(1, connectors.size())) {
- dspCase = dspCase.when(dsp.endpoint(), dsp.dataSpaceName());
- }
-
- return dspCase.else_(DSL.val(dataSpaceConfig.defaultDataSpace()));
- }
-
- public CatalogQueryFields withSuffix(String additionalSuffix) {
- return new CatalogQueryFields(
- connectorTable.as(withSuffix(connectorTable, additionalSuffix)),
- dataOfferTable.as(withSuffix(dataOfferTable, additionalSuffix)),
- dataOfferViewCountTable.as(withSuffix(dataOfferViewCountTable, additionalSuffix)),
- dataSpaceConfig
- );
- }
-
- private String withSuffix(Table table, String additionalSuffix) {
- return "%s_%s".formatted(table.getName(), additionalSuffix);
- }
-
- public Field getViewCount() {
- var subquery = DSL.select(DSL.count())
- .from(dataOfferViewCountTable)
- .where(dataOfferViewCountTable.ASSET_ID.eq(dataOfferTable.ASSET_ID)
- .and(dataOfferViewCountTable.CONNECTOR_ENDPOINT.eq(connectorTable.ENDPOINT)));
-
- return subquery.asField();
- }
-
- public Field getOrganizationName() {
- return organizationName(connectorTable.MDS_ID);
- }
-
- public static Field offlineSinceOrLastUpdatedAt(Connector connectorTable) {
- return DSL.coalesce(
- connectorTable.LAST_SUCCESSFUL_REFRESH_AT,
- connectorTable.CREATED_AT
- );
- }
-
- public static Field organizationName(Field mdsId) {
- var om = Tables.ORGANIZATION_METADATA;
- var organizationName = DSL.select(om.NAME)
- .from(om)
- .where(om.MDS_ID.eq(mdsId))
- .asField()
- .cast(String.class);
- return coalesce(organizationName, "Unknown");
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFilterService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFilterService.java
deleted file mode 100644
index b92af8920..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryFilterService.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogQueryFilter;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.Connector;
-import de.sovity.edc.ext.brokerserver.services.api.filtering.CatalogSearchService;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.Condition;
-import org.jooq.impl.DSL;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-
-@RequiredArgsConstructor
-public class CatalogQueryFilterService {
- private final BrokerServerSettings brokerServerSettings;
- private final CatalogSearchService catalogSearchService;
-
- public Condition filterDbQuery(CatalogQueryFields fields, String searchQuery, List filters) {
- var conditions = new ArrayList();
- conditions.add(catalogSearchService.filterBySearch(fields, searchQuery));
- conditions.add(onlyOnlineOrRecentlyOfflineConnectors(fields.getConnectorTable()));
- conditions.addAll(filters.stream().map(CatalogQueryFilter::queryFilterClauseOrNull)
- .filter(Objects::nonNull).map(it -> it.filterDataOffers(fields)).toList());
- return DSL.and(conditions);
- }
-
- @NotNull
- private Condition onlyOnlineOrRecentlyOfflineConnectors(Connector c) {
- var maxOfflineDuration = brokerServerSettings.getHideOfflineDataOffersAfter();
-
- Condition maxOfflineDurationNotExceeded;
- if (maxOfflineDuration == null) {
- maxOfflineDurationNotExceeded = DSL.trueCondition();
- } else {
- maxOfflineDurationNotExceeded = c.LAST_SUCCESSFUL_REFRESH_AT.greaterThan(OffsetDateTime.now().minus(maxOfflineDuration));
- }
-
- return DSL.or(
- c.ONLINE_STATUS.eq(ConnectorOnlineStatus.ONLINE),
- maxOfflineDurationNotExceeded
- );
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryService.java
deleted file mode 100644
index 04a585e35..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQueryService.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageSortingType;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogPageRs;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogQueryFilter;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.PageQuery;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class CatalogQueryService {
- private final CatalogQueryDataOfferFetcher catalogQueryDataOfferFetcher;
- private final CatalogQueryAvailableFilterFetcher catalogQueryAvailableFilterFetcher;
- private final BrokerServerSettings brokerServerSettings;
-
- /**
- * Query all data required for the catalog page
- *
- * @param dsl transaction
- * @param searchQuery search query
- * @param filters filters (queries + filter clauses)
- * @param sorting sorting
- * @param pageQuery pagination
- * @return {@link CatalogPageRs}
- */
- public CatalogPageRs queryCatalogPage(
- DSLContext dsl,
- String searchQuery,
- List filters,
- CatalogPageSortingType sorting,
- PageQuery pageQuery
- ) {
- var fields = new CatalogQueryFields(
- Tables.CONNECTOR,
- Tables.DATA_OFFER,
- Tables.DATA_OFFER_VIEW_COUNT,
- brokerServerSettings.getDataSpaceConfig()
- );
-
- var availableFilterValues = catalogQueryAvailableFilterFetcher
- .queryAvailableFilterValues(fields, searchQuery, filters);
-
- var dataOffers = catalogQueryDataOfferFetcher.queryDataOffers(fields, searchQuery, filters, sorting, pageQuery);
-
- var numTotalDataOffers = catalogQueryDataOfferFetcher.queryNumDataOffers(fields, searchQuery, filters);
-
- return dsl.select(
- dataOffers.as("dataOffers"),
- availableFilterValues.as("availableFilterValues"),
- numTotalDataOffers.as("numTotalDataOffers")
- ).fetchOneInto(CatalogPageRs.class);
- }
-
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQuerySortingService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQuerySortingService.java
deleted file mode 100644
index c06aaecb1..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/CatalogQuerySortingService.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog;
-
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageSortingType;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.OrderField;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class CatalogQuerySortingService {
- @NotNull
- public List> getOrderBy(CatalogQueryFields fields, @NonNull CatalogPageSortingType sorting) {
- List> orderBy;
- if (sorting == CatalogPageSortingType.TITLE) {
- orderBy = List.of(
- fields.getDataOfferTable().ASSET_TITLE.asc(),
- fields.getConnectorTable().ENDPOINT.asc()
- );
- } else if (sorting == CatalogPageSortingType.MOST_RECENT) {
- orderBy = List.of(
- fields.getDataOfferTable().CREATED_AT.desc(),
- fields.getConnectorTable().ENDPOINT.asc()
- );
- } else if (sorting == CatalogPageSortingType.ORIGINATOR) {
- orderBy = List.of(
- fields.getConnectorTable().ENDPOINT.asc(),
- fields.getDataOfferTable().ASSET_TITLE.asc()
- );
- } else if (sorting == CatalogPageSortingType.VIEW_COUNT) {
- orderBy = List.of(
- fields.getViewCount().desc(),
- fields.getConnectorTable().ENDPOINT.asc()
- );
- } else {
- throw new IllegalArgumentException("Unknown %s: %s".formatted(CatalogPageSortingType.class.getName(), sorting));
- }
- return orderBy;
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/AvailableFilterValuesQuery.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/AvailableFilterValuesQuery.java
deleted file mode 100644
index 9dabec513..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/AvailableFilterValuesQuery.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog.models;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import org.jooq.Field;
-
-@FunctionalInterface
-public interface AvailableFilterValuesQuery {
-
- /**
- * Gets the values for a given filter attribute from a list of data offers.
- *
- * @param fields a
- * @return field / multiset field that will contain the available values
- */
- Field getAttributeValueField(CatalogQueryFields fields);
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogPageRs.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogPageRs.java
deleted file mode 100644
index 88a199df1..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogPageRs.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog.models;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.util.List;
-
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class CatalogPageRs {
- String availableFilterValues;
- List dataOffers;
- int numTotalDataOffers;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogQueryFilter.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogQueryFilter.java
deleted file mode 100644
index 9dcccd038..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogQueryFilter.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog.models;
-
-import lombok.NonNull;
-
-public record CatalogQueryFilter(
- @NonNull String name,
- @NonNull AvailableFilterValuesQuery valueQuery,
- CatalogQuerySelectedFilterQuery queryFilterClauseOrNull
-) {
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogQuerySelectedFilterQuery.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogQuerySelectedFilterQuery.java
deleted file mode 100644
index a3b2bcfa3..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/CatalogQuerySelectedFilterQuery.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog.models;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import org.jooq.Condition;
-
-@FunctionalInterface
-public interface CatalogQuerySelectedFilterQuery {
-
- /**
- * Adds a filter to a Catalog Query.
- *
- * @param fields fields and tables available in the catalog query
- * @return {@link Condition}
- */
- Condition filterDataOffers(CatalogQueryFields fields);
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/DataOfferListEntryRs.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/DataOfferListEntryRs.java
deleted file mode 100644
index 1c97cf430..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/DataOfferListEntryRs.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog.models;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model.ContractOfferRs;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DataOfferListEntryRs {
- String assetId;
- String assetJsonLd;
- OffsetDateTime createdAt;
- OffsetDateTime updatedAt;
- List contractOffers;
- String connectorEndpoint;
- ConnectorOnlineStatus connectorOnlineStatus;
- String connectorParticipantId;
- String organizationName;
- OffsetDateTime connectorOfflineSinceOrLastUpdatedAt;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/PageQuery.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/PageQuery.java
deleted file mode 100644
index c5cd659bb..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/catalog/models/PageQuery.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.catalog.models;
-
-public record PageQuery(int offset, int limit) {
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorDetailQueryService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorDetailQueryService.java
deleted file mode 100644
index f4acc2589..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorDetailQueryService.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.connector;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.model.ConnectorDetailsRs;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementErrorStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.Connector;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
-import org.jooq.Field;
-import org.jooq.impl.DSL;
-
-import java.math.BigDecimal;
-
-public class ConnectorDetailQueryService {
- public ConnectorDetailsRs queryConnectorDetailPage(DSLContext dsl, String connectorEndpoint) {
- var c = Tables.CONNECTOR;
-
- return dsl.select(
- c.ENDPOINT.as("endpoint"),
- c.PARTICIPANT_ID.as("participantId"),
- CatalogQueryFields.organizationName(c.MDS_ID).as("organizationName"),
- c.CREATED_AT.as("createdAt"),
- c.LAST_SUCCESSFUL_REFRESH_AT.as("lastSuccessfulRefreshAt"),
- c.LAST_REFRESH_ATTEMPT_AT.as("lastRefreshAttemptAt"),
- c.ONLINE_STATUS.as("onlineStatus"),
- dataOfferCount(c.ENDPOINT).as("numDataOffers"),
- getAvgSuccessfulCrawlTimeInMs(c).as("connectorCrawlingTimeAvg"))
- .from(c)
- .where(c.ENDPOINT.eq(connectorEndpoint))
- .groupBy(c.ENDPOINT)
- .fetchOneInto(ConnectorDetailsRs.class);
- }
-
- @NotNull
- private Field getAvgSuccessfulCrawlTimeInMs(Connector c) {
- var betm = Tables.BROKER_EXECUTION_TIME_MEASUREMENT;
- return DSL.select(DSL.avg(betm.DURATION_IN_MS))
- .from(betm)
- .where(betm.CONNECTOR_ENDPOINT.eq(c.ENDPOINT), betm.ERROR_STATUS.eq(MeasurementErrorStatus.OK))
- .asField();
- }
-
- private Field dataOfferCount(Field endpoint) {
- var d = Tables.DATA_OFFER;
- return DSL.select(DSL.count()).from(d).where(d.CONNECTOR_ENDPOINT.eq(endpoint)).asField();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorListQueryService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorListQueryService.java
deleted file mode 100644
index 524631b69..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/ConnectorListQueryService.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.connector;
-
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageSortingType;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.model.ConnectorListEntryRs;
-import de.sovity.edc.ext.brokerserver.dao.utils.SearchUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.Connector;
-import lombok.NonNull;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
-import org.jooq.Field;
-import org.jooq.OrderField;
-import org.jooq.impl.DSL;
-
-import java.util.List;
-
-public class ConnectorListQueryService {
- public List queryConnectorPage(DSLContext dsl, String searchQuery, ConnectorPageSortingType sorting) {
- var c = Tables.CONNECTOR;
- var filterBySearchQuery = SearchUtils.simpleSearch(searchQuery, List.of(
- c.ENDPOINT,
- c.PARTICIPANT_ID,
- CatalogQueryFields.organizationName(c.MDS_ID)
- ));
-
- return dsl.select(
- c.ENDPOINT.as("endpoint"),
- c.PARTICIPANT_ID.as("participantId"),
- CatalogQueryFields.organizationName(c.MDS_ID).as("organizationName"),
- c.CREATED_AT.as("createdAt"),
- c.LAST_SUCCESSFUL_REFRESH_AT.as("lastSuccessfulRefreshAt"),
- c.LAST_REFRESH_ATTEMPT_AT.as("lastRefreshAttemptAt"),
- c.ONLINE_STATUS.as("onlineStatus"),
- dataOfferCount(c.ENDPOINT).as("numDataOffers")
- )
- .from(c)
- .where(filterBySearchQuery)
- .orderBy(sortConnectorPage(c, sorting))
- .fetchInto(ConnectorListEntryRs.class);
- }
-
- @NotNull
- private List> sortConnectorPage(Connector c, @NonNull ConnectorPageSortingType sorting) {
- var alphabetically = c.ENDPOINT.asc();
- var recentFirst = c.CREATED_AT.desc();
- var onlineStatus = DSL.case_(c.ONLINE_STATUS)
- .when(ConnectorOnlineStatus.ONLINE, 1)
- .when(ConnectorOnlineStatus.OFFLINE, 2)
- .else_(3)
- .asc();
-
- if (sorting == ConnectorPageSortingType.ONLINE_STATUS) {
- return List.of(onlineStatus, alphabetically);
- } else if (sorting == ConnectorPageSortingType.TITLE) {
- return List.of(alphabetically, recentFirst);
- } else if (sorting == ConnectorPageSortingType.MOST_RECENT) {
- return List.of(recentFirst, alphabetically);
- }
-
- throw new IllegalArgumentException("Unhandled sorting type: " + sorting);
- }
-
- private Field dataOfferCount(Field endpoint) {
- var d = Tables.DATA_OFFER;
- return DSL.select(DSL.count()).from(d).where(d.CONNECTOR_ENDPOINT.eq(endpoint)).asField();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorDetailsRs.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorDetailsRs.java
deleted file mode 100644
index 071d8e5ce..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorDetailsRs.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.connector.model;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.time.OffsetDateTime;
-
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ConnectorDetailsRs {
- String endpoint;
- String participantId;
- String organizationName;
- OffsetDateTime createdAt;
- OffsetDateTime lastSuccessfulRefreshAt;
- OffsetDateTime lastRefreshAttemptAt;
- ConnectorOnlineStatus onlineStatus;
- Integer numDataOffers;
- Long connectorCrawlingTimeAvg;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorListEntryRs.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorListEntryRs.java
deleted file mode 100644
index 96ba710f6..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/connector/model/ConnectorListEntryRs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.connector.model;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.time.OffsetDateTime;
-
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ConnectorListEntryRs {
- String endpoint;
- String participantId;
- String organizationName;
- OffsetDateTime createdAt;
- OffsetDateTime lastSuccessfulRefreshAt;
- OffsetDateTime lastRefreshAttemptAt;
- ConnectorOnlineStatus onlineStatus;
- Integer numDataOffers;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/DataOfferDetailPageQueryService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/DataOfferDetailPageQueryService.java
deleted file mode 100644
index f597ac124..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/DataOfferDetailPageQueryService.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.dataoffer;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryContractOfferFetcher;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model.DataOfferDetailRs;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-@RequiredArgsConstructor
-public class DataOfferDetailPageQueryService {
- private final CatalogQueryContractOfferFetcher catalogQueryContractOfferFetcher;
- private final BrokerServerSettings brokerServerSettings;
-
- public DataOfferDetailRs queryDataOfferDetailsPage(DSLContext dsl, String assetId, String endpoint) {
- // We are re-using the catalog page query stuff as long as we can get away with it
- var fields = new CatalogQueryFields(
- Tables.CONNECTOR,
- Tables.DATA_OFFER,
- Tables.DATA_OFFER_VIEW_COUNT,
- brokerServerSettings.getDataSpaceConfig()
- );
-
- var d = fields.getDataOfferTable();
- var c = fields.getConnectorTable();
-
- return dsl.select(
- d.ASSET_ID,
- d.ASSET_JSON_LD.cast(String.class).as("assetJsonLd"),
- d.CREATED_AT,
- d.UPDATED_AT,
- catalogQueryContractOfferFetcher.getContractOffers(fields.getDataOfferTable()).as("contractOffers"),
- fields.getOfflineSinceOrLastUpdatedAt().as("connectorOfflineSinceOrLastUpdatedAt"),
- c.ENDPOINT.as("connectorEndpoint"),
- c.ONLINE_STATUS.as("connectorOnlineStatus"),
- c.PARTICIPANT_ID.as("connectorParticipantId"),
- fields.getOrganizationName().as("organizationName"),
- fields.getViewCount().as("viewCount"))
- .from(d)
- .leftJoin(c).on(c.ENDPOINT.eq(d.CONNECTOR_ENDPOINT))
- .where(d.ASSET_ID.eq(assetId).and(d.CONNECTOR_ENDPOINT.eq(endpoint)))
- .fetchOneInto(DataOfferDetailRs.class);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/ViewCountLogger.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/ViewCountLogger.java
deleted file mode 100644
index 377abeafa..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/ViewCountLogger.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.dataoffer;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import org.jooq.DSLContext;
-
-import java.time.OffsetDateTime;
-
-public class ViewCountLogger {
- public void increaseDataOfferViewCount(DSLContext dsl, String assetId, String endpoint) {
- var v = Tables.DATA_OFFER_VIEW_COUNT;
- dsl.insertInto(v, v.ASSET_ID, v.CONNECTOR_ENDPOINT, v.DATE).values(assetId, endpoint, OffsetDateTime.now()).execute();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/ContractOfferRs.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/ContractOfferRs.java
deleted file mode 100644
index f10d1dd1e..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/ContractOfferRs.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.time.OffsetDateTime;
-
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class ContractOfferRs {
- String contractOfferId;
- String policyJson;
- OffsetDateTime createdAt;
- OffsetDateTime updatedAt;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/DataOfferDetailRs.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/DataOfferDetailRs.java
deleted file mode 100644
index 0b575849c..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/pages/dataoffer/model/DataOfferDetailRs.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DataOfferDetailRs {
- String assetId;
- String assetJsonLd;
- OffsetDateTime createdAt;
- OffsetDateTime updatedAt;
- List contractOffers;
- String connectorEndpoint;
- ConnectorOnlineStatus connectorOnlineStatus;
- String connectorParticipantId;
- String organizationName;
- OffsetDateTime connectorOfflineSinceOrLastUpdatedAt;
- Integer viewCount;
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/JsonDeserializationUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/JsonDeserializationUtils.java
deleted file mode 100644
index 1c3f4e2f9..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/JsonDeserializationUtils.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.utils;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import lombok.SneakyThrows;
-
-import java.util.List;
-
-/**
- * Some things are easier to fetch as json into a string with JooQ.
- * In that case we need to deserialize that string into an object of our choice afterwards.
- */
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class JsonDeserializationUtils {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final TypeReference>> TYPE_STRING_LIST_2 = new TypeReference<>() {
- };
-
- @SneakyThrows
- public static List> read2dStringList(String json) {
- return OBJECT_MAPPER.readValue(json, TYPE_STRING_LIST_2);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/LikeUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/LikeUtils.java
deleted file mode 100644
index c3823b04a..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/LikeUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.utils;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import org.apache.commons.lang3.StringUtils;
-import org.jooq.Condition;
-import org.jooq.Field;
-import org.jooq.impl.DSL;
-
-/**
- * Utilities for dealing with PostgreSQL Like Operation values
- */
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class LikeUtils {
-
- /**
- * Create LIKE condition value for "field contains word".
- *
- * @param field field
- * @param lowercaseWord word
- * @return "%escapedWord%"
- */
- public static Condition contains(Field field, String lowercaseWord) {
- if (StringUtils.isBlank(lowercaseWord)) {
- return DSL.trueCondition();
- }
-
- return field.likeIgnoreCase("%" + escape(lowercaseWord) + "%");
- }
-
-
- /**
- * Escapes "\", "%", "_" in given string for a LIKE operation
- *
- * @param string unescaped string
- * @return escaped string
- */
- public static String escape(String string) {
- return string.replace("\\", "\\\\")
- .replace("%", "\\%")
- .replace("_", "\\_");
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/MultisetUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/MultisetUtils.java
deleted file mode 100644
index 9cdcdd84d..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/MultisetUtils.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.utils;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import org.jooq.Field;
-import org.jooq.TableLike;
-import org.jooq.impl.DSL;
-
-import java.util.List;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class MultisetUtils {
- public static Field> multiset(TableLike table, Class type) {
- return DSL.multiset(table).convertFrom(it -> it.into(type));
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/SearchUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/SearchUtils.java
deleted file mode 100644
index e8a6ef310..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/dao/utils/SearchUtils.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.utils;
-
-import de.sovity.edc.ext.brokerserver.utils.StringUtils2;
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import org.jooq.Condition;
-import org.jooq.Field;
-import org.jooq.impl.DSL;
-
-import java.util.List;
-
-/**
- * DB Search Queries
- */
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class SearchUtils {
-
- /**
- * Simple search
- *
- * All search query words must be contained in at least one search target.
- *
- * @param searchQuery search query
- * @param searchTargets target fields
- * @return JOOQ Condition
- */
- public static Condition simpleSearch(String searchQuery, List> searchTargets) {
- var words = StringUtils2.lowercaseWords(searchQuery);
- return DSL.and(words.stream()
- .map(word -> anySearchTargetContains(searchTargets, word))
- .toList());
- }
-
- private static Condition anySearchTargetContains(List> searchTargets, String word) {
- return DSL.or(searchTargets.stream().map(field -> LikeUtils.contains(field, word)).toList());
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorCleaner.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorCleaner.java
deleted file mode 100644
index b21cc2787..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorCleaner.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services;
-
-import de.sovity.edc.ext.brokerserver.dao.utils.PostgresqlUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import org.jooq.DSLContext;
-
-import java.util.Collection;
-
-public class ConnectorCleaner {
- public void removeDataForDeadConnectors(DSLContext dsl, Collection endpoints) {
- var doco = Tables.CONTRACT_OFFER;
- var dof = Tables.DATA_OFFER;
- dsl.deleteFrom(doco).where(PostgresqlUtils.in(doco.CONNECTOR_ENDPOINT, endpoints)).execute();
- dsl.deleteFrom(dof).where(PostgresqlUtils.in(dof.CONNECTOR_ENDPOINT, endpoints)).execute();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorCreator.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorCreator.java
deleted file mode 100644
index 51b5f461e..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorCreator.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services;
-
-import de.sovity.edc.ext.brokerserver.dao.ConnectorQueries;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import de.sovity.edc.ext.brokerserver.utils.CollectionUtils2;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
-
-import java.time.OffsetDateTime;
-import java.util.Collection;
-import java.util.List;
-
-@RequiredArgsConstructor
-public class ConnectorCreator {
- private final ConnectorQueries connectorQueries;
-
- public void addConnector(DSLContext dsl, String connectorEndpoint) {
- addConnectors(dsl, List.of(connectorEndpoint));
- }
-
- public void addConnectors(DSLContext dsl, Collection connectorEndpoints) {
- // Don't create connectors that already exist
- var existingConnectors = connectorQueries.findExistingConnectors(dsl, connectorEndpoints);
- var newConnectors = CollectionUtils2.difference(connectorEndpoints, existingConnectors);
-
- var connectorRecords = newConnectors.stream()
- .map(String::trim)
- .map(this::newConnectorRow)
- .toList();
-
- if (!connectorRecords.isEmpty()) {
- dsl.batchStore(connectorRecords).execute();
- }
- }
-
- @NotNull
- private ConnectorRecord newConnectorRow(String endpoint) {
- var connector = new ConnectorRecord();
- connector.setEndpoint(endpoint);
- connector.setParticipantId("");
- connector.setCreatedAt(OffsetDateTime.now());
- connector.setOnlineStatus(ConnectorOnlineStatus.OFFLINE);
- connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
- connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
- return connector;
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorKiller.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorKiller.java
deleted file mode 100644
index f44353fdb..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/ConnectorKiller.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services;
-
-import de.sovity.edc.ext.brokerserver.dao.utils.PostgresqlUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import org.jooq.DSLContext;
-
-import java.util.Collection;
-
-public class ConnectorKiller {
- public void killConnectors(DSLContext dsl, Collection endpoints) {
- var c = Tables.CONNECTOR;
- dsl.update(c).set(c.ONLINE_STATUS, ConnectorOnlineStatus.DEAD).where(PostgresqlUtils.in(c.ENDPOINT, endpoints)).execute();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/KnownConnectorsInitializer.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/KnownConnectorsInitializer.java
deleted file mode 100644
index 117bdfbe8..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/KnownConnectorsInitializer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services;
-
-import de.sovity.edc.ext.brokerserver.BrokerServerExtension;
-import de.sovity.edc.ext.brokerserver.services.queue.ConnectorQueue;
-import de.sovity.edc.ext.brokerserver.services.queue.ConnectorRefreshPriority;
-import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.edc.spi.system.configuration.Config;
-import org.jooq.DSLContext;
-
-import java.util.Arrays;
-import java.util.List;
-
-@RequiredArgsConstructor
-public class KnownConnectorsInitializer {
- private final Config config;
- private final ConnectorQueue connectorQueue;
- private final ConnectorCreator connectorCreator;
-
- public void addKnownConnectorsOnStartup(DSLContext dsl) {
- var connectorEndpoints = getKnownConnectorsConfigValue();
- connectorCreator.addConnectors(dsl, connectorEndpoints);
- connectorQueue.addAll(connectorEndpoints, ConnectorRefreshPriority.ADDED_ON_STARTUP);
- }
-
- private List getKnownConnectorsConfigValue() {
- var knownConnectorsString = config.getString(BrokerServerExtension.KNOWN_CONNECTORS, "");
- return Arrays.stream(knownConnectorsString.split(",")).map(String::trim).filter(StringUtils::isNotBlank).distinct().toList();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/OfflineConnectorKiller.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/OfflineConnectorKiller.java
deleted file mode 100644
index 8b46d9c81..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/OfflineConnectorKiller.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services;
-
-import de.sovity.edc.ext.brokerserver.dao.ConnectorQueries;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-@RequiredArgsConstructor
-public class OfflineConnectorKiller {
- private final BrokerServerSettings brokerServerSettings;
- private final ConnectorQueries connectorQueries;
- private final BrokerEventLogger brokerEventLogger;
- private final ConnectorKiller connectorKiller;
- private final ConnectorCleaner connectorClearer;
-
- public void killIfOfflineTooLong(DSLContext dsl) {
- var killOfflineConnectorsAfter = brokerServerSettings.getKillOfflineConnectorsAfter();
- var toKill = connectorQueries.findAllConnectorsForKilling(dsl, killOfflineConnectorsAfter);
-
- connectorClearer.removeDataForDeadConnectors(dsl, toKill);
- connectorKiller.killConnectors(dsl, toKill);
-
- brokerEventLogger.addKilledDueToOfflineTooLongMessages(dsl, toKill);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AssetPropertyParser.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AssetPropertyParser.java
deleted file mode 100644
index 2cb4fdf2c..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AssetPropertyParser.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-
-import java.util.Map;
-
-@RequiredArgsConstructor
-public class AssetPropertyParser {
- private final ObjectMapper objectMapper;
-
- private final TypeReference> typeToken = new TypeReference<>() {
- };
-
- @SneakyThrows
- public Map parsePropertiesFromJsonString(String assetPropertiesJson) {
- return objectMapper.readValue(assetPropertiesJson, typeToken);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorDataOfferApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorDataOfferApiService.java
deleted file mode 100644
index baf08bc02..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorDataOfferApiService.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorDataOfferInfo;
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorDataOfferDetails;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class AuthorityPortalConnectorDataOfferApiService {
- private final AuthorityPortalConnectorQueryService authorityPortalConnectorQueryService;
- private final ConnectorOnlineStatusMapper connectorOnlineStatusMapper;
-
- public List getConnectorDataOffersByEndpoints(DSLContext dsl, List endpoints) {
- return authorityPortalConnectorQueryService.getConnectorsDataOffers(dsl, endpoints).stream()
- .map(it -> new AuthorityPortalConnectorDataOfferInfo(
- it.getConnectorEndpoint(),
- it.getParticipantId(),
- connectorOnlineStatusMapper.getOnlineStatus(it.getOnlineStatus()),
- it.getOfflineSinceOrLastUpdatedAt(),
- it.getDataOffers().stream().map(dataOffer -> new AuthorityPortalConnectorDataOfferDetails(dataOffer.getDataOfferId(), dataOffer.getDataOfferName())).toList()
- ))
- .toList();
- }
-
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorMetadataApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorMetadataApiService.java
deleted file mode 100644
index 60d103bef..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorMetadataApiService.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalConnectorInfo;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class AuthorityPortalConnectorMetadataApiService {
- private final AuthorityPortalConnectorQueryService authorityPortalConnectorQueryService;
- private final ConnectorOnlineStatusMapper connectorOnlineStatusMapper;
-
- public List getMetadataByEndpoints(DSLContext dsl, List endpoints) {
-
- return authorityPortalConnectorQueryService.getConnectorMetadata(dsl, endpoints).stream()
- .map(it -> new AuthorityPortalConnectorInfo(
- it.getConnectorEndpoint(),
- it.getParticipantId(),
- it.getDataOfferCount(),
- connectorOnlineStatusMapper.getOnlineStatus(it.getOnlineStatus()),
- it.getOfflineSinceOrLastUpdatedAt()
- ))
- .toList();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorQueryService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorQueryService.java
deleted file mode 100644
index bb432ae5c..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorQueryService.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.utils.PostgresqlUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import lombok.AccessLevel;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
-import org.jooq.Field;
-import org.jooq.impl.DSL;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-
-import static org.jooq.impl.DSL.coalesce;
-import static org.jooq.impl.DSL.count;
-import static org.jooq.impl.DSL.select;
-
-@RequiredArgsConstructor
-public class AuthorityPortalConnectorQueryService {
-
- @Data
- @FieldDefaults(level = AccessLevel.PRIVATE)
- public static class ConnectorMetadataRs {
- String connectorEndpoint;
- String participantId;
- ConnectorOnlineStatus onlineStatus;
- OffsetDateTime offlineSinceOrLastUpdatedAt;
- Integer dataOfferCount;
- }
-
- @Data
- @FieldDefaults(level = AccessLevel.PRIVATE)
- public static class ConnectorDetailsRs {
- String connectorEndpoint;
- String participantId;
- ConnectorOnlineStatus onlineStatus;
- OffsetDateTime offlineSinceOrLastUpdatedAt;
- List dataOffers;
- }
-
- @Data
- @FieldDefaults(level = AccessLevel.PRIVATE)
- public static class DataOfferRs {
- String dataOfferId;
- String dataOfferName;
- }
-
- @NotNull
- public List getConnectorMetadata(DSLContext dsl, List endpoints) {
- var c = Tables.CONNECTOR;
-
- return dsl.select(
- c.ENDPOINT.as("connectorEndpoint"),
- c.PARTICIPANT_ID.as("participantId"),
- c.ONLINE_STATUS.as("onlineStatus"),
- CatalogQueryFields.offlineSinceOrLastUpdatedAt(c).as("offlineSinceOrLastUpdatedAt"),
- getDataOfferCount(c.ENDPOINT).as("dataOfferCount")
- )
- .from(c)
- .where(PostgresqlUtils.in(c.ENDPOINT, endpoints))
- .fetchInto(ConnectorMetadataRs.class);
- }
-
- @NotNull
- public Field getDataOfferCount(Field connectorEndpoint) {
- var d = Tables.DATA_OFFER;
-
- return select(coalesce(count().cast(Integer.class), DSL.value(0)))
- .from(d)
- .where(d.CONNECTOR_ENDPOINT.eq(connectorEndpoint))
- .asField();
- }
-
- @NotNull
- public List getDataOffers(DSLContext dsl, String connectorEndpoint) {
- var d = Tables.DATA_OFFER;
-
- return dsl.select(
- d.ASSET_TITLE.as("dataOfferName"),
- d.ASSET_ID.as("dataOfferId")
- )
- .from(d)
- .where(d.CONNECTOR_ENDPOINT.eq(connectorEndpoint))
- .fetchInto(DataOfferRs.class);
- }
-
-
- @NotNull
- public List getConnectorsDataOffers(DSLContext dsl, List endpoints) {
- var c = Tables.CONNECTOR;
-
- var connectors = dsl.select(
- c.ENDPOINT.as("connectorEndpoint"),
- c.PARTICIPANT_ID.as("participantId"),
- c.ONLINE_STATUS.as("onlineStatus"),
- CatalogQueryFields.offlineSinceOrLastUpdatedAt(c).as("offlineSinceOrLastUpdatedAt")
- )
- .from(c)
- .where(PostgresqlUtils.in(c.ENDPOINT, endpoints))
- .fetchInto(ConnectorDetailsRs.class);
- connectors.forEach(connector -> connector.dataOffers = getDataOffers(dsl, connector.connectorEndpoint));
- return connectors;
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiService.java
deleted file mode 100644
index 84086463b..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiService.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.AuthorityPortalOrganizationMetadata;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.OrganizationMetadataRecord;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class AuthorityPortalOrganizationMetadataApiService {
-
- public void setOrganizationMetadata(DSLContext dsl, List organizationMetadata) {
- var records = organizationMetadata.stream().map(this::buildRecord).toList();
-
- dsl.deleteFrom(Tables.ORGANIZATION_METADATA).execute();
- dsl.batchInsert(records).execute();
- }
-
- @NotNull
- private OrganizationMetadataRecord buildRecord(AuthorityPortalOrganizationMetadata it) {
- var record = new OrganizationMetadataRecord();
- record.setMdsId(it.getMdsId());
- record.setName(it.getName());
-
- return record;
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiService.java
deleted file mode 100644
index d83d1cef5..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiService.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.CatalogContractOffer;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogDataOffer;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageResult;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageSortingItem;
-import de.sovity.edc.ext.brokerserver.api.model.CatalogPageSortingType;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.DataOfferListEntryRs;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model.ContractOfferRs;
-import de.sovity.edc.ext.brokerserver.services.api.filtering.CatalogFilterService;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Stream;
-
-@RequiredArgsConstructor
-public class CatalogApiService {
- private final PaginationMetadataUtils paginationMetadataUtils;
- private final CatalogQueryService catalogQueryService;
- private final DataOfferMappingUtils dataOfferMappingUtils;
- private final CatalogFilterService catalogFilterService;
- private final BrokerServerSettings brokerServerSettings;
-
- public CatalogPageResult catalogPage(DSLContext dsl, CatalogPageQuery query) {
- Objects.requireNonNull(query, "query must not be null");
-
-
- var filters = catalogFilterService.getCatalogQueryFilters(query.getFilter());
-
- var pageQuery = paginationMetadataUtils.getPageQuery(
- query.getPageOneBased(),
- brokerServerSettings.getCatalogPagePageSize()
- );
-
- var availableSortings = buildAvailableSortings();
- var sorting = query.getSorting();
- if (sorting == null) {
- sorting = availableSortings.get(0).getSorting();
- }
-
- // execute db query
- var catalogPageRs = catalogQueryService.queryCatalogPage(
- dsl,
- query.getSearchQuery(),
- filters,
- sorting,
- pageQuery
- );
-
- var paginationMetadata = paginationMetadataUtils.buildPaginationMetadata(
- query.getPageOneBased(),
- brokerServerSettings.getCatalogPagePageSize(),
- catalogPageRs.getDataOffers().size(),
- catalogPageRs.getNumTotalDataOffers()
- );
-
- var result = new CatalogPageResult();
- result.setAvailableSortings(availableSortings);
- result.setPaginationMetadata(paginationMetadata);
- result.setAvailableFilters(catalogFilterService.buildAvailableFilters(catalogPageRs.getAvailableFilterValues()));
- result.setDataOffers(buildCatalogDataOffers(catalogPageRs.getDataOffers()));
- return result;
- }
-
- private List buildCatalogDataOffers(List dataOfferRs) {
- return dataOfferRs.stream()
- .map(this::buildCatalogDataOffer)
- .toList();
- }
-
- private CatalogDataOffer buildCatalogDataOffer(DataOfferListEntryRs dataOfferRs) {
- var asset = dataOfferMappingUtils.buildUiAsset(
- dataOfferRs.getAssetJsonLd(),
- dataOfferRs.getConnectorEndpoint(),
- dataOfferRs.getConnectorParticipantId(),
- dataOfferRs.getOrganizationName()
- );
-
- var dataOffer = new CatalogDataOffer();
- dataOffer.setAssetId(dataOfferRs.getAssetId());
- dataOffer.setCreatedAt(dataOfferRs.getCreatedAt());
- dataOffer.setUpdatedAt(dataOfferRs.getUpdatedAt());
- dataOffer.setAsset(asset);
- dataOffer.setContractOffers(buildCatalogContractOffers(dataOfferRs));
- dataOffer.setConnectorEndpoint(dataOfferRs.getConnectorEndpoint());
- dataOffer.setConnectorOfflineSinceOrLastUpdatedAt(dataOfferRs.getConnectorOfflineSinceOrLastUpdatedAt());
- dataOffer.setConnectorOnlineStatus(getOnlineStatus(dataOfferRs));
- return dataOffer;
- }
-
- private List buildCatalogContractOffers(DataOfferListEntryRs dataOfferRs) {
- return dataOfferRs.getContractOffers().stream()
- .map(this::buildCatalogContractOffer)
- .toList();
- }
-
- private CatalogContractOffer buildCatalogContractOffer(ContractOfferRs contractOfferDbRow) {
- var contractOffer = new CatalogContractOffer();
- contractOffer.setContractOfferId(contractOfferDbRow.getContractOfferId());
- contractOffer.setContractPolicy(dataOfferMappingUtils.buildUiPolicy(contractOfferDbRow.getPolicyJson()));
- contractOffer.setCreatedAt(contractOfferDbRow.getCreatedAt());
- contractOffer.setUpdatedAt(contractOfferDbRow.getUpdatedAt());
- return contractOffer;
- }
-
- private ConnectorOnlineStatus getOnlineStatus(DataOfferListEntryRs dataOfferRs) {
- return switch (dataOfferRs.getConnectorOnlineStatus()) {
- case ONLINE -> ConnectorOnlineStatus.ONLINE;
- case OFFLINE -> ConnectorOnlineStatus.OFFLINE;
- case DEAD -> ConnectorOnlineStatus.DEAD;
- default -> throw new IllegalStateException("Unknown ConnectorOnlineStatus from DAO for API: " + dataOfferRs.getConnectorOnlineStatus());
- };
- }
-
- private static List buildAvailableSortings() {
- return Stream.of(
- CatalogPageSortingType.MOST_RECENT,
- CatalogPageSortingType.TITLE,
- CatalogPageSortingType.ORIGINATOR,
- CatalogPageSortingType.VIEW_COUNT
- ).map(it -> new CatalogPageSortingItem(it, it.getTitle())).toList();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorApiService.java
deleted file mode 100644
index ffa3c9c5e..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorApiService.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorCreationRequest;
-import de.sovity.edc.ext.brokerserver.api.model.AddedConnector;
-import de.sovity.edc.ext.brokerserver.dao.ConnectorQueries;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
-import de.sovity.edc.ext.brokerserver.utils.MdsIdUtils;
-import de.sovity.edc.ext.brokerserver.utils.UrlUtils;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.List;
-import java.util.Objects;
-
-import static de.sovity.edc.ext.brokerserver.services.queue.ConnectorRefreshPriority.ADDED_ON_API_CALL;
-import static java.util.stream.Collectors.toSet;
-
-@RequiredArgsConstructor
-public class ConnectorApiService {
- private final ConnectorService connectorService;
- private final BrokerEventLogger brokerEventLogger;
- private final ConnectorQueries connectorQueries;
-
-
- public void addConnectors(DSLContext dsl, List connectorEndpoints) {
- var existingEndpoints = connectorService.getConnectorEndpoints(dsl);
- var endpoints = connectorEndpoints.stream()
- .filter(Objects::nonNull)
- .map(String::trim)
- .filter(UrlUtils::isValidUrl)
- .filter(endpoint -> !existingEndpoints.contains(endpoint))
- .collect(toSet());
- connectorService.addConnectors(dsl, endpoints, ADDED_ON_API_CALL);
- }
-
- public void addConnectorsWithMdsIds(DSLContext dsl, ConnectorCreationRequest connectorCreationRequests) {
- var connectors = connectorCreationRequests.getConnectors();
- var existingEndpoints = connectorService.getConnectorEndpoints(dsl);
-
- connectors.removeIf(it -> it.getConnectorEndpoint() == null || it.getMdsId() == null);
- connectors.forEach(it -> {
- it.setConnectorEndpoint(it.getConnectorEndpoint().trim());
- it.setMdsId(it.getMdsId().trim());
- });
- connectors.removeIf(it ->
- !UrlUtils.isValidUrl(it.getConnectorEndpoint())
- || !MdsIdUtils.isValidMdsId(it.getMdsId())
- || existingEndpoints.contains(it.getConnectorEndpoint())
- );
-
- var endpoints = connectors.stream().map(AddedConnector::getConnectorEndpoint).collect(toSet());
- connectorService.addConnectors(dsl, endpoints, ADDED_ON_API_CALL);
- addMdsIdsToConnectors(dsl, connectors);
- }
-
- public void deleteConnectors(DSLContext dsl, List connectorEndpoints) {
- connectorService.deleteConnectors(dsl, connectorEndpoints);
- brokerEventLogger.logConnectorsDeleted(dsl, connectorEndpoints);
- }
-
- private void addMdsIdsToConnectors(DSLContext dsl, List connectors) {
- connectors.forEach(it -> {
- var connector = connectorQueries.findByEndpoint(dsl, it.getConnectorEndpoint());
- if (connector != null) {
- connector.setMdsId(it.getMdsId());
- connector.update();
- }
- });
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorDetailApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorDetailApiService.java
deleted file mode 100644
index f44097b6e..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorDetailApiService.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorDetailPageResult;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.ConnectorDetailQueryService;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.Objects;
-
-@RequiredArgsConstructor
-public class ConnectorDetailApiService {
- private final ConnectorDetailQueryService connectorDetailQueryService;
- private final ConnectorOnlineStatusMapper connectorOnlineStatusMapper;
-
- public ConnectorDetailPageResult connectorDetailPage(DSLContext dsl, ConnectorDetailPageQuery query) {
- Objects.requireNonNull(query, "query must not be null");
-
- var connectorDbRow = connectorDetailQueryService.queryConnectorDetailPage(dsl, query.getConnectorEndpoint());
- var dto = new ConnectorDetailPageResult();
- dto.setParticipantId(connectorDbRow.getParticipantId());
- dto.setEndpoint(connectorDbRow.getEndpoint());
- dto.setOrganizationName(connectorDbRow.getOrganizationName());
- dto.setCreatedAt(connectorDbRow.getCreatedAt());
- dto.setLastRefreshAttemptAt(connectorDbRow.getLastRefreshAttemptAt());
- dto.setLastSuccessfulRefreshAt(connectorDbRow.getLastSuccessfulRefreshAt());
- dto.setOnlineStatus(connectorOnlineStatusMapper.getOnlineStatus(connectorDbRow.getOnlineStatus()));
- dto.setNumDataOffers(connectorDbRow.getNumDataOffers());
- dto.setConnectorCrawlingTimeAvg(connectorDbRow.getConnectorCrawlingTimeAvg());
- return dto;
- }
-
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorListApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorListApiService.java
deleted file mode 100644
index ca0a8d965..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorListApiService.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorListEntry;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageResult;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageSortingItem;
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorPageSortingType;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.ConnectorListQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.connector.model.ConnectorListEntryRs;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Stream;
-
-@RequiredArgsConstructor
-public class ConnectorListApiService {
- private final ConnectorListQueryService connectorListQueryService;
- private final ConnectorOnlineStatusMapper connectorOnlineStatusMapper;
- private final PaginationMetadataUtils paginationMetadataUtils;
-
- public ConnectorPageResult connectorListPage(DSLContext dsl, ConnectorPageQuery query) {
- Objects.requireNonNull(query, "query must not be null");
-
- var availableSortings = buildAvailableSortings();
- var sorting = query.getSorting();
- if (sorting == null) {
- sorting = availableSortings.get(0).getSorting();
- }
-
- var connectorDbRows = connectorListQueryService.queryConnectorPage(dsl, query.getSearchQuery(), sorting);
-
- var result = new ConnectorPageResult();
- result.setAvailableSortings(availableSortings);
- result.setPaginationMetadata(paginationMetadataUtils.buildDummyPaginationMetadata(connectorDbRows.size()));
- result.setConnectors(buildConnectorListEntries(connectorDbRows));
- return result;
- }
-
- private List buildConnectorListEntries(List connectors) {
- return connectors.stream().map(this::buildConnectorListEntry).toList();
- }
-
- private ConnectorListEntry buildConnectorListEntry(ConnectorListEntryRs connector) {
- var dto = new ConnectorListEntry();
- dto.setParticipantId(connector.getParticipantId());
- dto.setEndpoint(connector.getEndpoint());
- dto.setOrganizationName(connector.getOrganizationName());
- dto.setCreatedAt(connector.getCreatedAt());
- dto.setLastRefreshAttemptAt(connector.getLastRefreshAttemptAt());
- dto.setLastSuccessfulRefreshAt(connector.getLastSuccessfulRefreshAt());
- dto.setOnlineStatus(connectorOnlineStatusMapper.getOnlineStatus(connector.getOnlineStatus()));
- dto.setNumDataOffers(connector.getNumDataOffers());
- return dto;
- }
-
- private List buildAvailableSortings() {
- return Stream.of(
- ConnectorPageSortingType.MOST_RECENT,
- ConnectorPageSortingType.TITLE
- ).map(it -> new ConnectorPageSortingItem(it, it.getTitle())).toList();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorOnlineStatusMapper.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorOnlineStatusMapper.java
deleted file mode 100644
index 15f204711..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorOnlineStatusMapper.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorOnlineStatus;
-import lombok.RequiredArgsConstructor;
-
-@RequiredArgsConstructor
-public class ConnectorOnlineStatusMapper {
-
- public ConnectorOnlineStatus getOnlineStatus(de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus onlineStatus) {
- return switch (onlineStatus) {
- case ONLINE -> ConnectorOnlineStatus.ONLINE;
- case OFFLINE -> ConnectorOnlineStatus.OFFLINE;
- case DEAD -> ConnectorOnlineStatus.DEAD;
- default -> throw new IllegalStateException("Unknown ConnectorOnlineStatus from DAO for API: " + onlineStatus);
- };
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorService.java
deleted file mode 100644
index b3996dcb8..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorService.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.services.ConnectorCreator;
-import de.sovity.edc.ext.brokerserver.services.queue.ConnectorQueue;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-import org.jooq.Record;
-import org.jooq.TableField;
-
-import java.util.Collection;
-import java.util.Set;
-
-@RequiredArgsConstructor
-public class ConnectorService {
- private final ConnectorCreator connectorCreator;
- private final ConnectorQueue connectorQueue;
-
- public void addConnectors(DSLContext dsl, Collection connectorEndpoints, int priority) {
- connectorCreator.addConnectors(dsl, connectorEndpoints);
- connectorQueue.addAll(connectorEndpoints, priority);
- }
-
- public void deleteConnectors(DSLContext dsl, Collection endpoints) {
- removeConnectorRows(dsl, Tables.BROKER_EXECUTION_TIME_MEASUREMENT.CONNECTOR_ENDPOINT, endpoints);
- removeConnectorRows(dsl, Tables.CONTRACT_OFFER.CONNECTOR_ENDPOINT, endpoints);
- removeConnectorRows(dsl, Tables.DATA_OFFER.CONNECTOR_ENDPOINT, endpoints);
- removeConnectorRows(dsl, Tables.DATA_OFFER_VIEW_COUNT.CONNECTOR_ENDPOINT, endpoints);
- removeConnectorRows(dsl, Tables.CONNECTOR.ENDPOINT, endpoints);
- }
-
- public Set getConnectorEndpoints(DSLContext dsl) {
- return dsl.select(Tables.CONNECTOR.ENDPOINT).from(Tables.CONNECTOR).fetchSet(Tables.CONNECTOR.ENDPOINT);
- }
-
- private void removeConnectorRows(
- DSLContext dsl,
- TableField endpointField,
- Collection endpoints
- ) {
- dsl.deleteFrom(endpointField.getTable()).where(endpointField.in(endpoints)).execute();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiService.java
deleted file mode 100644
index a79bd8b96..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiService.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailContractOffer;
-import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.api.model.DataOfferDetailPageResult;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.DataOfferDetailPageQueryService;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.ViewCountLogger;
-import de.sovity.edc.ext.brokerserver.dao.pages.dataoffer.model.ContractOfferRs;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-import org.jooq.DSLContext;
-
-import java.util.List;
-import java.util.Objects;
-
-@RequiredArgsConstructor
-public class DataOfferDetailApiService {
- private final DataOfferDetailPageQueryService dataOfferDetailPageQueryService;
- private final ViewCountLogger viewCountLogger;
- private final DataOfferMappingUtils dataOfferMappingUtils;
-
- public DataOfferDetailPageResult dataOfferDetailPage(DSLContext dsl, DataOfferDetailPageQuery query) {
- Objects.requireNonNull(query, "query must not be null");
-
- var dataOffer = dataOfferDetailPageQueryService.queryDataOfferDetailsPage(dsl, query.getAssetId(), query.getConnectorEndpoint());
- var asset = dataOfferMappingUtils.buildUiAsset(
- dataOffer.getAssetJsonLd(),
- dataOffer.getConnectorEndpoint(),
- dataOffer.getConnectorParticipantId(),
- dataOffer.getOrganizationName()
- );
- viewCountLogger.increaseDataOfferViewCount(dsl, query.getAssetId(), query.getConnectorEndpoint());
-
- var result = new DataOfferDetailPageResult();
- result.setAssetId(dataOffer.getAssetId());
- result.setConnectorEndpoint(dataOffer.getConnectorEndpoint());
- result.setConnectorOnlineStatus(mapConnectorOnlineStatus(dataOffer.getConnectorOnlineStatus()));
- result.setConnectorOfflineSinceOrLastUpdatedAt(dataOffer.getConnectorOfflineSinceOrLastUpdatedAt());
- result.setAsset(asset);
- result.setCreatedAt(dataOffer.getCreatedAt());
- result.setUpdatedAt(dataOffer.getUpdatedAt());
- result.setContractOffers(buildDataOfferDetailContractOffers(dataOffer.getContractOffers()));
- result.setViewCount(dataOffer.getViewCount());
- return result;
- }
-
- private ConnectorOnlineStatus mapConnectorOnlineStatus(
- de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus connectorOnlineStatus
- ) {
- if (connectorOnlineStatus == null) {
- return ConnectorOnlineStatus.OFFLINE;
- }
-
- return switch (connectorOnlineStatus) {
- case ONLINE -> ConnectorOnlineStatus.ONLINE;
- case OFFLINE -> ConnectorOnlineStatus.OFFLINE;
- case DEAD -> ConnectorOnlineStatus.DEAD;
- };
- }
-
- private List buildDataOfferDetailContractOffers(List contractOffers) {
- return contractOffers.stream().map(this::buildDataOfferDetailContractOffer).toList();
- }
-
- @NotNull
- private DataOfferDetailContractOffer buildDataOfferDetailContractOffer(ContractOfferRs offer) {
- var newOffer = new DataOfferDetailContractOffer();
- newOffer.setCreatedAt(offer.getCreatedAt());
- newOffer.setUpdatedAt(offer.getUpdatedAt());
- newOffer.setContractOfferId(offer.getContractOfferId());
- newOffer.setContractPolicy(dataOfferMappingUtils.buildUiPolicy(offer.getPolicyJson()));
- return newOffer;
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferMappingUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferMappingUtils.java
deleted file mode 100644
index 77f0cd8f4..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferMappingUtils.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
-import de.sovity.edc.ext.wrapper.api.common.model.UiAsset;
-import de.sovity.edc.ext.wrapper.api.common.model.UiPolicy;
-import de.sovity.edc.utils.JsonUtils;
-import lombok.RequiredArgsConstructor;
-
-@RequiredArgsConstructor
-public class DataOfferMappingUtils {
- private final PolicyMapper policyMapper;
- private final AssetMapper assetMapper;
-
- public UiAsset buildUiAsset(String assetJsonLd, String endpoint, String participantId, String organizationName) {
- var asset = assetMapper.buildAsset(JsonUtils.parseJsonObj(assetJsonLd));
- var uiAsset = assetMapper.buildUiAsset(asset, endpoint, participantId);
- uiAsset.setCreatorOrganizationName(organizationName);
- return uiAsset;
- }
-
- public UiPolicy buildUiPolicy(String policyJson) {
- var policy = policyMapper.buildPolicy(policyJson);
- return policyMapper.buildUiPolicy(policy);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/PaginationMetadataUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/PaginationMetadataUtils.java
deleted file mode 100644
index d69cbb707..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/PaginationMetadataUtils.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.api.model.PaginationMetadata;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.PageQuery;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-
-@RequiredArgsConstructor
-public class PaginationMetadataUtils {
-
- @NotNull
- public PaginationMetadata buildDummyPaginationMetadata(int numResults) {
- return new PaginationMetadata(numResults, numResults, 1, numResults);
- }
-
- public PageQuery getPageQuery(Integer pageOneBased, int pageSize) {
- int pageZeroBased = getPageZeroBased(pageOneBased);
- int offset = pageZeroBased * pageSize;
- return new PageQuery(offset, pageSize);
- }
-
- public PaginationMetadata buildPaginationMetadata(Integer pageOneBased, int pageSize, int numVisible, int numTotalResults) {
- int pageZeroBased = getPageZeroBased(pageOneBased);
- var paginationMetadata = new PaginationMetadata();
- paginationMetadata.setNumTotal(numTotalResults);
- paginationMetadata.setNumVisible(numVisible);
- paginationMetadata.setPageOneBased(pageZeroBased + 1);
- paginationMetadata.setPageSize(pageSize);
- return paginationMetadata;
- }
-
- private int getPageZeroBased(Integer pageOneBased) {
- return pageOneBased == null ? 0 : (pageOneBased - 1);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/AttributeFilterQuery.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/AttributeFilterQuery.java
deleted file mode 100644
index 18c9218ba..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/AttributeFilterQuery.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api.filtering;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import org.jooq.Condition;
-
-import java.util.Collection;
-
-@FunctionalInterface
-public interface AttributeFilterQuery {
-
- /**
- * Filters a Catalog DB Query for a given Filter Attribute with selected values
- *
- * @param fields available tables and fields during the catalog query
- * @param values values to be filtered by. Usually this should mean that only one of the values needs to be present.
- * @return {@link Condition}
- */
- Condition filterDataOffers(CatalogQueryFields fields, Collection values);
-
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterAttributeDefinition.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterAttributeDefinition.java
deleted file mode 100644
index 9d13a2c00..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterAttributeDefinition.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api.filtering;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.AvailableFilterValuesQuery;
-
-/**
- * Implementation of a filter attribute definition for the catalog.
- *
- * @param name technical id of the attribute
- * @param label UI showing label for the attribute
- * @param valueGetter query existing values from DB
- * @param filterApplier apply a filter to a data offer query
- */
-public record CatalogFilterAttributeDefinition(
- String name,
- String label,
- AvailableFilterValuesQuery valueGetter,
- AttributeFilterQuery filterApplier
-) {
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterAttributeDefinitionService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterAttributeDefinitionService.java
deleted file mode 100644
index 088b2cc84..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterAttributeDefinitionService.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api.filtering;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.utils.PostgresqlUtils;
-import org.jooq.Field;
-
-import java.util.function.Function;
-
-public class CatalogFilterAttributeDefinitionService {
-
- public CatalogFilterAttributeDefinition forField(
- Function> fieldExtractor,
- String name,
- String label
- ) {
- return new CatalogFilterAttributeDefinition(
- name,
- label,
- fieldExtractor::apply,
- (fields, values) -> PostgresqlUtils.in(fieldExtractor.apply(fields), values)
- );
- }
-
- public CatalogFilterAttributeDefinition buildDataSpaceFilter() {
- return new CatalogFilterAttributeDefinition(
- "dataSpace",
- "Data Space",
- CatalogQueryFields::getDataSpace,
- (fields, values) -> PostgresqlUtils.in(fields.getDataSpace(), values)
- );
- }
-
- public CatalogFilterAttributeDefinition buildConnectorEndpointFilter() {
- return new CatalogFilterAttributeDefinition(
- "connectorEndpoint",
- "Connector",
- fields -> fields.getDataOfferTable().CONNECTOR_ENDPOINT,
- (fields, values) -> PostgresqlUtils.in(fields.getDataOfferTable().CONNECTOR_ENDPOINT, values)
- );
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterService.java
deleted file mode 100644
index 860a796ab..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogFilterService.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api.filtering;
-
-import de.sovity.edc.ext.brokerserver.api.model.CnfFilter;
-import de.sovity.edc.ext.brokerserver.api.model.CnfFilterAttribute;
-import de.sovity.edc.ext.brokerserver.api.model.CnfFilterItem;
-import de.sovity.edc.ext.brokerserver.api.model.CnfFilterValue;
-import de.sovity.edc.ext.brokerserver.api.model.CnfFilterValueAttribute;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogQueryFilter;
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.models.CatalogQuerySelectedFilterQuery;
-import de.sovity.edc.ext.brokerserver.dao.utils.JsonDeserializationUtils;
-import de.sovity.edc.ext.brokerserver.utils.CollectionUtils2;
-import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.Validate;
-import org.jooq.impl.DSL;
-
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import static java.util.stream.Collectors.toMap;
-
-@RequiredArgsConstructor
-public class CatalogFilterService {
- private final CatalogFilterAttributeDefinitionService catalogFilterAttributeDefinitionService;
-
- private final Comparator caseInsensitiveEmptyStringLast = (s1, s2) -> {
- int result = s1.compareToIgnoreCase(s2);
- if (s1.isEmpty() && !s2.isEmpty()) {
- return 1;
- } else if (!s1.isEmpty() && s2.isEmpty()) {
- return -1;
- } else {
- return result;
- }
- };
-
-
- /**
- * Currently supported filters for the catalog page.
- *
- * @return attribute definitions
- */
- private List getAvailableFilters() {
- return List.of(
- catalogFilterAttributeDefinitionService.buildDataSpaceFilter(),
- catalogFilterAttributeDefinitionService.forField(
- fields -> fields.getDataOfferTable().DATA_CATEGORY,
- "dataCategory",
- "Data Category"
- ),
- catalogFilterAttributeDefinitionService.forField(
- fields -> fields.getDataOfferTable().DATA_SUBCATEGORY,
- "dataSubcategory",
- "Data Subcategory"
- ),
- catalogFilterAttributeDefinitionService.forField(
- fields -> fields.getDataOfferTable().DATA_MODEL,
- "dataModel",
- "Data Model"
- ),
- catalogFilterAttributeDefinitionService.forField(
- fields -> fields.getDataOfferTable().TRANSPORT_MODE,
- "transportMode",
- "Transport Mode"
- ),
- catalogFilterAttributeDefinitionService.forField(
- fields -> fields.getDataOfferTable().GEO_REFERENCE_METHOD,
- "geoReferenceMethod",
- "Geo Reference Method"
- ),
- catalogFilterAttributeDefinitionService.forField(
- CatalogQueryFields::getOrganizationName,
- "curatorOrganizationName",
- "Organization Name"
- ),
- catalogFilterAttributeDefinitionService.forField(
- fields -> DSL.coalesce(fields.getConnectorTable().MDS_ID, "Unknown"),
- "curatorMdsId",
- "MDS ID"
- ),
- catalogFilterAttributeDefinitionService.buildConnectorEndpointFilter()
- );
- }
-
- public List getCatalogQueryFilters(CnfFilterValue cnfFilterValue) {
- var values = getCnfFilterValuesMap(cnfFilterValue);
- return getAvailableFilters().stream()
- .map(filter -> new CatalogQueryFilter(
- filter.name(),
- filter.valueGetter(),
- getQueryFilter(filter, values.get(filter.name()))
- ))
- .toList();
- }
-
- private CatalogQuerySelectedFilterQuery getQueryFilter(CatalogFilterAttributeDefinition filter, List values) {
- if (CollectionUtils2.isNotEmpty(values)) {
- return fields -> filter.filterApplier().filterDataOffers(fields, values);
- }
- return null;
- }
-
- public CnfFilter buildAvailableFilters(String filterValuesJson) {
- var filterValues = JsonDeserializationUtils.read2dStringList(filterValuesJson);
- var filterAttributes = zipAvailableFilters(getAvailableFilters(), filterValues)
- .map(availableFilter -> new CnfFilterAttribute(
- availableFilter.definition().name(),
- availableFilter.definition().label(),
- buildAvailableFilterValues(availableFilter)
- ))
- .toList();
- return new CnfFilter(filterAttributes);
- }
-
- private List buildAvailableFilterValues(AvailableFilter availableFilter) {
- return availableFilter.availableValues().stream()
- .sorted(caseInsensitiveEmptyStringLast)
- .map(value -> new CnfFilterItem(value, value))
- .toList();
- }
-
- private Stream zipAvailableFilters(List availableFilters, List> filterValues) {
- Validate.isTrue(
- availableFilters.size() == filterValues.size(),
- "Number of available filters and filter values must match: %d != %d",
- availableFilters.size(),
- filterValues.size()
- );
- return Stream.iterate(0, i -> i + 1)
- .limit(availableFilters.size())
- .map(i -> new AvailableFilter(availableFilters.get(i), filterValues.get(i)));
- }
-
- private record AvailableFilter(CatalogFilterAttributeDefinition definition, List availableValues) {
- }
-
- private Map> getCnfFilterValuesMap(CnfFilterValue cnfFilterValue) {
- if (cnfFilterValue == null || cnfFilterValue.getSelectedAttributeValues() == null) {
- return Map.of();
- }
- return cnfFilterValue.getSelectedAttributeValues().stream()
- .filter(it -> it.getId() != null && CollectionUtils2.isNotEmpty(it.getSelectedIds()))
- .collect(toMap(CnfFilterValueAttribute::getId, CnfFilterValueAttribute::getSelectedIds));
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogSearchService.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogSearchService.java
deleted file mode 100644
index b7abd8b6e..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/api/filtering/CatalogSearchService.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api.filtering;
-
-import de.sovity.edc.ext.brokerserver.dao.pages.catalog.CatalogQueryFields;
-import de.sovity.edc.ext.brokerserver.dao.utils.SearchUtils;
-import lombok.RequiredArgsConstructor;
-import org.jooq.Condition;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class CatalogSearchService {
-
- public Condition filterBySearch(CatalogQueryFields fields, String searchQuery) {
- return SearchUtils.simpleSearch(searchQuery, List.of(
- fields.getDataOfferTable().ASSET_ID,
- fields.getDataOfferTable().ASSET_TITLE,
- fields.getDataOfferTable().DATA_CATEGORY,
- fields.getDataOfferTable().DATA_SUBCATEGORY,
- fields.getDataOfferTable().DESCRIPTION,
- fields.getDataOfferTable().CURATOR_ORGANIZATION_NAME,
- fields.getDataOfferTable().KEYWORDS_COMMA_JOINED,
- fields.getConnectorTable().ENDPOINT,
- fields.getConnectorTable().PARTICIPANT_ID,
- fields.getOrganizationName()
- ));
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/AdminApiKeyValidator.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/AdminApiKeyValidator.java
deleted file mode 100644
index e5f9571b9..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/AdminApiKeyValidator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.config;
-
-import jakarta.ws.rs.WebApplicationException;
-import jakarta.ws.rs.core.Response;
-import lombok.RequiredArgsConstructor;
-
-@RequiredArgsConstructor
-public class AdminApiKeyValidator {
- private final BrokerServerSettings brokerServerSettings;
-
- public void validateAdminApiKey(String adminApiKey) {
- if (!brokerServerSettings.getAdminApiKey().equals(adminApiKey)) {
- throw new WebApplicationException("Invalid admin API key", Response.Status.UNAUTHORIZED);
- }
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/BrokerServerSettingsFactory.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/BrokerServerSettingsFactory.java
deleted file mode 100644
index a09814341..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/BrokerServerSettingsFactory.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.config;
-
-import de.sovity.edc.ext.brokerserver.BrokerServerExtension;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.Validate;
-import org.eclipse.edc.spi.monitor.Monitor;
-import org.eclipse.edc.spi.system.configuration.Config;
-
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.List;
-
-@RequiredArgsConstructor
-public class BrokerServerSettingsFactory {
- private final Config config;
- private final Monitor monitor;
-
- public BrokerServerSettings buildBrokerServerSettings() {
- var adminApiKey = Validate.notBlank(config.getString(BrokerServerExtension.ADMIN_API_KEY),
- "Need to configure %s.".formatted(BrokerServerExtension.ADMIN_API_KEY));
- var hideOfflineDataOffersAfter = getDuration(BrokerServerExtension.HIDE_OFFLINE_DATA_OFFERS_AFTER, null);
- var catalogPagePageSize = config.getInteger(BrokerServerExtension.CATALOG_PAGE_PAGE_SIZE, 20);
- var dataSpaceConfig = buildDataSpaceConfig(config);
- var numThreads = config.getInteger(BrokerServerExtension.NUM_THREADS, 1);
- var killOfflineConnectorsAfter = getDuration(BrokerServerExtension.KILL_OFFLINE_CONNECTORS_AFTER, Duration.ofDays(5));
- var maxDataOffers = config.getInteger(BrokerServerExtension.MAX_DATA_OFFERS_PER_CONNECTOR, -1);
- var maxContractOffers = config.getInteger(BrokerServerExtension.MAX_CONTRACT_OFFERS_PER_DATA_OFFER, -1);
-
- return BrokerServerSettings.builder()
- .adminApiKey(adminApiKey)
- .hideOfflineDataOffersAfter(hideOfflineDataOffersAfter)
- .catalogPagePageSize(catalogPagePageSize)
- .dataSpaceConfig(dataSpaceConfig)
- .numThreads(numThreads)
- .killOfflineConnectorsAfter(killOfflineConnectorsAfter)
- .maxDataOffersPerConnector(maxDataOffers)
- .maxContractOffersPerDataOffer(maxContractOffers)
- .build();
- }
-
- private DataSpaceConfig buildDataSpaceConfig(Config config) {
- var dataSpaceConfig = new DataSpaceConfig(getKnownDataSpaceEndpoints(config), getDefaultDataSpace(config));
- monitor.info("Default Dataspace Name: %s".formatted(dataSpaceConfig.defaultDataSpace()));
- dataSpaceConfig.dataSpaceConnectors().forEach(dataSpaceConnector -> monitor.info("Using Dataspace Name %s for %s."
- .formatted(dataSpaceConnector.dataSpaceName(), dataSpaceConnector.endpoint())));
- if (dataSpaceConfig.dataSpaceConnectors().isEmpty()) {
- monitor.info("No additional data space names configured.");
- }
- return dataSpaceConfig;
- }
-
- private List getKnownDataSpaceEndpoints(Config config) {
- // Example: "Example1=http://connector-endpoint1.org,Example2=http://connector-endpoint2.org"
- var dataSpacesConfig = config.getString(BrokerServerExtension.KNOWN_DATASPACE_CONNECTORS, "");
-
- return Arrays.stream(dataSpacesConfig.split(","))
- .map(String::trim)
- .map(it -> it.split("="))
- .filter(it -> it.length == 2)
- .map(it -> {
- var dataSpaceName = it[0].trim();
- var dataSpaceEndpoint = it[1].trim();
- return new DataSpaceConnector(dataSpaceEndpoint, dataSpaceName);
- })
- .filter(it -> StringUtils.isNotBlank(it.endpoint()) && StringUtils.isNotBlank(it.endpoint()))
- .toList();
- }
-
- private String getDefaultDataSpace(Config config) {
- return config.getString(BrokerServerExtension.DEFAULT_CONNECTOR_DATASPACE, "Default");
- }
-
- private Duration getDuration(@NonNull String configProperty, Duration defaultValue) {
- var value = config.getString(configProperty, "");
-
- if (StringUtils.isBlank(value)) {
- return defaultValue;
- }
-
- return Duration.parse(value);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/DataSpaceConfig.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/DataSpaceConfig.java
deleted file mode 100644
index 705a5c1cc..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/DataSpaceConfig.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.config;
-
-import java.util.List;
-
-public record DataSpaceConfig(List dataSpaceConnectors, String defaultDataSpace) {
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/DataSpaceConnector.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/DataSpaceConnector.java
deleted file mode 100644
index 589f08aa0..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/config/DataSpaceConnector.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.config;
-
-/**
- * We have special connectors that represent entire other data spaces.
- * Here we associate the name of the data space with the connector endpoint.
- */
-public record DataSpaceConnector(String endpoint, String dataSpaceName) {
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventLogger.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventLogger.java
deleted file mode 100644
index 439e81be0..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventLogger.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.logging;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.BrokerEventStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.BrokerEventType;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.time.OffsetDateTime;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Updates a single connector.
- */
-@RequiredArgsConstructor
-public class BrokerEventLogger {
-
- public void logConnectorsDeleted(DSLContext dsl, Collection connectorEndpoints) {
- var records = connectorEndpoints.stream().map(connectorEndpoint -> {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_DELETED);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(connectorEndpoint);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.setUserMessage("Connector was deleted.");
- return logEntry;
- }).toList();
- dsl.batchInsert(records).execute();
- }
-
- public void logConnectorUpdated(DSLContext dsl, String connectorEndpoint, ConnectorChangeTracker changes) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_UPDATED);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(connectorEndpoint);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.setUserMessage(changes.toString());
- logEntry.insert();
- }
-
- public void logConnectorOffline(DSLContext dsl, String connectorEndpoint, BrokerEventErrorMessage errorMessage) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_STATUS_CHANGE_OFFLINE);
- logEntry.setEventStatus(BrokerEventStatus.ERROR);
- logEntry.setConnectorEndpoint(connectorEndpoint);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.setUserMessage("Connector is offline.");
- logEntry.setErrorStack(errorMessage.stackTraceOrNull());
- logEntry.insert();
- }
-
- public void logConnectorOnline(DSLContext dsl, String connectorEndpoint) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_STATUS_CHANGE_ONLINE);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(connectorEndpoint);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.setUserMessage("Connector is online.");
- logEntry.insert();
- }
-
- public void logConnectorUpdateDataOfferLimitExceeded(DSLContext dsl, Integer maxDataOffersPerConnector, String endpoint) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_DATA_OFFER_LIMIT_EXCEEDED);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(endpoint);
- logEntry.setUserMessage("Connector has more than %d data offers. Exceeding data offers will be ignored.".formatted(maxDataOffersPerConnector));
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.insert();
- }
-
- public void logConnectorUpdateDataOfferLimitOk(DSLContext dsl, String endpoint) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_DATA_OFFER_LIMIT_OK);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(endpoint);
- logEntry.setUserMessage("Connector is not exceeding the maximum number of data offers limit anymore.");
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.insert();
- }
-
- public void logConnectorUpdateContractOfferLimitExceeded(DSLContext dsl, Integer maxContractOffersPerConnector, String endpoint) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_CONTRACT_OFFER_LIMIT_EXCEEDED);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(endpoint);
- logEntry.setUserMessage("Some data offers have more than %d contract offers. Exceeding contract offers will be ignored.: ".formatted(maxContractOffersPerConnector));
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.insert();
- }
-
- public void logConnectorUpdateContractOfferLimitOk(DSLContext dsl, String endpoint) {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_CONTRACT_OFFER_LIMIT_OK);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setConnectorEndpoint(endpoint);
- logEntry.setUserMessage("Connector is not exceeding the maximum number of contract offers per data offer limit anymore.");
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.insert();
- }
-
- public void addKilledDueToOfflineTooLongMessages(DSLContext dsl, List deletedConnectorEndpoints) {
- var logEntries = deletedConnectorEndpoints.stream().map(endpoint -> {
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_KILLED_DUE_TO_OFFLINE_FOR_TOO_LONG);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.setUserMessage("Connector was marked as dead for being offline too long.");
- logEntry.setConnectorEndpoint(endpoint);
- return logEntry;
- }).collect(Collectors.toList());
-
- dsl.batchInsert(logEntries).execute();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerExecutionTimeLogger.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerExecutionTimeLogger.java
deleted file mode 100644
index 7743ac753..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerExecutionTimeLogger.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.logging;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementErrorStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementType;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.time.OffsetDateTime;
-
-/**
- * Updates a single connector.
- */
-@RequiredArgsConstructor
-public class BrokerExecutionTimeLogger {
- public void logExecutionTime(DSLContext dsl, String connectorEndpoint, long executionTimeInMs, MeasurementErrorStatus errorStatus) {
- var logEntry = dsl.newRecord(Tables.BROKER_EXECUTION_TIME_MEASUREMENT);
- logEntry.setConnectorEndpoint(connectorEndpoint);
- logEntry.setDurationInMs(executionTimeInMs);
- logEntry.setType(MeasurementType.CONNECTOR_REFRESH);
- logEntry.setErrorStatus(errorStatus);
- logEntry.setConnectorEndpoint(connectorEndpoint);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.insert();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/queue/ConnectorQueue.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/queue/ConnectorQueue.java
deleted file mode 100644
index a1d790494..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/queue/ConnectorQueue.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.queue;
-
-import de.sovity.edc.ext.brokerserver.services.refreshing.ConnectorUpdater;
-import lombok.RequiredArgsConstructor;
-
-import java.util.ArrayList;
-import java.util.Collection;
-
-@RequiredArgsConstructor
-public class ConnectorQueue {
- private final ConnectorUpdater connectorUpdater;
- private final ThreadPool threadPool;
-
- /**
- * Enqueues connectors for update.
- *
- * @param endpoints connector endpoints
- * @param priority priority from {@link ConnectorRefreshPriority}
- */
- public void addAll(Collection endpoints, int priority) {
- var queuedConnectorEndpoints = threadPool.getQueuedConnectorEndpoints();
- endpoints = new ArrayList<>(endpoints);
- endpoints.removeIf(queuedConnectorEndpoints::contains);
-
- for (String endpoint : endpoints) {
- threadPool.enqueueConnectorRefreshTask(priority, () -> connectorUpdater.updateConnector(endpoint), endpoint);
- }
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdateSuccessWriter.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdateSuccessWriter.java
deleted file mode 100644
index 51172fd21..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdateSuccessWriter.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
-import de.sovity.edc.ext.brokerserver.services.logging.ConnectorChangeTracker;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferLimitsEnforcer;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.DataOfferWriter;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedCatalog;
-import de.sovity.edc.ext.brokerserver.utils.MdsIdUtils;
-import lombok.RequiredArgsConstructor;
-import org.jooq.DSLContext;
-
-import java.time.OffsetDateTime;
-import java.util.Objects;
-
-@RequiredArgsConstructor
-public class ConnectorUpdateSuccessWriter {
- private final BrokerEventLogger brokerEventLogger;
- private final DataOfferWriter dataOfferWriter;
- private final DataOfferLimitsEnforcer dataOfferLimitsEnforcer;
-
- public void handleConnectorOnline(
- DSLContext dsl,
- ConnectorRecord connector,
- FetchedCatalog catalog
- ) {
- // Limit data offers and log limitation if necessary
- var limitedDataOffers = dataOfferLimitsEnforcer.enforceLimits(catalog.getDataOffers());
- dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connector, limitedDataOffers);
-
- // Log Status Change and set status to online if necessary
- if (connector.getOnlineStatus() != ConnectorOnlineStatus.ONLINE || connector.getLastRefreshAttemptAt() == null) {
- brokerEventLogger.logConnectorOnline(dsl, connector.getEndpoint());
- connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
- }
-
- // Track changes for final log message
- var changes = new ConnectorChangeTracker();
- updateConnector(connector, catalog, changes);
-
- // Update data offers
- dataOfferWriter.updateDataOffers(dsl, connector.getEndpoint(), limitedDataOffers.abbreviatedDataOffers(), changes);
-
- // Log event if changes are present
- if (!changes.isEmpty()) {
- brokerEventLogger.logConnectorUpdated(dsl, connector.getEndpoint(), changes);
- }
- }
-
- private static void updateConnector(ConnectorRecord connector, FetchedCatalog catalog, ConnectorChangeTracker changes) {
- var now = OffsetDateTime.now();
- var participantId = catalog.getParticipantId();
-
- connector.setLastSuccessfulRefreshAt(now);
- connector.setLastRefreshAttemptAt(now);
- if (!Objects.equals(connector.getParticipantId(), participantId)) {
- connector.setParticipantId(participantId);
- connector.setMdsId(MdsIdUtils.getMdsIdFromParticipantId(participantId));
- changes.setParticipantIdChanged(participantId);
- }
- connector.update();
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/exceptions/ConnectorUnreachableException.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/exceptions/ConnectorUnreachableException.java
deleted file mode 100644
index de9172f2a..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/exceptions/ConnectorUnreachableException.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.exceptions;
-
-public class ConnectorUnreachableException extends RuntimeException {
- public ConnectorUnreachableException(String message, Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferPatchApplier.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferPatchApplier.java
deleted file mode 100644
index 91cb0395f..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferPatchApplier.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
-
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.DataOfferPatch;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import org.jooq.DSLContext;
-
-@RequiredArgsConstructor
-public class DataOfferPatchApplier {
-
- @SneakyThrows
- public void writeDataOfferPatch(DSLContext dsl, DataOfferPatch dataOfferPatch) {
- if (!dataOfferPatch.getDataOffersToUpdate().isEmpty()) {
- dsl.batchUpdate(dataOfferPatch.getDataOffersToUpdate()).execute();
- }
- if (!dataOfferPatch.getContractOffersToUpdate().isEmpty()) {
- dsl.batchUpdate(dataOfferPatch.getContractOffersToUpdate()).execute();
- }
-
- // insert: parent entity first
- if (!dataOfferPatch.getDataOffersToInsert().isEmpty()) {
- dsl.batchInsert(dataOfferPatch.getDataOffersToInsert()).execute();
- }
- if (!dataOfferPatch.getContractOffersToInsert().isEmpty()) {
- dsl.batchInsert(dataOfferPatch.getContractOffersToInsert()).execute();
- }
-
- // delete: child entity first
- if (!dataOfferPatch.getContractOffersToDelete().isEmpty()) {
- dsl.batchDelete(dataOfferPatch.getContractOffersToDelete()).execute();
- }
- if (!dataOfferPatch.getDataOffersToDelete().isEmpty()) {
- dsl.batchDelete(dataOfferPatch.getDataOffersToDelete()).execute();
- }
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferRecordUpdater.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferRecordUpdater.java
deleted file mode 100644
index 97b1e2102..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferRecordUpdater.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
-
-import de.sovity.edc.ext.brokerserver.dao.utils.JsonbUtils;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.DataOfferRecord;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
-import de.sovity.edc.ext.brokerserver.utils.JsonUtils2;
-import lombok.RequiredArgsConstructor;
-import org.jooq.JSONB;
-
-import java.time.OffsetDateTime;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.function.Consumer;
-import java.util.function.Function;
-
-/**
- * Creates or updates {@link DataOfferRecord} DB Rows.
- *
- * (Or at least prepares them for batch inserts / updates)
- */
-@RequiredArgsConstructor
-public class DataOfferRecordUpdater {
- /**
- * Create a new {@link DataOfferRecord}.
- *
- * @param connectorEndpoint connector endpoint
- * @param fetchedDataOffer new db row data
- * @return new db row
- */
- public DataOfferRecord newDataOffer(String connectorEndpoint, FetchedDataOffer fetchedDataOffer) {
- var dataOffer = new DataOfferRecord();
- dataOffer.setConnectorEndpoint(connectorEndpoint);
- dataOffer.setAssetId(fetchedDataOffer.getAssetId());
- dataOffer.setCreatedAt(OffsetDateTime.now());
- updateDataOffer(dataOffer, fetchedDataOffer, true);
- return dataOffer;
- }
-
-
- /**
- * Update existing {@link DataOfferRecord}.
- *
- * @param dataOffer existing row
- * @param fetchedDataOffer changes to be incorporated
- * @param changed whether the data offer should be marked as updated simply because the contract offers changed
- * @return whether any fields were updated
- */
- public boolean updateDataOffer(
- DataOfferRecord dataOffer,
- FetchedDataOffer fetchedDataOffer,
- boolean changed
- ) {
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getAssetTitle,
- DataOfferRecord::getAssetTitle,
- dataOffer::setAssetTitle
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getDescription,
- DataOfferRecord::getDescription,
- dataOffer::setDescription
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getCuratorOrganizationName,
- DataOfferRecord::getCuratorOrganizationName,
- dataOffer::setCuratorOrganizationName
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getDataCategory,
- DataOfferRecord::getDataCategory,
- dataOffer::setDataCategory
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getDataSubcategory,
- DataOfferRecord::getDataSubcategory,
- dataOffer::setDataSubcategory
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getDataModel,
- DataOfferRecord::getDataModel,
- dataOffer::setDataModel
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getTransportMode,
- DataOfferRecord::getTransportMode,
- dataOffer::setTransportMode
- );
-
- changed |= updateField(
- dataOffer,
- fetchedDataOffer,
- FetchedDataOffer::getGeoReferenceMethod,
- DataOfferRecord::getGeoReferenceMethod,
- dataOffer::setGeoReferenceMethod
- );
-
- changed |= updateKeywords(dataOffer, fetchedDataOffer);
-
- changed |= updateAssetJsonLd(dataOffer, fetchedDataOffer);
-
- if (changed) {
- dataOffer.setUpdatedAt(OffsetDateTime.now());
- }
-
- return changed;
- }
-
- private boolean updateField(
- DataOfferRecord dataOffer,
- FetchedDataOffer fetchedDataOffer,
- Function fetchedField,
- Function existingField,
- Consumer setter
- ) {
- var fetched = fetchedField.apply(fetchedDataOffer);
- if (fetched == null) {
- fetched = "";
- }
-
- var existing = existingField.apply(dataOffer);
- if (existing == null) {
- existing = "";
- }
-
-
- if (Objects.equals(fetched, existing)) {
- return false;
- }
-
- setter.accept(fetched);
- return true;
- }
-
- private boolean updateKeywords(
- DataOfferRecord dataOffer,
- FetchedDataOffer fetchedDataOffer
- ) {
- List fetched = fetchedDataOffer.getKeywords();
- if (fetched == null) {
- fetched = List.of();
- }
-
- String[] existing = dataOffer.getKeywords();
- if (existing == null) {
- existing = new String[0];
- }
-
- if (Objects.equals(new HashSet<>(fetched), new HashSet<>(Arrays.asList(existing)))) {
- return false;
- }
-
- dataOffer.setKeywords(fetched.toArray(new String[0]));
- dataOffer.setKeywordsCommaJoined(String.join(",", fetched));
- return true;
- }
-
- private boolean updateAssetJsonLd(
- DataOfferRecord dataOffer,
- FetchedDataOffer fetchedDataOffer
- ) {
- String existing = JsonbUtils.getDataOrNull(dataOffer.getAssetJsonLd());
- var fetched = fetchedDataOffer.getAssetJsonLd();
- if (JsonUtils2.isEqualJson(fetched, existing)) {
- return false;
- }
-
- dataOffer.setAssetJsonLd(JSONB.jsonb(fetched));
- return true;
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferWriter.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferWriter.java
deleted file mode 100644
index e69e446ee..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferWriter.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
-
-import de.sovity.edc.ext.brokerserver.services.logging.ConnectorChangeTracker;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import org.jooq.DSLContext;
-
-import java.util.Collection;
-
-@RequiredArgsConstructor
-public class DataOfferWriter {
- private final DataOfferPatchBuilder dataOfferPatchBuilder;
- private final DataOfferPatchApplier dataOfferPatchApplier;
-
- /**
- * Updates a connector's data offers with given {@link FetchedDataOffer}s.
- *
- * @param dsl dsl
- * @param connectorEndpoint connector endpoint
- * @param fetchedDataOffers fetched data offers
- * @param changes change tracker for log message
- */
- @SneakyThrows
- public void updateDataOffers(DSLContext dsl, String connectorEndpoint, Collection fetchedDataOffers, ConnectorChangeTracker changes) {
- var patch = dataOfferPatchBuilder.buildDataOfferPatch(dsl, connectorEndpoint, fetchedDataOffers);
- changes.setNumOffersAdded(patch.getDataOffersToInsert().size());
- changes.setNumOffersUpdated(patch.getDataOffersToUpdate().size());
- changes.setNumOffersDeleted(patch.getDataOffersToDelete().size());
- dataOfferPatchApplier.writeDataOfferPatch(dsl, patch);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/FetchedCatalogBuilder.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/FetchedCatalogBuilder.java
deleted file mode 100644
index 64d850281..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/FetchedCatalogBuilder.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
-
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedCatalog;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedContractOffer;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
-import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
-import de.sovity.edc.utils.JsonUtils;
-import de.sovity.edc.utils.catalog.model.DspCatalog;
-import de.sovity.edc.utils.catalog.model.DspContractOffer;
-import de.sovity.edc.utils.catalog.model.DspDataOffer;
-import jakarta.json.JsonObject;
-import lombok.RequiredArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.List;
-
-@RequiredArgsConstructor
-public class FetchedCatalogBuilder {
- private final AssetMapper assetMapper;
-
- public FetchedCatalog buildFetchedCatalog(DspCatalog catalog) {
- var participantId = catalog.getParticipantId();
-
- var fetchedDataOffers = catalog.getDataOffers().stream()
- .map(dspDataOffer -> buildFetchedDataOffer(dspDataOffer, participantId))
- .toList();
-
- var fetchedCatalog = new FetchedCatalog();
- fetchedCatalog.setParticipantId(participantId);
- fetchedCatalog.setDataOffers(fetchedDataOffers);
-
- return fetchedCatalog;
- }
-
- @NotNull
- private FetchedDataOffer buildFetchedDataOffer(DspDataOffer dspDataOffer, String participantId) {
- var assetJsonLd = assetMapper.buildAssetJsonLdFromDatasetProperties(dspDataOffer.getAssetPropertiesJsonLd());
-
- var fetchedDataOffer = new FetchedDataOffer();
- setAssetMetadata(fetchedDataOffer, assetJsonLd, participantId);
- fetchedDataOffer.setContractOffers(buildFetchedContractOffers(dspDataOffer.getContractOffers()));
- return fetchedDataOffer;
- }
-
- @NotNull
- private List buildFetchedContractOffers(List offers) {
- return offers.stream()
- .map(this::buildFetchedContractOffer)
- .toList();
- }
-
- @NotNull
- private FetchedContractOffer buildFetchedContractOffer(DspContractOffer offer) {
- var contractOffer = new FetchedContractOffer();
- contractOffer.setContractOfferId(offer.getContractOfferId());
- contractOffer.setPolicyJson(JsonUtils.toJson(offer.getPolicyJsonLd()));
- return contractOffer;
- }
-
- /**
- * This method was extract so tests could re-use the logic of assetJsonLd -> fetchedDataOffer -> dataOfferRecord
- *
- * @param fetchedDataOffer fetchedDataOffer
- * @param assetJsonLd assetJsonLd
- */
- public void setAssetMetadata(FetchedDataOffer fetchedDataOffer, JsonObject assetJsonLd, String participantId) {
- var uiAsset = assetMapper.buildUiAsset(assetJsonLd, "http://if-you-see-this-this-is-a-bug", participantId);
- fetchedDataOffer.setAssetId(uiAsset.getAssetId());
- fetchedDataOffer.setAssetJsonLd(JsonUtils.toJson(assetJsonLd));
-
- // Most of these fields are extracted so our DB does not need to
- // semantically interpret JSON-LD when sorting, searching and filtering
- fetchedDataOffer.setAssetTitle(uiAsset.getTitle());
- fetchedDataOffer.setDescription(uiAsset.getDescription());
- fetchedDataOffer.setCuratorOrganizationName(uiAsset.getCreatorOrganizationName());
-
- fetchedDataOffer.setDataCategory(uiAsset.getDataCategory());
- fetchedDataOffer.setDataSubcategory(uiAsset.getDataSubcategory());
- fetchedDataOffer.setDataModel(uiAsset.getDataModel());
- fetchedDataOffer.setGeoReferenceMethod(uiAsset.getGeoReferenceMethod());
- fetchedDataOffer.setTransportMode(uiAsset.getTransportMode());
- fetchedDataOffer.setKeywords(uiAsset.getKeywords());
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/DataOfferPatch.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/DataOfferPatch.java
deleted file mode 100644
index a7a2b49bd..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/DataOfferPatch.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers.model;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ContractOfferRecord;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.DataOfferRecord;
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.experimental.FieldDefaults;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Contains planned DB Row changes to be applied as batch.
- */
-@Getter
-@Setter
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DataOfferPatch {
- List dataOffersToInsert = new ArrayList<>();
- List dataOffersToUpdate = new ArrayList<>();
- List dataOffersToDelete = new ArrayList<>();
-
- List contractOffersToInsert = new ArrayList<>();
- List contractOffersToUpdate = new ArrayList<>();
- List contractOffersToDelete = new ArrayList<>();
-
- public void insertDataOffer(DataOfferRecord offer) {
- dataOffersToInsert.add(offer);
- }
-
- public void updateDataOffer(DataOfferRecord offer) {
- dataOffersToUpdate.add(offer);
- }
-
- public void deleteDataOffer(DataOfferRecord offer) {
- dataOffersToDelete.add(offer);
- }
-
- public void insertContractOffer(ContractOfferRecord offer) {
- contractOffersToInsert.add(offer);
- }
-
- public void updateContractOffer(ContractOfferRecord offer) {
- contractOffersToUpdate.add(offer);
- }
-
- public void deleteContractOffer(ContractOfferRecord offer) {
- contractOffersToDelete.add(offer);
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/MdsIdUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/MdsIdUtils.java
deleted file mode 100644
index a832e20b6..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/MdsIdUtils.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.utils;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class MdsIdUtils {
- public static String getMdsIdFromParticipantId(String participantId) {
- if (participantId == null || !participantId.matches("^MDSL[A-Za-z0-9]+\\.C[A-Za-z0-9]+")) {
- return null;
- }
-
- return participantId.split("\\.")[0];
- }
-
- public static Boolean isValidMdsId(String mdsId) {
- return mdsId != null && mdsId.matches("^MDSL[A-Za-z0-9]+");
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/StreamUtils2.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/StreamUtils2.java
deleted file mode 100644
index 8a5701286..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/StreamUtils2.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.utils;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-
-import java.util.HashSet;
-import java.util.function.Function;
-import java.util.function.Predicate;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class StreamUtils2 {
-
- /**
- * Returns a predicate that filters out all elements that have the same key as a previous element.
- *
- * @param keyFn key extractor
- * @param item type
- * @param key type
- * @return predicate to be used in {@link java.util.stream.Stream#filter(Predicate)}
- */
- public static Predicate distinctByKey(Function keyFn) {
- var keys = new HashSet<>();
- return t -> keys.add(keyFn.apply(t));
- }
-}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/UrlUtils.java b/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/UrlUtils.java
deleted file mode 100644
index ae720063b..000000000
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/utils/UrlUtils.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.utils;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class UrlUtils {
-
- public static boolean isValidUrl(String url) {
- try {
- new URL(url).toURI();
- return true;
- } catch (MalformedURLException | URISyntaxException e) {
- return false;
- }
- }
-}
diff --git a/extensions/broker-server/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/broker-server/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension
deleted file mode 100644
index 80d56e9c3..000000000
--- a/extensions/broker-server/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension
+++ /dev/null
@@ -1 +0,0 @@
-de.sovity.edc.ext.brokerserver.BrokerServerExtension
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestAsset.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestAsset.java
deleted file mode 100644
index 5680a20c1..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestAsset.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.DataOfferRecord;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdParser;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.AssetJsonLdUtils;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.EdcPropertyUtils;
-import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder;
-import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.DataSourceMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper;
-import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper;
-import de.sovity.edc.ext.wrapper.api.common.model.DataSourceType;
-import de.sovity.edc.ext.wrapper.api.common.model.UiAssetCreateRequest;
-import de.sovity.edc.ext.wrapper.api.common.model.UiDataSource;
-import de.sovity.edc.ext.wrapper.api.common.model.UiDataSourceHttpData;
-import jakarta.json.JsonObject;
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import org.jetbrains.annotations.NotNull;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class TestAsset {
-
- public static JsonObject getAssetJsonLd(String assetId) {
- return getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id(assetId)
- .build()
- );
- }
-
- public static JsonObject getAssetJsonLd(UiAssetCreateRequest request) {
- var dataSource = UiDataSource.builder()
- .type(DataSourceType.HTTP_DATA)
- .httpData(UiDataSourceHttpData.builder()
- .baseUrl("https://example.com")
- .build())
- .build();
- var withDataSource = request.toBuilder().dataSource(dataSource).build();
- return buildAssetJsonLdBuilder().createAssetJsonLd(withDataSource, "orgName");
- }
-
- /**
- * Sets assetJsonLd and other extracted fields.
- *
- * This method keeps our tests consistent if we change the extracted fields.
- *
- * @param dataOfferRecord data offer record to be updated
- * @param assetJsonLd asset json ld
- * @param participantId required because the organization name will default to the participant id if unset
- */
- public static void setDataOfferAssetMetadata(DataOfferRecord dataOfferRecord, JsonObject assetJsonLd, String participantId) {
- // We trickily use the real code to update all the extracted values from the asset JSON-LD
- var fetchedCatalogBuilder = BrokerServerExtensionContext.instance.fetchedCatalogBuilder();
- var dataOfferRecordUpdater = BrokerServerExtensionContext.instance.dataOfferRecordUpdater();
-
- var fetchedDataOffer = new FetchedDataOffer();
- fetchedCatalogBuilder.setAssetMetadata(fetchedDataOffer, assetJsonLd, participantId);
-
- dataOfferRecord.setAssetId(fetchedDataOffer.getAssetId());
- dataOfferRecordUpdater.updateDataOffer(dataOfferRecord, fetchedDataOffer, false);
- }
-
- public static AssetJsonLdBuilder buildAssetJsonLdBuilder() {
- return new AssetJsonLdBuilder(
- new DataSourceMapper(
- new EdcPropertyUtils(),
- new HttpDataSourceMapper(new HttpHeaderMapper())
- ),
- buildAssetJsonLdParser(),
- new AssetEditRequestMapper()
- );
- }
-
- @NotNull
- private static AssetJsonLdParser buildAssetJsonLdParser() {
- return new AssetJsonLdParser(
- new AssetJsonLdUtils(),
- new ShortDescriptionBuilder(),
- "https://my-connector"::equals
- );
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestPolicy.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestPolicy.java
deleted file mode 100644
index 3087968d9..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestPolicy.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import de.sovity.edc.ext.brokerserver.client.gen.JSON;
-import de.sovity.edc.ext.wrapper.api.common.model.OperatorDto;
-import de.sovity.edc.ext.wrapper.api.common.model.UiPolicyConstraint;
-import de.sovity.edc.ext.wrapper.api.common.model.UiPolicyCreateRequest;
-import de.sovity.edc.ext.wrapper.api.common.model.UiPolicyLiteral;
-import de.sovity.edc.ext.wrapper.api.common.model.UiPolicyLiteralType;
-import de.sovity.edc.utils.JsonUtils;
-import org.jooq.JSONB;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-
-public class TestPolicy {
- private static OffsetDateTime today = OffsetDateTime.now();
-
- public static UiPolicyConstraint createAfterYesterdayConstraint() {
- return UiPolicyConstraint.builder()
- .left("POLICY_EVALUATION_TIME")
- .operator(OperatorDto.GT)
- .right(UiPolicyLiteral.builder()
- .type(UiPolicyLiteralType.STRING)
- .value(today.minusDays(1).toString())
- .build())
- .build();
- }
-
- public static de.sovity.edc.client.gen.model.UiPolicyCreateRequest createAfterYesterdayPolicyEdcGen() {
- return jsonCast(createAfterYesterdayPolicy(), de.sovity.edc.client.gen.model.UiPolicyCreateRequest.class);
- }
-
- private static R jsonCast(T obj, Class clazz) {
- return JSON.deserialize(JSON.serialize(obj), clazz);
- }
-
- public static UiPolicyCreateRequest createAfterYesterdayPolicy() {
- return UiPolicyCreateRequest.builder()
- .constraints(List.of(createAfterYesterdayConstraint()))
- .build();
- }
-
- public static JSONB createAfterYesterdayPolicyJson() {
- var createRequest = TestPolicy.createAfterYesterdayPolicy();
- return getPolicyJsonLd(createRequest);
- }
-
- /**
- * This method only works in integration tests, because it depends on the broker server extension context.
- */
- public static JSONB getPolicyJsonLd(UiPolicyCreateRequest createRequest) {
- var policyMapper = BrokerServerExtensionContext.instance.policyMapper();
- var jsonLd = policyMapper.buildPolicyJsonLd(policyMapper.buildPolicy(createRequest));
- return JSONB.jsonb(JsonUtils.toJson(jsonLd));
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestUtils.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestUtils.java
deleted file mode 100644
index 54a2218d9..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/TestUtils.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver;
-
-import de.sovity.edc.ext.brokerserver.client.BrokerServerClient;
-import de.sovity.edc.ext.brokerserver.client.gen.ApiException;
-import de.sovity.edc.ext.brokerserver.db.PostgresFlywayExtension;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import org.assertj.core.api.ThrowableAssert;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.eclipse.edc.junit.testfixtures.TestUtils.getFreePort;
-
-public class TestUtils {
- private static final int MANAGEMENT_PORT = getFreePort();
- private static final int PROTOCOL_PORT = getFreePort();
- private static final String MANAGEMENT_PATH = "/api/management";
- private static final String PROTOCOL_PATH = "/api/dsp";
- public static final String MANAGEMENT_API_KEY = "123456";
- public static final String MANAGEMENT_ENDPOINT = "http://localhost:" + MANAGEMENT_PORT + MANAGEMENT_PATH;
- public static final String ADMIN_API_KEY = "123456";
-
-
- public static final String PROTOCOL_HOST = "http://localhost:" + PROTOCOL_PORT;
- public static final String PROTOCOL_ENDPOINT = PROTOCOL_HOST + PROTOCOL_PATH;
- public static final String PARTICIPANT_ID = "MDSL1234ZZ.C4321AA";
- public static final String CURATOR_NAME = "My Org";
-
- @NotNull
- public static Map createConfiguration(
- TestDatabase testDatabase,
- Map additionalConfigProperties
- ) {
- Map config = new HashMap<>();
-
- config.put("web.http.port", String.valueOf(getFreePort()));
- config.put("web.http.path", "/api");
- config.put("web.http.management.port", String.valueOf(MANAGEMENT_PORT));
- config.put("web.http.management.path", MANAGEMENT_PATH);
- config.put("web.http.protocol.port", String.valueOf(PROTOCOL_PORT));
- config.put("web.http.protocol.path", PROTOCOL_PATH);
- config.put("edc.api.auth.key", MANAGEMENT_API_KEY);
- config.put("edc.dsp.callback.address", PROTOCOL_ENDPOINT);
- config.put("edc.oauth.provider.audience", "idsc:IDS_CONNECTORS_ALL");
-
- config.put("edc.participant.id", PARTICIPANT_ID);
- config.put("my.edc.participant.id", PARTICIPANT_ID);
- config.put("my.edc.title", "My Connector");
- config.put("my.edc.description", "My Connector Description");
- config.put("my.edc.curator.url", "https://connector.my-org");
- config.put("my.edc.curator.name", CURATOR_NAME);
- config.put("my.edc.maintainer.url", "https://maintainer-org");
- config.put("my.edc.maintainer.name", "Maintainer Org");
-
- config.put(PostgresFlywayExtension.JDBC_URL, testDatabase.getJdbcUrl());
- config.put(PostgresFlywayExtension.JDBC_USER, testDatabase.getJdbcUser());
- config.put(PostgresFlywayExtension.JDBC_PASSWORD, testDatabase.getJdbcPassword());
- config.put(PostgresFlywayExtension.DB_CONNECTION_POOL_SIZE, "20");
- config.put(PostgresFlywayExtension.DB_CONNECTION_TIMEOUT_IN_MS, "3000");
- config.put(PostgresFlywayExtension.FLYWAY_CLEAN_ENABLE, "true");
- config.put(PostgresFlywayExtension.FLYWAY_CLEAN, "true");
- config.put(BrokerServerExtension.NUM_THREADS, "0");
- config.put(BrokerServerExtension.ADMIN_API_KEY, ADMIN_API_KEY);
- config.putAll(getCoreEdcJdbcConfig(testDatabase));
- config.putAll(additionalConfigProperties);
- return config;
- }
-
- private static Map getCoreEdcJdbcConfig(TestDatabase testDatabase) {
- Map config = new HashMap<>();
- List.of("asset",
- "contractdefinition",
- "contractnegotiation",
- "policy",
- "transferprocess",
- "dataplaneinstance"
- ).forEach(it -> {
- config.put("edc.datasource.%s.name".formatted(it), it);
- config.put("edc.datasource.%s.url".formatted(it), testDatabase.getJdbcUrl());
- config.put("edc.datasource.%s.user".formatted(it), testDatabase.getJdbcUser());
- config.put("edc.datasource.%s.password".formatted(it), testDatabase.getJdbcPassword());
- });
- return config;
- }
-
- public static BrokerServerClient brokerServerClient() {
- return BrokerServerClient.builder()
- .managementApiUrl(TestUtils.MANAGEMENT_ENDPOINT)
- .managementApiKey(TestUtils.MANAGEMENT_API_KEY)
- .build();
- }
-
-
- public static void assertIs401(ThrowableAssert.ThrowingCallable callable) {
- assertThatThrownBy(callable)
- .isInstanceOf(ApiException.class)
- .satisfies(ex -> {
- var apiException = (ApiException) ex;
- assertThat(apiException.getCode()).isEqualTo(401);
- });
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/dao/utils/LikeUtilsTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/dao/utils/LikeUtilsTest.java
deleted file mode 100644
index 13fe324ec..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/dao/utils/LikeUtilsTest.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.dao.utils;
-
-import org.junit.jupiter.api.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class LikeUtilsTest {
- @Test
- void escape() {
- assertThat(LikeUtils.escape("a\\b_c%d")).isEqualTo("a\\\\b\\_c\\%d");
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/FlywayTestUtils.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/FlywayTestUtils.java
deleted file mode 100644
index 059f81474..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/FlywayTestUtils.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.db;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import org.eclipse.edc.spi.monitor.ConsoleMonitor;
-import org.eclipse.edc.spi.system.configuration.Config;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class FlywayTestUtils {
-
- public static void migrate(TestDatabase testDatabase) {
- var monitor = new ConsoleMonitor();
- var config = mock(Config.class);
- when(config.getBoolean(eq(PostgresFlywayExtension.FLYWAY_CLEAN_ENABLE), any())).thenReturn(true);
- when(config.getBoolean(eq(PostgresFlywayExtension.FLYWAY_CLEAN), any())).thenReturn(true);
-
- var flywayFactory = new FlywayFactory(config);
- var dataSource = testDatabase.getDataSource();
- var flyway = flywayFactory.setupFlyway(dataSource);
- var flywayMigrator = new FlywayMigrator(flyway, config, monitor);
- flywayMigrator.migrateAndRepair();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabase.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabase.java
deleted file mode 100644
index e593df41c..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabase.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.db;
-
-import de.sovity.edc.ext.brokerserver.db.utils.JdbcCredentials;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.extension.AfterAllCallback;
-import org.junit.jupiter.api.extension.BeforeAllCallback;
-
-import javax.sql.DataSource;
-import java.util.function.Consumer;
-
-public interface TestDatabase extends BeforeAllCallback, AfterAllCallback {
- String getJdbcUrl();
-
- String getJdbcUser();
-
- String getJdbcPassword();
-
- /**
- * New {@link DslContextFactory} from the test database's credentials
- *
- * @return {@link DslContextFactory}
- */
- default DslContextFactory getDslContextFactory() {
- var dataSource = getDataSource();
- return new DslContextFactory(dataSource);
- }
-
- /**
- * Returns a {@link DataSource} to the test database
- *
- * @return {@link DataSource}
- */
- default DataSource getDataSource() {
- var jdbcCredentials = new JdbcCredentials(getJdbcUrl(), getJdbcUser(), getJdbcPassword());
- return DataSourceFactory.newDataSource(jdbcCredentials, 20, 30_000);
- }
-
- /**
- * Runs given code within a test transaction.
- *
- * Globally hijacks all {@link DslContextFactory}s to use this test transaction.
- *
- * @param code code to run within the test transaction
- */
- default void testTransaction(Consumer code) {
- try {
- getDslContextFactory().transaction(dsl -> DslContextFactoryHijacker.withParentDslContext(dsl, () -> {
- code.accept(dsl);
- throw new TestDatabaseCancelTransactionException();
- }));
- } catch (TestDatabaseCancelTransactionException e) {
- // Ignore
- }
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseFactory.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseFactory.java
deleted file mode 100644
index cc272fcb8..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseFactory.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.db;
-
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class TestDatabaseFactory {
-
- /**
- * Returns a JUnit 5 Extension that launches a testcontainer.
- *
- * @return {@link TestDatabase}
- */
- public static TestDatabase getTestDatabase() {
- return new TestDatabaseViaTestcontainers();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseViaEnv.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseViaEnv.java
deleted file mode 100644
index ad2a5c5f9..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseViaEnv.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.db;
-
-import org.apache.commons.lang3.Validate;
-import org.junit.jupiter.api.extension.ExtensionContext;
-
-public class TestDatabaseViaEnv implements TestDatabase {
- public static final String SKIP_TESTCONTAINERS = "SKIP_TESTCONTAINERS";
- public static final String TEST_POSTGRES_JDBC_URL = "TEST_POSTGRES_JDBC_URL";
- public static final String TEST_POSTGRES_JDBC_USER = "TEST_POSTGRES_JDBC_USER";
- public static final String TEST_POSTGRES_JDBC_PASSWORD = "TEST_POSTGRES_JDBC_PASSWORD";
-
- @Override
- public void afterAll(ExtensionContext context) throws Exception {
-
- }
-
- @Override
- public void beforeAll(ExtensionContext context) throws Exception {
-
- }
-
- public String getJdbcUrl() {
- return getRequiredEnv(TEST_POSTGRES_JDBC_URL);
- }
-
- public String getJdbcUser() {
- return getRequiredEnv(TEST_POSTGRES_JDBC_USER);
- }
-
- public String getJdbcPassword() {
- return getRequiredEnv(TEST_POSTGRES_JDBC_PASSWORD);
- }
-
- private static String getRequiredEnv(String name) {
- String value = System.getenv(name);
- Validate.notBlank(value, "Need env var %s since %s is true", name, SKIP_TESTCONTAINERS);
- return value;
- }
-
- public static boolean isSkipTestcontainers() {
- return "true".equals(System.getenv(SKIP_TESTCONTAINERS));
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseViaTestcontainers.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseViaTestcontainers.java
deleted file mode 100644
index fd3d1caf0..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/db/TestDatabaseViaTestcontainers.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.db;
-
-import de.sovity.edc.utils.versions.GradleVersions;
-import org.junit.jupiter.api.extension.ExtensionContext;
-import org.testcontainers.containers.PostgreSQLContainer;
-
-public class TestDatabaseViaTestcontainers implements TestDatabase {
- private PostgreSQLContainer container = new PostgreSQLContainer<>(GradleVersions.POSTGRES_IMAGE_TAG)
- .withUsername("edc")
- .withPassword("edc");
-
- @Override
- public void afterAll(ExtensionContext context) throws Exception {
- container.stop();
- }
-
- @Override
- public void beforeAll(ExtensionContext context) throws Exception {
- container.start();
- }
-
- public String getJdbcUrl() {
- return container.getJdbcUrl();
- }
-
- public String getJdbcUser() {
- return container.getUsername();
- }
-
- public String getJdbcPassword() {
- return container.getPassword();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AddConnectorsApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AddConnectorsApiTest.java
deleted file mode 100644
index 058882217..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AddConnectorsApiTest.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.TestUtils;
-import de.sovity.edc.ext.brokerserver.client.BrokerServerClient;
-import de.sovity.edc.ext.brokerserver.client.gen.model.AddedConnector;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorCreationRequest;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorListEntry;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorPageQuery;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestUtils.ADMIN_API_KEY;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class AddConnectorsApiTest {
- BrokerServerClient client;
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
- client = brokerServerClient();
- }
-
- @Test
- void testAddConnectors() {
- TEST_DATABASE.testTransaction(dsl -> {
- client.brokerServerApi().addConnectors(ADMIN_API_KEY, List.of());
-
- client.brokerServerApi().addConnectors(ADMIN_API_KEY, Arrays.asList(
- null,
- "",
- " ",
- "\t",
- "http://a",
- "http://b"
- ));
-
- assertThat(client.brokerServerApi().connectorPage(new ConnectorPageQuery()).getConnectors())
- .extracting(ConnectorListEntry::getEndpoint)
- .containsExactlyInAnyOrder("http://a", "http://b");
-
- client.brokerServerApi().addConnectors(ADMIN_API_KEY, Arrays.asList(
- "http://b",
- " http://b\r\n",
- "http://c"
- ));
-
- assertThat(client.brokerServerApi().connectorPage(new ConnectorPageQuery()).getConnectors())
- .extracting(ConnectorListEntry::getEndpoint)
- .containsExactlyInAnyOrder("http://a", "http://b", "http://c");
- });
- }
-
- @Test
- void testAddConnectorsWithMdsIds() {
- TEST_DATABASE.testTransaction(dsl -> {
- client.brokerServerApi().addConnectorsWithMdsIds(ADMIN_API_KEY, new ConnectorCreationRequest(List.of()));
-
- client.brokerServerApi().addConnectorsWithMdsIds(ADMIN_API_KEY, new ConnectorCreationRequest(Arrays.asList(
- new AddedConnector(
- null,
- "MDSL1234"
- ),
- new AddedConnector(
- "",
- "MDSL1234"
- ),
- new AddedConnector(
- " ",
- "MDSL1234"
- ),
- new AddedConnector(
- "\t",
- "MDSL1234"
- ),
- new AddedConnector(
- "http://a",
- "MDSL1234"
- ),
- new AddedConnector(
- " http://b\r\n",
- "MDSL1234"
- ),
- new AddedConnector(
- "http://c",
- null
- ),
- new AddedConnector(
- "http://d",
- ""
- ),
- new AddedConnector(
- "http://e",
- " "
- ),
- new AddedConnector(
- "http://f",
- "\t"
- )
- )));
-
- assertThat(client.brokerServerApi().connectorPage(new ConnectorPageQuery()).getConnectors())
- .extracting(ConnectorListEntry::getEndpoint)
- .containsExactlyInAnyOrder("http://a", "http://b");
-
- client.brokerServerApi().addConnectorsWithMdsIds(ADMIN_API_KEY, new ConnectorCreationRequest(Arrays.asList(
- new AddedConnector(
- "http://b",
- "MDSL1234"
- ),
- new AddedConnector(
- " http://b\r\n",
- "MDSL1234"
- ),
- new AddedConnector(
- "http://c",
- "MDSL1234"
- )
- )));
-
- assertThat(client.brokerServerApi().connectorPage(new ConnectorPageQuery()).getConnectors())
- .extracting(ConnectorListEntry::getEndpoint)
- .containsExactlyInAnyOrder("http://a", "http://b", "http://c");
- });
- }
-
- @Test
- void testAddWrongApiKey() {
- TEST_DATABASE.testTransaction(dsl -> TestUtils.assertIs401(() ->
- client.brokerServerApi().addConnectors("wrong-api-key", List.of())));
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorMetadataApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorMetadataApiTest.java
deleted file mode 100644
index f8c230ab6..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalConnectorMetadataApiTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.TestPolicy;
-import de.sovity.edc.ext.brokerserver.client.gen.model.AuthorityPortalConnectorInfo;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestAsset.getAssetJsonLd;
-import static de.sovity.edc.ext.brokerserver.TestAsset.setDataOfferAssetMetadata;
-import static de.sovity.edc.ext.brokerserver.TestUtils.ADMIN_API_KEY;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class AuthorityPortalConnectorMetadataApiTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
- }
-
- @Test
- void testConnectorMetadataByEndpoints() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var now = OffsetDateTime.now().withNano(0);
-
- createConnector(dsl, now, 1);
- createDataOffer(dsl, now, 1, 1);
- createDataOffer(dsl, now, 1, 2);
-
- createConnector(dsl, now, 2);
- createDataOffer(dsl, now, 2, 1);
-
- createConnector(dsl, now, 3);
- createDataOffer(dsl, now, 3, 1);
-
- createConnector(dsl, now, 4);
-
- // act
- var actual = brokerServerClient().brokerServerApi().getConnectorMetadata(
- ADMIN_API_KEY,
- Arrays.asList(
- getEndpoint(1),
- getEndpoint(1), // having this twice should not crash the query
- getEndpoint(2),
- getEndpoint(4),
- getEndpoint(5) // having this not existing should not crash the query
- ));
-
- // assert
- var first = forConnector(actual, 1);
- assertThat(first.getParticipantId()).isEqualTo("my-connector");
- assertThat(first.getDataOfferCount()).isEqualTo(2);
- assertThat(first.getOnlineStatus()).isEqualTo(AuthorityPortalConnectorInfo.OnlineStatusEnum.ONLINE);
- assertThat(first.getOfflineSinceOrLastUpdatedAt()).isEqualTo(now);
- var second = forConnector(actual, 2);
- assertThat(second.getDataOfferCount()).isEqualTo(1);
- assertThat(second.getOnlineStatus()).isEqualTo(AuthorityPortalConnectorInfo.OnlineStatusEnum.ONLINE);
- assertThat(second.getOfflineSinceOrLastUpdatedAt()).isEqualTo(now);
- var fourth = forConnector(actual, 4);
- assertThat(fourth.getDataOfferCount()).isEqualTo(0);
- assertThat(fourth.getOnlineStatus()).isEqualTo(AuthorityPortalConnectorInfo.OnlineStatusEnum.ONLINE);
- assertThat(fourth.getOfflineSinceOrLastUpdatedAt()).isEqualTo(now);
- });
- }
-
- private AuthorityPortalConnectorInfo forConnector(List actual, int iConnector) {
- return actual.stream()
- .filter(connectorMetadata ->
- getEndpoint(iConnector).equals(connectorMetadata.getConnectorEndpoint())
- )
- .findFirst()
- .orElseThrow();
- }
-
- private void createConnector(DSLContext dsl, OffsetDateTime now, int iConnector) {
- var connector = dsl.newRecord(Tables.CONNECTOR);
- connector.setParticipantId("my-connector");
- connector.setEndpoint(getEndpoint(iConnector));
- connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
- connector.setCreatedAt(now.minusDays(1));
- connector.setLastRefreshAttemptAt(now);
- connector.setLastSuccessfulRefreshAt(now);
- connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
- connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
- connector.insert();
- }
-
- private String getEndpoint(int iConnector) {
- return "https://connector-%d".formatted(iConnector);
- }
-
- private void createDataOffer(DSLContext dsl, OffsetDateTime now, int iConnector, int iDataOffer) {
- var connectorEndpoint = getEndpoint(iConnector);
- var assetJsonLd = getAssetJsonLd("my-asset-%d".formatted(iDataOffer));
-
- var dataOffer = dsl.newRecord(Tables.DATA_OFFER);
- setDataOfferAssetMetadata(dataOffer, assetJsonLd, "my-participant-id");
- dataOffer.setConnectorEndpoint(connectorEndpoint);
- dataOffer.setCreatedAt(now.minusDays(5));
- dataOffer.setUpdatedAt(now);
- dataOffer.insert();
-
- var contractOffer = dsl.newRecord(Tables.CONTRACT_OFFER);
- contractOffer.setContractOfferId("my-contract-offer-1");
- contractOffer.setConnectorEndpoint(connectorEndpoint);
- contractOffer.setAssetId(dataOffer.getAssetId());
- contractOffer.setCreatedAt(now.minusDays(5));
- contractOffer.setUpdatedAt(now);
- contractOffer.setPolicy(TestPolicy.createAfterYesterdayPolicyJson());
- contractOffer.insert();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalDataOfferApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalDataOfferApiTest.java
deleted file mode 100644
index 9cd29b55f..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalDataOfferApiTest.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.TestPolicy;
-import de.sovity.edc.ext.brokerserver.client.gen.ApiException;
-import de.sovity.edc.ext.brokerserver.client.gen.model.AuthorityPortalConnectorDataOfferInfo;
-import de.sovity.edc.ext.brokerserver.client.gen.model.AuthorityPortalConnectorInfo;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestAsset.getAssetJsonLd;
-import static de.sovity.edc.ext.brokerserver.TestAsset.setDataOfferAssetMetadata;
-import static de.sovity.edc.ext.brokerserver.TestUtils.ADMIN_API_KEY;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertThrows;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class AuthorityPortalDataOfferApiTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
- }
-
- @Test
- void testAuthenticationOfEndpoints() {
- TEST_DATABASE.testTransaction(dsl -> {
- var code = assertThrows(ApiException.class, () -> brokerServerClient().brokerServerApi().getConnectorDataOffers(
- ADMIN_API_KEY + "invalid",
- List.of())).getCode();
- assertThat(code).isEqualTo(401);
- });
- }
-
- @Test
- void testConnectorMetadataByEndpoints() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var now = OffsetDateTime.now().withNano(0);
-
- createConnector(dsl, now, 1);
- createDataOffer(dsl, now, 1, 1);
- createDataOffer(dsl, now, 1, 2);
-
- createConnector(dsl, now, 2);
- createDataOffer(dsl, now, 2, 1);
-
- createConnector(dsl, now, 4);
-
- // act
- var actual = brokerServerClient().brokerServerApi().getConnectorDataOffers(
- ADMIN_API_KEY,
- Arrays.asList(
- getEndpoint(1),
- getEndpoint(2),
- getEndpoint(4)
- ));
-
- // assert
- // connector 1 with two data offer
- var connector1 = forConnector(actual, 1);
- assertThat(connector1.getConnectorEndpoint()).isEqualTo(getEndpoint(1));
- assertThat(connector1.getParticipantId()).isEqualTo("my-connector");
- assertThat(connector1.getOnlineStatus().getValue()).isEqualTo(AuthorityPortalConnectorInfo.OnlineStatusEnum.ONLINE.getValue());
- assertThat(connector1.getOfflineSinceOrLastUpdatedAt()).isEqualTo(now);
- assertThat(connector1.getDataOffers().size()).isEqualTo(2);
- var connector1asset1 = connector1.getDataOffers().stream().filter(dataOffer -> dataOffer.getDataOfferId().equals(getAssetId(1))).findFirst().orElseThrow();
- assertThat(connector1asset1.getDataOfferId()).isEqualTo("my-asset-1");
- assertThat(connector1asset1.getDataOfferName()).isEqualTo("my-asset-1");
- var connector1asset2 = connector1.getDataOffers().stream().filter(dataOffer -> dataOffer.getDataOfferId().equals(getAssetId(2))).findFirst().orElseThrow();
- assertThat(connector1asset2.getDataOfferId()).isEqualTo("my-asset-2");
- assertThat(connector1asset2.getDataOfferName()).isEqualTo("my-asset-2");
-
- // connector 2 with one data offer
- var connector2 = forConnector(actual, 2);
- assertThat(connector2.getConnectorEndpoint()).isEqualTo(getEndpoint(2));
- assertThat(connector2.getParticipantId()).isEqualTo("my-connector");
- assertThat(connector2.getOnlineStatus().getValue()).isEqualTo(AuthorityPortalConnectorInfo.OnlineStatusEnum.ONLINE.getValue());
- assertThat(connector2.getOfflineSinceOrLastUpdatedAt()).isEqualTo(now);
- assertThat(connector2.getDataOffers().size()).isEqualTo(1);
- var connector2asset1 = connector2.getDataOffers().stream().filter(dataOffer -> dataOffer.getDataOfferId().equals(getAssetId(1))).findFirst().orElseThrow();
- assertThat(connector2asset1.getDataOfferId()).isEqualTo("my-asset-1");
- assertThat(connector2asset1.getDataOfferName()).isEqualTo("my-asset-1");
-
- // connector 4 without data offers
- var connector4 = forConnector(actual, 4);
- assertThat(connector4.getConnectorEndpoint()).isEqualTo(getEndpoint(4));
- assertThat(connector4.getParticipantId()).isEqualTo("my-connector");
- assertThat(connector4.getOnlineStatus().getValue()).isEqualTo(AuthorityPortalConnectorInfo.OnlineStatusEnum.ONLINE.getValue());
- assertThat(connector4.getOfflineSinceOrLastUpdatedAt()).isEqualTo(now);
- assertThat(connector4.getDataOffers().size()).isEqualTo(0);
- });
- }
-
- private AuthorityPortalConnectorDataOfferInfo forConnector(List actual, int iConnector) {
- return actual.stream()
- .filter(connectorMetadata ->
- getEndpoint(iConnector).equals(connectorMetadata.getConnectorEndpoint())
- )
- .findFirst()
- .orElseThrow();
- }
-
- private void createConnector(DSLContext dsl, OffsetDateTime now, int iConnector) {
- var connector = dsl.newRecord(Tables.CONNECTOR);
- connector.setParticipantId("my-connector");
- connector.setEndpoint(getEndpoint(iConnector));
- connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
- connector.setCreatedAt(now.minusDays(1));
- connector.setLastRefreshAttemptAt(now);
- connector.setLastSuccessfulRefreshAt(now);
- connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
- connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
- connector.insert();
- }
-
- private String getEndpoint(int iConnector) {
- return "https://connector-%d".formatted(iConnector);
- }
-
- private String getAssetId(int iDataOffer) {
- return "my-asset-%d".formatted(iDataOffer);
- }
-
- private void createDataOffer(DSLContext dsl, OffsetDateTime now, int iConnector, int iDataOffer) {
- var connectorEndpoint = getEndpoint(iConnector);
- var assetJsonLd = getAssetJsonLd(getAssetId(iDataOffer));
-
- var dataOffer = dsl.newRecord(Tables.DATA_OFFER);
- setDataOfferAssetMetadata(dataOffer, assetJsonLd, "my-participant-id");
- dataOffer.setConnectorEndpoint(connectorEndpoint);
- dataOffer.setCreatedAt(now.minusDays(5));
- dataOffer.setUpdatedAt(now);
- dataOffer.insert();
-
- var contractOffer = dsl.newRecord(Tables.CONTRACT_OFFER);
- contractOffer.setContractOfferId("my-contract-offer-1");
- contractOffer.setConnectorEndpoint(connectorEndpoint);
- contractOffer.setAssetId(dataOffer.getAssetId());
- contractOffer.setCreatedAt(now.minusDays(5));
- contractOffer.setUpdatedAt(now);
- contractOffer.setPolicy(TestPolicy.createAfterYesterdayPolicyJson());
- contractOffer.insert();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiTest.java
deleted file mode 100644
index fa907770b..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/AuthorityPortalOrganizationMetadataApiTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.client.gen.model.AuthorityPortalOrganizationMetadata;
-import de.sovity.edc.ext.brokerserver.client.gen.model.AuthorityPortalOrganizationMetadataRequest;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.OrganizationMetadataRecord;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestUtils.ADMIN_API_KEY;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class AuthorityPortalOrganizationMetadataApiTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
- }
-
- @Test
- void testSetOrganizationMetadata() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- createOrgMetadataInDb(dsl, "MDSL1111AA", "Test Org A");
- createOrgMetadataInDb(dsl, "MDSL2222BB", "Test Org B");
- createOrgMetadataInDb(dsl, "MDSL3333CC", "Test Org C");
-
- // act
- var orgMetadataRequest = new AuthorityPortalOrganizationMetadataRequest();
- orgMetadataRequest.setOrganizations(List.of(
- buildOrgMetadataRequestEntry("MDSL2222BB", "Test Org B"),
- buildOrgMetadataRequestEntry("MDSL3333CC", "Test Org C new"),
- buildOrgMetadataRequestEntry("MDSL4444DD", "Test Org D")
- ));
-
- brokerServerClient().brokerServerApi().setOrganizationMetadata(
- ADMIN_API_KEY,
- orgMetadataRequest
- );
-
- // assert
- var orgMetadata = getOrgMetadataFromDb(dsl);
- assertThat(orgMetadata).hasSize(3);
- assertThat(orgMetadata).extracting(OrganizationMetadataRecord::getName).containsExactlyInAnyOrder("Test Org B", "Test Org C new", "Test Org D");
- });
- }
-
- @Test
- void testSetEmptyOrganizationMetadata() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- createOrgMetadataInDb(dsl, "MDSL1111AA", "Test Org A");
-
- // act
- var orgMetadataRequest = new AuthorityPortalOrganizationMetadataRequest();
- orgMetadataRequest.setOrganizations(List.of());
-
- brokerServerClient().brokerServerApi().setOrganizationMetadata(
- ADMIN_API_KEY,
- orgMetadataRequest
- );
-
- // assert
- var orgMetadata = getOrgMetadataFromDb(dsl);
- assertThat(orgMetadata).isEmpty();
- });
- }
-
- private void createOrgMetadataInDb(DSLContext dsl, String mdsId, String name) {
- var organizationMetadata = dsl.newRecord(Tables.ORGANIZATION_METADATA);
- organizationMetadata.setMdsId(mdsId);
- organizationMetadata.setName(name);
- organizationMetadata.insert();
- }
-
- private List getOrgMetadataFromDb(DSLContext dsl) {
- return dsl.selectFrom(Tables.ORGANIZATION_METADATA).fetch();
- }
-
- private AuthorityPortalOrganizationMetadata buildOrgMetadataRequestEntry(String mdsId, String name) {
- var orgMetadata = new AuthorityPortalOrganizationMetadata();
- orgMetadata.setMdsId(mdsId);
- orgMetadata.setName(name);
- return orgMetadata;
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiTest.java
deleted file mode 100644
index fb508c39d..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/CatalogApiTest.java
+++ /dev/null
@@ -1,708 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import de.sovity.edc.ext.brokerserver.BrokerServerExtension;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CatalogDataOffer;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CatalogPageQuery;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CatalogPageResult;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CnfFilterAttribute;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CnfFilterItem;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CnfFilterValue;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CnfFilterValueAttribute;
-import de.sovity.edc.ext.brokerserver.client.gen.model.DataOfferDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.client.gen.model.DataOfferDetailPageResult;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.wrapper.api.common.model.UiAssetCreateRequest;
-import jakarta.json.JsonObject;
-import lombok.SneakyThrows;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.eclipse.edc.policy.model.Policy;
-import org.eclipse.edc.policy.model.PolicyType;
-import org.jooq.DSLContext;
-import org.jooq.JSONB;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestAsset.getAssetJsonLd;
-import static de.sovity.edc.ext.brokerserver.TestAsset.setDataOfferAssetMetadata;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static java.util.stream.IntStream.range;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class CatalogApiTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of(
- BrokerServerExtension.CATALOG_PAGE_PAGE_SIZE, "10",
- BrokerServerExtension.DEFAULT_CONNECTOR_DATASPACE, "MDS",
- BrokerServerExtension.KNOWN_DATASPACE_CONNECTORS, "Example1=https://my-connector2/api/dsp,Example2=https://my-connector3/api/dsp"
- )));
- }
-
- @Test
- void testDataSpace_two_dataspaces_filter_for_one() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset")
- .title("My Asset")
- .build()
- );
-
- // Dataspace: MDS
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd);
-
- // Dataspace: Example1
- createConnector(dsl, today, "https://my-connector2/api/dsp");
- createDataOffer(dsl, today, "https://my-connector2/api/dsp", assetJsonLd);
-
- var query = new CatalogPageQuery();
- query.setFilter(new CnfFilterValue(List.of(
- new CnfFilterValueAttribute("dataSpace", List.of("Example1"))
- )));
-
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
- assertThat(result.getDataOffers()).hasSize(1);
-
- var dataOfferResult = result.getDataOffers().get(0);
- assertThat(dataOfferResult.getConnectorEndpoint()).isEqualTo("https://my-connector2/api/dsp");
- });
- }
-
- @Test
- void testConnectorEndpointFilter_two_connectors_filter_for_one() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset")
- .title("My Asset")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd);
-
- createConnector(dsl, today, "https://my-connector2/api/dsp");
- createDataOffer(dsl, today, "https://my-connector2/api/dsp", assetJsonLd);
-
- var query = new CatalogPageQuery();
- query.setFilter(new CnfFilterValue(List.of(
- new CnfFilterValueAttribute("connectorEndpoint", List.of("https://my-connector/api/dsp"))
- )));
-
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
- assertThat(result.getDataOffers()).extracting(CatalogDataOffer::getConnectorEndpoint).containsExactly("https://my-connector/api/dsp");
- });
- }
-
- @Test
- void test_available_filter_values_to_filter_by() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- createConnector(dsl, today, "https://my-connector/api/dsp"); // Dataspace: MDS
- createConnector(dsl, today, "https://my-connector2/api/dsp"); // Dataspace: Example1
- createConnector(dsl, today, "https://my-connector3/api/dsp"); // Dataspace: Example2
-
- var assetJsonLd1 = getAssetJsonLd("my-asset-1");
- var assetJsonLd2 = getAssetJsonLd("my-asset-2");
- var assetJsonLd3 = getAssetJsonLd("my-asset-3");
-
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd1); // Dataspace: MDS
- createDataOffer(dsl, today, "https://my-connector2/api/dsp", assetJsonLd1); // Dataspace: Example1
- createDataOffer(dsl, today, "https://my-connector2/api/dsp", assetJsonLd2); // Dataspace: Example1
- createDataOffer(dsl, today, "https://my-connector3/api/dsp", assetJsonLd3); // Dataspace: Example2
-
- // get all available filter values
- var result = brokerServerClient().brokerServerApi().catalogPage(new CatalogPageQuery());
-
- // assert that the filter values are correct
- var dataSpace = getAvailableFilter(result, "dataSpace");
- assertThat(dataSpace.getValues()).containsExactly(
- new CnfFilterItem("Example1", "Example1"),
- new CnfFilterItem("Example2", "Example2"),
- new CnfFilterItem("MDS", "MDS")
- );
- });
- }
-
- @Test
- void testDataOfferDetails() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset")
- .title("My Asset")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd);
-
- var result = brokerServerClient().brokerServerApi().catalogPage(new CatalogPageQuery());
- assertThat(result.getDataOffers()).hasSize(1);
-
- var dataOfferResult = result.getDataOffers().get(0);
- assertThat(dataOfferResult.getConnectorEndpoint()).isEqualTo("https://my-connector/api/dsp");
- assertThat(dataOfferResult.getConnectorOfflineSinceOrLastUpdatedAt()).isEqualTo(today);
- assertThat(dataOfferResult.getConnectorOnlineStatus()).isEqualTo(CatalogDataOffer.ConnectorOnlineStatusEnum.ONLINE);
- assertThat(dataOfferResult.getAssetId()).isEqualTo("my-asset");
- assertThat(dataOfferResult.getAsset().getAssetId()).isEqualTo("my-asset");
- assertThat(dataOfferResult.getAsset().getTitle()).isEqualTo("My Asset");
- assertThat(dataOfferResult.getCreatedAt()).isEqualTo(today.minusDays(5));
- });
- }
-
- /**
- * Tests against an issue where empty available filter values resulted in NULLs
- */
- @Test
- void testEmptyConnector() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
- createConnector(dsl, today, "https://my-connector/api/dsp");
-
- // act
- var result = brokerServerClient().brokerServerApi().catalogPage(new CatalogPageQuery());
-
- // assert
- assertThat(result.getDataOffers()).isEmpty();
- assertThat(result.getAvailableFilters().getFields()).isNotEmpty();
- assertThat(result.getAvailableSortings()).isNotEmpty();
-
- // the most important thing is that the above code ran through as it crashed before
- });
- }
-
- @Test
- void testAvailableFilters_noFilter() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd1 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-1")
- .dataCategory("my-category-1")
- .transportMode("MY-TRANSPORT-MODE-1")
- .dataSubcategory("MY-SUBCATEGORY-2")
- .dataModel("my-data-model")
- .geoReferenceMethod("my-geo-ref")
- .build()
- );
-
- var assetJsonLd2 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-2")
- .dataCategory("my-category-1")
- .transportMode("my-transport-mode-2")
- .dataSubcategory("MY-SUBCATEGORY-2")
- .build()
- );
-
- var assetJsonLd3 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-3")
- .dataCategory("my-category-1")
- .transportMode("MY-TRANSPORT-MODE-1")
- .dataSubcategory("my-subcategory-1")
- .build()
- );
-
- var assetJsonLd4 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-4")
- .dataCategory("my-category-1")
- .transportMode("")
- .build()
- );
-
- createOrganizationMetadata(dsl, "MDSL123456AA", "Test Org");
- createConnector(dsl, today, "https://my-connector/api/dsp", "MDSL123456AA");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd1);
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd2);
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd3);
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd4);
-
- var result = brokerServerClient().brokerServerApi().catalogPage(new CatalogPageQuery());
-
- assertThat(result.getAvailableFilters().getFields())
- .extracting(CnfFilterAttribute::getId)
- .containsExactly(
- "dataSpace",
- "dataCategory",
- "dataSubcategory",
- "dataModel",
- "transportMode",
- "geoReferenceMethod",
- "curatorOrganizationName",
- "curatorMdsId",
- "connectorEndpoint"
- );
-
- assertThat(result.getAvailableFilters().getFields())
- .extracting(CnfFilterAttribute::getTitle)
- .containsExactly(
- "Data Space",
- "Data Category",
- "Data Subcategory",
- "Data Model",
- "Transport Mode",
- "Geo Reference Method",
- "Organization Name",
- "MDS ID",
- "Connector"
- );
-
- var dataSpace = getAvailableFilter(result, "dataSpace");
- assertThat(dataSpace.getValues()).extracting(CnfFilterItem::getId).containsExactly("MDS");
- assertThat(dataSpace.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("MDS");
-
- var dataCategory = getAvailableFilter(result, "dataCategory");
- assertThat(dataCategory.getValues()).extracting(CnfFilterItem::getId).containsExactly("my-category-1");
- assertThat(dataCategory.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("my-category-1");
-
- var dataSubcategory = getAvailableFilter(result, "dataSubcategory");
- assertThat(dataSubcategory.getValues()).extracting(CnfFilterItem::getId).containsExactly("my-subcategory-1", "MY-SUBCATEGORY-2", "");
- assertThat(dataSubcategory.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("my-subcategory-1", "MY-SUBCATEGORY-2", "");
-
- var dataModel = getAvailableFilter(result, "dataModel");
- assertThat(dataModel.getValues()).extracting(CnfFilterItem::getId).containsExactly("my-data-model", "");
- assertThat(dataModel.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("my-data-model", "");
-
- var transportMode = getAvailableFilter(result, "transportMode");
- assertThat(transportMode.getValues()).extracting(CnfFilterItem::getId).containsExactly("MY-TRANSPORT-MODE-1", "my-transport-mode-2", "");
- assertThat(transportMode.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("MY-TRANSPORT-MODE-1", "my-transport-mode-2", "");
-
- var geoReferenceMethod = getAvailableFilter(result, "geoReferenceMethod");
- assertThat(geoReferenceMethod.getValues()).extracting(CnfFilterItem::getId).containsExactly("my-geo-ref", "");
- assertThat(geoReferenceMethod.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("my-geo-ref", "");
-
- var curatorOrganizationName = getAvailableFilter(result, "curatorOrganizationName");
- assertThat(curatorOrganizationName.getValues()).extracting(CnfFilterItem::getId).containsExactly("Test Org");
- assertThat(curatorOrganizationName.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("Test Org");
-
- var curatorMdsId = getAvailableFilter(result, "curatorMdsId");
- assertThat(curatorMdsId.getValues()).extracting(CnfFilterItem::getId).containsExactly("MDSL123456AA");
- assertThat(curatorMdsId.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("MDSL123456AA");
-
- var connectorEndpoint = getAvailableFilter(result, "connectorEndpoint");
- assertThat(connectorEndpoint.getValues()).extracting(CnfFilterItem::getId).containsExactly("https://my-connector/api/dsp");
- assertThat(connectorEndpoint.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("https://my-connector/api/dsp");
- });
- }
-
-
- /**
- * Regression Test against bug where asset names with capital letters were not hit by search.
- *
- * It was caused by search terms getting lower cased while the LIKE operation being case-sensitive.
- */
- @Test
- void testSearchCaseInsensitive() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("123")
- .title("Hello")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd);
-
-
- // act
- var query = new CatalogPageQuery();
- query.setSearchQuery("Hello");
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
-
- // assert
- assertThat(result.getDataOffers()).extracting(CatalogDataOffer::getAssetId).containsExactly("123");
- });
- }
-
- private CnfFilterAttribute getAvailableFilter(CatalogPageResult result, String filterId) {
- return result.getAvailableFilters().getFields().stream()
- .filter(it -> it.getId().equals(filterId)).findFirst()
- .orElseThrow(() -> new IllegalStateException("Filter not found"));
- }
-
- @Test
- void testAvailableFilters_withFilter() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd1 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-1")
- .dataCategory("my-category")
- .dataSubcategory("my-subcategory")
- .build()
- );
-
- var assetJsonLd2 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-2")
- .dataSubcategory("my-other-subcategory")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd1);
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd2);
-
- var query = new CatalogPageQuery();
- query.setFilter(new CnfFilterValue(List.of(
- new CnfFilterValueAttribute("dataCategory", List.of(""))
- )));
-
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
-
- var dataCategory = getAvailableFilter(result, "dataCategory");
- assertThat(dataCategory.getValues()).extracting(CnfFilterItem::getId).containsExactly("my-category", "");
- assertThat(dataCategory.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("my-category", "");
-
- var dataSubcategory = getAvailableFilter(result, "dataSubcategory");
- assertThat(dataSubcategory.getValues()).extracting(CnfFilterItem::getId).containsExactly("my-other-subcategory");
- assertThat(dataSubcategory.getValues()).extracting(CnfFilterItem::getTitle).containsExactly("my-other-subcategory");
- });
- }
-
- @Test
- void testPagination_firstPage() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- range(0, 15).forEach(i -> createDataOffer(dsl, today, "https://my-connector/api/dsp", getAssetJsonLd("my-asset-%d".formatted(i))));
- range(0, 15).forEach(i -> createDataOffer(dsl, today, "https://my-connector/api/dsp", getAssetJsonLd("some-other-asset-%d".formatted(i))));
-
- var query = new CatalogPageQuery();
- query.setSearchQuery("my-asset");
- query.setSorting(CatalogPageQuery.SortingEnum.TITLE);
-
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
- assertThat(result.getDataOffers()).extracting(CatalogDataOffer::getAssetId)
- .isEqualTo(range(0, 10).mapToObj("my-asset-%d"::formatted).toList());
-
- var actual = result.getPaginationMetadata();
- assertThat(actual.getPageOneBased()).isEqualTo(1);
- assertThat(actual.getPageSize()).isEqualTo(10);
- assertThat(actual.getNumVisible()).isEqualTo(10);
- assertThat(actual.getNumTotal()).isEqualTo(15);
- });
- }
-
- @Test
- void testPagination_secondPage() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- range(0, 15).forEach(i -> createDataOffer(dsl, today, "https://my-connector/api/dsp", getAssetJsonLd("my-asset-%d".formatted(i))));
- range(0, 15).forEach(i -> createDataOffer(dsl, today, "https://my-connector/api/dsp", getAssetJsonLd("some-other-asset-%d".formatted(i))));
-
-
- var query = new CatalogPageQuery();
- query.setSearchQuery("my-asset");
- query.setPageOneBased(2);
- query.setSorting(CatalogPageQuery.SortingEnum.TITLE);
-
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
-
- assertThat(result.getDataOffers()).extracting(CatalogDataOffer::getAssetId)
- .isEqualTo(range(10, 15).mapToObj("my-asset-%d"::formatted).toList());
-
- var actual = result.getPaginationMetadata();
- assertThat(actual.getPageOneBased()).isEqualTo(2);
- assertThat(actual.getPageSize()).isEqualTo(10);
- assertThat(actual.getNumVisible()).isEqualTo(5);
- assertThat(actual.getNumTotal()).isEqualTo(15);
- });
- }
-
- @Test
- void testSortingByPopularity() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var endpoint = "https://my-connector/api/dsp";
- createConnector(dsl, today, endpoint);
- createDataOffer(dsl, today, endpoint, getAssetJsonLd("asset-1"));
- createDataOffer(dsl, today, endpoint, getAssetJsonLd("asset-2"));
- createDataOffer(dsl, today, endpoint, getAssetJsonLd("asset-3"));
-
- range(0, 3).forEach(i -> dataOfferDetails(endpoint, "asset-1"));
- range(0, 5).forEach(i -> dataOfferDetails(endpoint, "asset-2"));
-
-
- var query = new CatalogPageQuery();
- query.setSorting(CatalogPageQuery.SortingEnum.VIEW_COUNT);
-
- var result = brokerServerClient().brokerServerApi().catalogPage(query);
- assertThat(result.getDataOffers()).extracting(CatalogDataOffer::getAssetId).containsExactly(
- "asset-2",
- "asset-1",
- "asset-3"
- );
- });
- }
-
- @Test
- void testFilterByOrgName() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var endpoint1 = "https://my-connector-1/api/dsp";
- createConnector(dsl, today, endpoint1, "MDSL1111AA");
- createDataOffer(dsl, today, endpoint1, getAssetJsonLd("asset-1"));
-
- var endpoint2 = "https://my-connector-2/api/dsp";
- createConnector(dsl, today, endpoint2, "MDSL2222BB");
- createDataOffer(dsl, today, endpoint2, getAssetJsonLd("asset-2"));
-
- createOrganizationMetadata(dsl, "MDSL1111AA", "Test Org");
-
- // act
- var query = new CatalogPageQuery();
- query.setFilter(new CnfFilterValue(List.of(
- new CnfFilterValueAttribute("curatorOrganizationName", List.of("Test Org"))
- )));
-
- var actual = brokerServerClient().brokerServerApi().catalogPage(query);
-
- // assert
- assertThat(actual.getDataOffers()).extracting(CatalogDataOffer::getConnectorEndpoint).containsExactly(endpoint1);
- });
- }
-
- @Test
- void testSearchForOrgName() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var endpoint1 = "https://my-connector-1/api/dsp";
- createConnector(dsl, today, endpoint1, "MDSL1111AA");
- createDataOffer(dsl, today, endpoint1, getAssetJsonLd("asset-1"));
-
- var endpoint2 = "https://my-connector-2/api/dsp";
- createConnector(dsl, today, endpoint2, "MDSL2222BB");
- createDataOffer(dsl, today, endpoint2, getAssetJsonLd("asset-2"));
-
- createOrganizationMetadata(dsl, "MDSL1111AA", "Test Org");
-
- // act
- var query = new CatalogPageQuery();
- query.setSearchQuery("tEsT");
-
- var actual = brokerServerClient().brokerServerApi().catalogPage(query);
-
- // assert
- assertThat(actual.getDataOffers()).extracting(CatalogDataOffer::getConnectorEndpoint).containsExactly(endpoint1);
- });
- }
-
- @Test
- void testFilterByMdsId() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var endpoint1 = "https://my-connector-1/api/dsp";
- createConnector(dsl, today, endpoint1, "MDSL1111AA");
- createDataOffer(dsl, today, endpoint1, getAssetJsonLd("asset-1"));
-
- var endpoint2 = "https://my-connector-2/api/dsp";
- createConnector(dsl, today, endpoint2, "MDSL2222BB");
- createDataOffer(dsl, today, endpoint2, getAssetJsonLd("asset-2"));
-
- // act
- var query = new CatalogPageQuery();
- query.setFilter(new CnfFilterValue(List.of(
- new CnfFilterValueAttribute("curatorMdsId", List.of("MDSL1111AA"))
- )));
-
- var actual = brokerServerClient().brokerServerApi().catalogPage(query);
-
- // assert
- assertThat(actual.getDataOffers()).extracting(CatalogDataOffer::getConnectorEndpoint).containsExactly(endpoint1);
- });
- }
-
- @Test
- void testFilterByUnknown() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var endpoint1 = "https://my-connector-1/api/dsp";
- createConnector(dsl, today, endpoint1, "MDSL1111AA");
- createDataOffer(dsl, today, endpoint1, getAssetJsonLd("asset-1"));
-
- var endpoint2 = "https://my-connector-2/api/dsp";
- createConnector(dsl, today, endpoint2, "MDSL2222BB");
- createDataOffer(dsl, today, endpoint2, getAssetJsonLd("asset-2"));
-
- createOrganizationMetadata(dsl, "MDSL1111AA", "Test Org");
-
- // act
- var query = new CatalogPageQuery();
- query.setFilter(new CnfFilterValue(List.of(
- new CnfFilterValueAttribute("curatorOrganizationName", List.of("Unknown"))
- )));
-
- var actual = brokerServerClient().brokerServerApi().catalogPage(query);
-
- // assert
- assertThat(actual.getDataOffers()).extracting(CatalogDataOffer::getConnectorEndpoint).containsExactly(endpoint2);
- });
- }
-
- @Test
- void testSearchForUnknown() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var today = OffsetDateTime.now().withNano(0);
-
- var endpoint1 = "https://my-connector-1/api/dsp";
- createConnector(dsl, today, endpoint1, "MDSL1111AA");
- createDataOffer(dsl, today, endpoint1, getAssetJsonLd("asset-1"));
-
- var endpoint2 = "https://my-connector-2/api/dsp";
- createConnector(dsl, today, endpoint2, "MDSL2222BB");
- createDataOffer(dsl, today, endpoint2, getAssetJsonLd("asset-2"));
-
- createOrganizationMetadata(dsl, "MDSL1111AA", "Test Org");
-
- // act
- var query = new CatalogPageQuery();
- query.setSearchQuery("uNkN");
-
- var actual = brokerServerClient().brokerServerApi().catalogPage(query);
-
- // assert
- assertThat(actual.getDataOffers()).extracting(CatalogDataOffer::getConnectorEndpoint).containsExactly(endpoint2);
- });
- }
-
- private void createDataOffer(DSLContext dsl, OffsetDateTime today, String connectorEndpoint, JsonObject assetJsonLd) {
- var dataOffer = dsl.newRecord(Tables.DATA_OFFER);
- setDataOfferAssetMetadata(dataOffer, assetJsonLd, "my-participant-id");
- dataOffer.setConnectorEndpoint(connectorEndpoint);
- dataOffer.setCreatedAt(today.minusDays(5));
- dataOffer.setUpdatedAt(today);
- dataOffer.insert();
-
- var contractOffer = dsl.newRecord(Tables.CONTRACT_OFFER);
- contractOffer.setContractOfferId("my-contract-offer-1");
- contractOffer.setConnectorEndpoint(connectorEndpoint);
- contractOffer.setAssetId(dataOffer.getAssetId());
- contractOffer.setCreatedAt(today.minusDays(5));
- contractOffer.setUpdatedAt(today);
- contractOffer.setPolicy(JSONB.jsonb(policyToJson(dummyPolicy())));
- contractOffer.insert();
- }
-
- private void createConnector(DSLContext dsl, OffsetDateTime today, String connectorEndpoint) {
- createConnector(dsl, today, connectorEndpoint, null);
- }
-
- private void createConnector(DSLContext dsl, OffsetDateTime today, String connectorEndpoint, String mdsId) {
- var connector = dsl.newRecord(Tables.CONNECTOR);
- connector.setParticipantId("my-connector");
- connector.setMdsId(mdsId);
- connector.setEndpoint(connectorEndpoint);
- connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
- connector.setCreatedAt(today.minusDays(1));
- connector.setLastRefreshAttemptAt(today);
- connector.setLastSuccessfulRefreshAt(today);
- connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
- connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
- connector.insert();
- }
-
- private void createOrganizationMetadata(DSLContext dsl, String mdsId, String name) {
- var organizationMetadata = dsl.newRecord(Tables.ORGANIZATION_METADATA);
- organizationMetadata.setMdsId(mdsId);
- organizationMetadata.setName(name);
- organizationMetadata.insert();
- }
-
- private Policy dummyPolicy() {
- return Policy.Builder.newInstance()
- .type(PolicyType.SET)
- .build();
- }
-
- private DataOfferDetailPageResult dataOfferDetails(String endpoint, String assetId) {
- var query = DataOfferDetailPageQuery.builder()
- .connectorEndpoint(endpoint)
- .assetId(assetId)
- .build();
- return brokerServerClient().brokerServerApi().dataOfferDetailPage(query);
- }
-
- private String policyToJson(Policy policy) {
- return toJson(policy);
- }
-
- @SneakyThrows
- private String toJson(Object o) {
- return new ObjectMapper().writeValueAsString(o);
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorApiTest.java
deleted file mode 100644
index fa3419438..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/ConnectorApiTest.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.TestPolicy;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorPageQuery;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementErrorStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementType;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import de.sovity.edc.ext.wrapper.api.common.model.UiAssetCreateRequest;
-import jakarta.json.JsonObject;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestAsset.getAssetJsonLd;
-import static de.sovity.edc.ext.brokerserver.TestAsset.setDataOfferAssetMetadata;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class ConnectorApiTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
- }
-
- @Test
- void testQueryConnectors() {
- TEST_DATABASE.testTransaction(dsl -> {
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-1")
- .title("My Asset 1")
- .dataCategory("my-category")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd);
-
- var result = brokerServerClient().brokerServerApi().connectorPage(new ConnectorPageQuery());
- assertThat(result.getConnectors()).hasSize(1);
-
- var connector = result.getConnectors().get(0);
- assertThat(connector.getParticipantId()).isEqualTo("my-participant-id");
- assertThat(connector.getEndpoint()).isEqualTo("https://my-connector/api/dsp");
- assertThat(connector.getCreatedAt()).isEqualTo(today.minusDays(1));
- assertThat(connector.getLastRefreshAttemptAt()).isEqualTo(today);
- assertThat(connector.getLastSuccessfulRefreshAt()).isEqualTo(today);
- assertThat(connector.getNumDataOffers()).isEqualTo(1);
- });
- }
-
- @Test
- void testQueryConnectorDetails() {
- TEST_DATABASE.testTransaction(dsl -> {
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-1")
- .title("My Asset 1")
- .dataCategory("my-category")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createConnector(dsl, today, "https://my-connector2/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd);
-
- var connector = brokerServerClient().brokerServerApi().connectorDetailPage(new ConnectorDetailPageQuery("https://my-connector/api/dsp"));
- assertThat(connector.getParticipantId()).isEqualTo("my-participant-id");
- assertThat(connector.getEndpoint()).isEqualTo("https://my-connector/api/dsp");
- assertThat(connector.getCreatedAt()).isEqualTo(today.minusDays(1));
- assertThat(connector.getLastRefreshAttemptAt()).isEqualTo(today);
- assertThat(connector.getLastSuccessfulRefreshAt()).isEqualTo(today);
- assertThat(connector.getConnectorCrawlingTimeAvg()).isEqualTo(150L);
- });
- }
-
- private void createConnector(DSLContext dsl, OffsetDateTime today, String connectorEndpoint) {
- var connector = dsl.newRecord(Tables.CONNECTOR);
- connector.setParticipantId("my-participant-id");
- connector.setEndpoint(connectorEndpoint);
- connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
- connector.setCreatedAt(today.minusDays(1));
- connector.setLastRefreshAttemptAt(today);
- connector.setLastSuccessfulRefreshAt(today);
- connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
- connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
- connector.insert();
-
- addCrawlingTime(dsl, today, connector, 100L);
- addCrawlingTime(dsl, today.plusHours(5), connector, 200L);
- }
-
- private static void addCrawlingTime(DSLContext dsl, OffsetDateTime today, ConnectorRecord connector, Long duration) {
- var crawlingTime = dsl.newRecord(Tables.BROKER_EXECUTION_TIME_MEASUREMENT);
- crawlingTime.setConnectorEndpoint(connector.getEndpoint());
- crawlingTime.setDurationInMs(duration);
- crawlingTime.setCreatedAt(today);
- crawlingTime.setType(MeasurementType.CONNECTOR_REFRESH);
- crawlingTime.setErrorStatus(MeasurementErrorStatus.OK);
- crawlingTime.insert();
- }
-
- private void createDataOffer(DSLContext dsl, OffsetDateTime today, String connectorEndpoint, JsonObject assetJsonLd) {
- var dataOffer = dsl.newRecord(Tables.DATA_OFFER);
- setDataOfferAssetMetadata(dataOffer, assetJsonLd, "my-participant-id");
- dataOffer.setConnectorEndpoint(connectorEndpoint);
- dataOffer.setCreatedAt(today.minusDays(5));
- dataOffer.setUpdatedAt(today);
- dataOffer.insert();
-
- var contractOffer = dsl.newRecord(Tables.CONTRACT_OFFER);
- contractOffer.setContractOfferId("my-contract-offer-1");
- contractOffer.setConnectorEndpoint(connectorEndpoint);
- contractOffer.setAssetId(dataOffer.getAssetId());
- contractOffer.setCreatedAt(today.minusDays(5));
- contractOffer.setUpdatedAt(today);
- contractOffer.setPolicy(TestPolicy.createAfterYesterdayPolicyJson());
- contractOffer.insert();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiTest.java
deleted file mode 100644
index 007ac7ffc..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/DataOfferDetailApiTest.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.client.gen.model.DataOfferDetailPageQuery;
-import de.sovity.edc.ext.brokerserver.client.gen.model.DataOfferDetailPageResult;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.wrapper.api.common.model.UiAssetCreateRequest;
-import jakarta.json.JsonObject;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.AssertionUtils.assertEqualUsingJson;
-import static de.sovity.edc.ext.brokerserver.TestAsset.getAssetJsonLd;
-import static de.sovity.edc.ext.brokerserver.TestAsset.setDataOfferAssetMetadata;
-import static de.sovity.edc.ext.brokerserver.TestPolicy.createAfterYesterdayConstraint;
-import static de.sovity.edc.ext.brokerserver.TestPolicy.createAfterYesterdayPolicyJson;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class DataOfferDetailApiTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of(
- )));
- }
-
- @Test
- void testQueryDataOfferDetails() {
- TEST_DATABASE.testTransaction(dsl -> {
- var today = OffsetDateTime.now().withNano(0);
-
- var assetJsonLd1 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-1")
- .title("My Asset 1")
- .dataCategory("my-category")
- .build()
- );
-
- var assetJsonLd2 = getAssetJsonLd(
- UiAssetCreateRequest.builder()
- .id("my-asset-2")
- .title("My Asset 2")
- .dataCategory("my-category-2")
- .build()
- );
-
- createConnector(dsl, today, "https://my-connector/api/dsp");
- createDataOffer(dsl, today, "https://my-connector/api/dsp", assetJsonLd1);
-
- createDataOfferView(dsl, today, "https://my-connector/api/dsp", "my-asset-1");
- createDataOfferView(dsl, today, "https://my-connector/api/dsp", "my-asset-1");
-
- createConnector(dsl, today, "https://my-connector2/api/dsp");
- createDataOffer(dsl, today, "https://my-connector2/api/dsp", assetJsonLd2);
-
- var actual = brokerServerClient().brokerServerApi().dataOfferDetailPage(new DataOfferDetailPageQuery("https://my-connector/api/dsp", "my-asset-1"));
- assertThat(actual.getAssetId()).isEqualTo("my-asset-1");
- assertThat(actual.getConnectorEndpoint()).isEqualTo("https://my-connector/api/dsp");
- assertThat(actual.getConnectorOfflineSinceOrLastUpdatedAt()).isEqualTo(today);
- assertThat(actual.getConnectorOnlineStatus()).isEqualTo(DataOfferDetailPageResult.ConnectorOnlineStatusEnum.ONLINE);
- assertThat(actual.getCreatedAt()).isEqualTo(today.minusDays(5));
- assertThat(actual.getAsset().getAssetId()).isEqualTo("my-asset-1");
- assertThat(actual.getAsset().getDataCategory()).isEqualTo("my-category");
- assertThat(actual.getAsset().getTitle()).isEqualTo("My Asset 1");
- assertThat(actual.getUpdatedAt()).isEqualTo(today);
- assertThat(actual.getContractOffers()).hasSize(1);
- var contractOffer = actual.getContractOffers().get(0);
- assertThat(contractOffer.getContractOfferId()).isEqualTo("my-contract-offer-1");
- assertEqualUsingJson(contractOffer.getContractPolicy().getConstraints().get(0), createAfterYesterdayConstraint());
- assertThat(contractOffer.getCreatedAt()).isEqualTo(today.minusDays(5));
- assertThat(contractOffer.getUpdatedAt()).isEqualTo(today);
- assertThat(actual.getViewCount()).isEqualTo(2);
- });
- }
-
- private void createConnector(DSLContext dsl, OffsetDateTime today, String connectorEndpoint) {
- var connector = dsl.newRecord(Tables.CONNECTOR);
- connector.setParticipantId("my-connector");
- connector.setEndpoint(connectorEndpoint);
- connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
- connector.setCreatedAt(today.minusDays(1));
- connector.setLastRefreshAttemptAt(today);
- connector.setLastSuccessfulRefreshAt(today);
- connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
- connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
- connector.insert();
- }
-
- private void createDataOffer(DSLContext dsl, OffsetDateTime today, String connectorEndpoint, JsonObject assetJsonLd) {
- var dataOffer = dsl.newRecord(Tables.DATA_OFFER);
- setDataOfferAssetMetadata(dataOffer, assetJsonLd, "my-participant-id");
- dataOffer.setConnectorEndpoint(connectorEndpoint);
- dataOffer.setCreatedAt(today.minusDays(5));
- dataOffer.setUpdatedAt(today);
- dataOffer.insert();
-
- var contractOffer = dsl.newRecord(Tables.CONTRACT_OFFER);
- contractOffer.setContractOfferId("my-contract-offer-1");
- contractOffer.setConnectorEndpoint(connectorEndpoint);
- contractOffer.setAssetId(dataOffer.getAssetId());
- contractOffer.setCreatedAt(today.minusDays(5));
- contractOffer.setUpdatedAt(today);
- contractOffer.setPolicy(createAfterYesterdayPolicyJson());
- contractOffer.insert();
- }
-
- private static void createDataOfferView(DSLContext dsl, OffsetDateTime date, String connectorEndpoint, String assetId) {
- var view = dsl.newRecord(Tables.DATA_OFFER_VIEW_COUNT);
- view.setAssetId(assetId);
- view.setConnectorEndpoint(connectorEndpoint);
- view.setDate(date);
- view.insert();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/DeleteConnectorsApiTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/DeleteConnectorsApiTest.java
deleted file mode 100644
index fb6ba3aca..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/api/DeleteConnectorsApiTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.api;
-
-import de.sovity.edc.ext.brokerserver.TestPolicy;
-import de.sovity.edc.ext.brokerserver.TestUtils;
-import de.sovity.edc.ext.brokerserver.client.BrokerServerClient;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorListEntry;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorPageQuery;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.BrokerEventStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.BrokerEventType;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementErrorStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementType;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.jooq.DSLContext;
-import org.jooq.Record;
-import org.jooq.TableField;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestAsset.getAssetJsonLd;
-import static de.sovity.edc.ext.brokerserver.TestAsset.setDataOfferAssetMetadata;
-import static de.sovity.edc.ext.brokerserver.TestUtils.ADMIN_API_KEY;
-import static de.sovity.edc.ext.brokerserver.TestUtils.brokerServerClient;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-
-@ApiTest
-@ExtendWith(EdcExtension.class)
-class DeleteConnectorsApiTest {
- BrokerServerClient client;
- String firstConnector = "http://a";
- String otherConnector = "http://b";
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
- client = brokerServerClient();
- }
-
- @Test
- void testRemoveConnectors() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- setupConnectorData(dsl, firstConnector);
- setupConnectorData(dsl, otherConnector);
-
- var connectorsBefore = List.of(firstConnector, otherConnector);
- assertContainsEndpoints(dsl, Tables.BROKER_EXECUTION_TIME_MEASUREMENT.CONNECTOR_ENDPOINT, connectorsBefore);
- assertContainsEndpoints(dsl, Tables.CONTRACT_OFFER.CONNECTOR_ENDPOINT, connectorsBefore);
- assertContainsEndpoints(dsl, Tables.DATA_OFFER.CONNECTOR_ENDPOINT, connectorsBefore);
- assertContainsEndpoints(dsl, Tables.DATA_OFFER_VIEW_COUNT.CONNECTOR_ENDPOINT, connectorsBefore);
- assertContainsEndpoints(dsl, Tables.CONNECTOR.ENDPOINT, connectorsBefore);
-
- // act
- client.brokerServerApi().deleteConnectors(ADMIN_API_KEY, List.of(firstConnector));
-
- // assert
- assertThat(client.brokerServerApi().connectorPage(new ConnectorPageQuery()).getConnectors())
- .extracting(ConnectorListEntry::getEndpoint)
- .containsExactly(otherConnector);
-
- var connectorsAfter = List.of(otherConnector);
- assertContainsEndpoints(dsl, Tables.BROKER_EXECUTION_TIME_MEASUREMENT.CONNECTOR_ENDPOINT, connectorsAfter);
- assertContainsEndpoints(dsl, Tables.CONTRACT_OFFER.CONNECTOR_ENDPOINT, connectorsAfter);
- assertContainsEndpoints(dsl, Tables.DATA_OFFER.CONNECTOR_ENDPOINT, connectorsAfter);
- assertContainsEndpoints(dsl, Tables.DATA_OFFER_VIEW_COUNT.CONNECTOR_ENDPOINT, connectorsAfter);
- assertContainsEndpoints(dsl, Tables.CONNECTOR.ENDPOINT, connectorsAfter);
- });
- }
-
- private void assertContainsEndpoints(
- DSLContext dsl,
- TableField endpointField,
- Collection expected
- ) {
- var actual = dsl.select(endpointField).from(endpointField.getTable()).fetchSet(endpointField);
- assertThat(actual).containsExactlyInAnyOrderElementsOf(expected);
- }
-
- public void setupConnectorData(DSLContext dsl, String endpoint) {
- client.brokerServerApi().addConnectors(ADMIN_API_KEY, List.of(endpoint));
-
- var assetId = "my-asset";
-
- var dataOffer = dsl.newRecord(Tables.DATA_OFFER);
- setDataOfferAssetMetadata(dataOffer, getAssetJsonLd("my-asset"), "my-participant-id");
- dataOffer.setConnectorEndpoint(endpoint);
- dataOffer.setCreatedAt(OffsetDateTime.now());
- dataOffer.setUpdatedAt(OffsetDateTime.now());
- dataOffer.insert();
-
- var contractOffer = dsl.newRecord(Tables.CONTRACT_OFFER);
- contractOffer.setAssetId(assetId);
- contractOffer.setConnectorEndpoint(endpoint);
- contractOffer.setContractOfferId("my-asset-co");
- contractOffer.setCreatedAt(OffsetDateTime.now());
- contractOffer.setPolicy(TestPolicy.createAfterYesterdayPolicyJson());
- contractOffer.setUpdatedAt(OffsetDateTime.now());
- contractOffer.insert();
-
- var logEntry = dsl.newRecord(Tables.BROKER_EVENT_LOG);
- logEntry.setEvent(BrokerEventType.CONNECTOR_UPDATED);
- logEntry.setUserMessage("Hello World!");
- logEntry.setAssetId(assetId);
- logEntry.setCreatedAt(OffsetDateTime.now());
- logEntry.setConnectorEndpoint(endpoint);
- logEntry.setEventStatus(BrokerEventStatus.OK);
- logEntry.insert();
-
- var measurement = dsl.newRecord(Tables.BROKER_EXECUTION_TIME_MEASUREMENT);
- measurement.setConnectorEndpoint(endpoint);
- measurement.setCreatedAt(OffsetDateTime.now());
- measurement.setDurationInMs(500L);
- measurement.setErrorStatus(MeasurementErrorStatus.OK);
- measurement.setType(MeasurementType.CONNECTOR_REFRESH);
- measurement.insert();
-
- var view = dsl.newRecord(Tables.DATA_OFFER_VIEW_COUNT);
- view.setConnectorEndpoint(endpoint);
- view.setAssetId(assetId);
- view.setDate(OffsetDateTime.now());
- view.insert();
- }
-
-
- @Test
- void testDeleteWrongApiKey() {
- TEST_DATABASE.testTransaction(dsl -> TestUtils.assertIs401(() ->
- client.brokerServerApi().deleteConnectors("wrong-api-key", List.of())));
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventLoggerTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventLoggerTest.java
deleted file mode 100644
index eee984c6f..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventLoggerTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.logging;
-
-import de.sovity.edc.ext.brokerserver.db.FlywayTestUtils;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.util.Set;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class BrokerEventLoggerTest {
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @BeforeAll
- static void setup() {
- FlywayTestUtils.migrate(TEST_DATABASE);
- }
-
- @Test
- void testDataOfferWriter_allSortsOfUpdates() {
- TEST_DATABASE.testTransaction(dsl -> {
- var brokerEventLogger = new BrokerEventLogger();
-
- // Test that insertions insert required fields and don't cause DB errors
- String endpoint = "https://example.com/api/dsp";
- brokerEventLogger.logConnectorUpdated(dsl, endpoint, new ConnectorChangeTracker());
- brokerEventLogger.logConnectorOnline(dsl, endpoint);
- brokerEventLogger.logConnectorOffline(dsl, endpoint, new BrokerEventErrorMessage("Message", "Stacktrace"));
- brokerEventLogger.logConnectorUpdateContractOfferLimitExceeded(dsl, 10, endpoint);
- brokerEventLogger.logConnectorUpdateContractOfferLimitOk(dsl, endpoint);
- brokerEventLogger.logConnectorUpdateDataOfferLimitExceeded(dsl, 10, endpoint);
- brokerEventLogger.logConnectorUpdateDataOfferLimitOk(dsl, endpoint);
- brokerEventLogger.logConnectorsDeleted(dsl, Set.of(endpoint));
-
- assertThat(numLogEntries(dsl)).isEqualTo(8);
- });
- }
-
- private Integer numLogEntries(DSLContext dsl) {
- return dsl.selectCount().from(Tables.BROKER_EVENT_LOG).fetchOne().component1();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/queue/ThreadPoolQueueTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/queue/ThreadPoolQueueTest.java
deleted file mode 100644
index 55b2b1efa..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/queue/ThreadPoolQueueTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.queue;
-
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-class ThreadPoolQueueTest {
-
-
- /**
- * Regression against bug where the queue did not act like a queue.
- */
- @Test
- void testOrdering() {
- Runnable noop = () -> {
- };
-
- var queue = new ThreadPoolTaskQueue();
- queue.add(new ThreadPoolTask(1, noop, "1.0"));
- queue.add(new ThreadPoolTask(2, noop, "2.0"));
- queue.add(new ThreadPoolTask(1, noop, "1.1"));
- queue.add(new ThreadPoolTask(2, noop, "2.1"));
- queue.add(new ThreadPoolTask(0, noop, "0.0"));
-
- var result = new ArrayList();
- queue.getQueue().drainTo(result);
-
- assertThat(result).extracting(ThreadPoolTask::getConnectorEndpoint)
- .containsExactly("0.0", "1.0", "1.1", "2.0", "2.1");
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/queue/ThreadPoolTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/queue/ThreadPoolTest.java
deleted file mode 100644
index a45e23d0b..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/queue/ThreadPoolTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.queue;
-
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import lombok.SneakyThrows;
-import org.apache.commons.lang3.Validate;
-import org.eclipse.edc.spi.monitor.Monitor;
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-class ThreadPoolTest {
-
- /**
- * Regression against bug where parallelity wasn't actually enabled
- */
- @Test
- @SneakyThrows
- void testParallelExecution() {
- ThreadPool threadPool = newThreadPool(2);
- var latch = new CountDownLatch(2);
- var result = new ArrayList();
-
- var a = new BlockedRunnable(() -> {
- result.add("a");
- latch.countDown();
- });
- var b = new BlockedRunnable(() -> {
- result.add("b");
- latch.countDown();
- });
-
- threadPool.enqueueConnectorRefreshTask(0, a, "a");
- threadPool.enqueueConnectorRefreshTask(0, b, "b");
-
- b.release();
- Thread.sleep(250); // For some reason this is required
- a.release();
-
- Validate.isTrue(latch.await(500, TimeUnit.MILLISECONDS), "latch timed out");
- assertThat(result).containsExactly("b", "a");
- }
-
-
- @NotNull
- private ThreadPool newThreadPool(int numThreads) {
- var monitor = mock(Monitor.class);
- var brokerServerSettings = mock(BrokerServerSettings.class);
- when(brokerServerSettings.getNumThreads()).thenReturn(numThreads);
- var queue = new ThreadPoolTaskQueue();
- return new ThreadPool(queue, brokerServerSettings, monitor);
- }
-
- private static class BlockedRunnable implements Runnable {
- private static final Object GLOBAL_LOCK = new Object();
- private final Runnable runnable;
- private final ReentrantLock lock = new ReentrantLock();
-
- private BlockedRunnable(Runnable runnable) {
- this.runnable = runnable;
- lock.lock();
- }
-
- public void release() {
- lock.unlock();
- }
-
- @Override
- @SneakyThrows
- public void run() {
- var ok = lock.tryLock(10, TimeUnit.SECONDS);
- Validate.isTrue(ok, "Program is stuck!");
-
- // Prevent concurrency issues within the test code
- synchronized (GLOBAL_LOCK) {
- runnable.run();
- }
- }
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdaterTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdaterTest.java
deleted file mode 100644
index 3e8b82369..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdaterTest.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing;
-
-import de.sovity.edc.client.EdcClient;
-import de.sovity.edc.client.gen.model.ContractDefinitionRequest;
-import de.sovity.edc.client.gen.model.DataSourceType;
-import de.sovity.edc.client.gen.model.PolicyDefinitionCreateRequest;
-import de.sovity.edc.client.gen.model.UiAssetCreateRequest;
-import de.sovity.edc.client.gen.model.UiCriterion;
-import de.sovity.edc.client.gen.model.UiCriterionLiteral;
-import de.sovity.edc.client.gen.model.UiCriterionLiteralType;
-import de.sovity.edc.client.gen.model.UiCriterionOperator;
-import de.sovity.edc.client.gen.model.UiDataSource;
-import de.sovity.edc.client.gen.model.UiDataSourceHttpData;
-import de.sovity.edc.ext.brokerserver.AssertionUtils;
-import de.sovity.edc.ext.brokerserver.BrokerServerExtensionContext;
-import de.sovity.edc.ext.brokerserver.TestUtils;
-import de.sovity.edc.ext.brokerserver.client.BrokerServerClient;
-import de.sovity.edc.ext.brokerserver.client.gen.model.CatalogPageQuery;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorListEntry;
-import de.sovity.edc.ext.brokerserver.client.gen.model.ConnectorPageQuery;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.Tables;
-import de.sovity.edc.utils.jsonld.vocab.Prop;
-import org.eclipse.edc.junit.annotations.ApiTest;
-import org.eclipse.edc.junit.extensions.EdcExtension;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.OffsetDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.List;
-import java.util.Map;
-
-import static de.sovity.edc.ext.brokerserver.TestPolicy.createAfterYesterdayConstraint;
-import static de.sovity.edc.ext.brokerserver.TestPolicy.createAfterYesterdayPolicyEdcGen;
-import static de.sovity.edc.ext.brokerserver.TestUtils.createConfiguration;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.within;
-
-@ApiTest
-class ConnectorUpdaterTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- @RegisterExtension
- static EdcExtension consumerEdcContext = new EdcExtension();
-
- private EdcClient providerClient;
-
- private BrokerServerClient brokerServerClient;
-
- @BeforeEach
- void setUp(EdcExtension extension) {
- extension.setConfiguration(createConfiguration(TEST_DATABASE, Map.of()));
-
- providerClient = EdcClient.builder()
- .managementApiUrl(TestUtils.MANAGEMENT_ENDPOINT)
- .managementApiKey(TestUtils.MANAGEMENT_API_KEY)
- .build();
-
- brokerServerClient = BrokerServerClient.builder()
- .managementApiUrl(TestUtils.MANAGEMENT_ENDPOINT)
- .managementApiKey(TestUtils.MANAGEMENT_API_KEY)
- .build();
- }
-
- @Test
- void testConnectorUpdate() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- var connectorUpdater = BrokerServerExtensionContext.instance.connectorUpdater();
- var connectorCreator = BrokerServerExtensionContext.instance.connectorCreator();
- String connectorEndpoint = TestUtils.PROTOCOL_ENDPOINT;
-
- var policyId = createPolicyDefinition();
- var assetId = createAsset();
- createContractDefinition(policyId, assetId);
- connectorCreator.addConnector(dsl, connectorEndpoint);
-
- // act
- connectorUpdater.updateConnector(connectorEndpoint);
- var connectorPage = brokerServerClient.brokerServerApi().connectorPage(new ConnectorPageQuery());
-
- // assert
- var catalog = brokerServerClient.brokerServerApi().catalogPage(new CatalogPageQuery());
- assertThat(catalog.getDataOffers()).hasSize(1);
- var dataOffer = catalog.getDataOffers().get(0);
- assertThat(dataOffer.getContractOffers()).hasSize(1);
- var contractOffer = dataOffer.getContractOffers().get(0);
- var asset = dataOffer.getAsset();
- assertThat(asset.getAssetId()).isEqualTo(assetId);
- assertThat(asset.getTitle()).isEqualTo("AssetName");
- assertThat(asset.getConnectorEndpoint()).isEqualTo(TestUtils.PROTOCOL_ENDPOINT);
- assertThat(asset.getParticipantId()).isEqualTo(TestUtils.PARTICIPANT_ID);
- assertThat(asset.getKeywords()).isEqualTo(List.of("keyword1", "keyword2"));
- assertThat(asset.getDescription()).isEqualTo("AssetDescription");
- assertThat(asset.getVersion()).isEqualTo("1.0.0");
- assertThat(asset.getLanguage()).isEqualTo("en");
- assertThat(asset.getMediaType()).isEqualTo("application/json");
- assertThat(asset.getDataCategory()).isEqualTo("dataCategory");
- assertThat(asset.getDataSubcategory()).isEqualTo("dataSubcategory");
- assertThat(asset.getDataModel()).isEqualTo("dataModel");
- assertThat(asset.getGeoReferenceMethod()).isEqualTo("geoReferenceMethod");
- assertThat(asset.getTransportMode()).isEqualTo("transportMode");
- assertThat(asset.getLicenseUrl()).isEqualTo("https://license-url");
- assertThat(asset.getKeywords()).isEqualTo(List.of("keyword1", "keyword2"));
- assertThat(asset.getCreatorOrganizationName()).isEqualTo("Unknown");
- assertThat(asset.getPublisherHomepage()).isEqualTo("publisherHomepage");
- assertThat(asset.getHttpDatasourceHintsProxyMethod()).isFalse();
- assertThat(asset.getHttpDatasourceHintsProxyPath()).isFalse();
- assertThat(asset.getHttpDatasourceHintsProxyQueryParams()).isFalse();
- assertThat(asset.getHttpDatasourceHintsProxyBody()).isFalse();
- assertThat(asset.getCustomJsonAsString())
- .isEqualTo("{\"a\":\"x\"}");
- assertThat(dataOffer.getAsset().getCustomJsonLdAsString())
- .isEqualTo("{\"http://unknown/b\":{\"http://unknown/c\":\"y\"}}");
- assertThat(dataOffer.getAsset().getPrivateCustomJsonAsString()).isNull();
- assertThat(dataOffer.getAsset().getPrivateCustomJsonLdAsString()).isEqualTo("{}");
- var policy = contractOffer.getContractPolicy();
- assertThat(policy.getConstraints()).hasSize(1);
- AssertionUtils.assertEqualUsingJson(policy.getConstraints().get(0), createAfterYesterdayConstraint());
-
- var connector = connectorPage.getConnectors().get(0);
- assertThat(connector.getOnlineStatus()).isEqualTo(ConnectorListEntry.OnlineStatusEnum.ONLINE);
- assertThat(connector.getParticipantId()).isEqualTo(TestUtils.PARTICIPANT_ID);
- assertThat(connector.getOrganizationName()).isEqualTo("Unknown");
- assertThat(connector.getLastRefreshAttemptAt()).isCloseTo(OffsetDateTime.now(), within(1, ChronoUnit.SECONDS));
-
- var connectorRecord = dsl.selectFrom(Tables.CONNECTOR).fetchOne();
- assertThat(connectorRecord.getMdsId()).isEqualTo("MDSL1234ZZ");
- });
- }
-
- private String createPolicyDefinition() {
- var policyDefinition = PolicyDefinitionCreateRequest.builder()
- .policyDefinitionId("policy-1")
- .policy(createAfterYesterdayPolicyEdcGen())
- .build();
-
- return providerClient.uiApi().createPolicyDefinition(policyDefinition).getId();
- }
-
- public void createContractDefinition(String policyId, String assetId) {
- providerClient.uiApi().createContractDefinition(ContractDefinitionRequest.builder()
- .contractDefinitionId("cd-1")
- .accessPolicyId(policyId)
- .contractPolicyId(policyId)
- .assetSelector(List.of(UiCriterion.builder()
- .operandLeft(Prop.Edc.ID)
- .operator(UiCriterionOperator.EQ)
- .operandRight(UiCriterionLiteral.builder()
- .type(UiCriterionLiteralType.VALUE)
- .value(assetId)
- .build())
- .build()))
- .build());
- }
-
- private String createAsset() {
- var dataSource = UiDataSource.builder()
- .type(DataSourceType.HTTP_DATA)
- .httpData(UiDataSourceHttpData.builder()
- .baseUrl("http://some.url")
- .build())
- .build();
-
- return providerClient.uiApi().createAsset(UiAssetCreateRequest.builder()
- .id("asset-1")
- .title("AssetName")
- .description("AssetDescription")
- .licenseUrl("https://license-url")
- .version("1.0.0")
- .language("en")
- .mediaType("application/json")
- .dataCategory("dataCategory")
- .dataSubcategory("dataSubcategory")
- .dataModel("dataModel")
- .geoReferenceMethod("geoReferenceMethod")
- .transportMode("transportMode")
- .keywords(List.of("keyword1", "keyword2"))
- .publisherHomepage("publisherHomepage")
- .dataSource(dataSource)
- .customJsonAsString("{\"a\":\"x\"}")
- .customJsonLdAsString("{\"http://unknown/b\":{\"http://unknown/c\":\"y\"}}")
- .privateCustomJsonAsString("{\"a-private\":\"x-private\"}")
- .privateCustomJsonLdAsString("{\"http://unknown/b-private\":{\"http://unknown/c-private\":\"y-private\"}}")
- .build()).getId();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferWriterTestDydi.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferWriterTestDydi.java
deleted file mode 100644
index d44383937..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferWriterTestDydi.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
-
-import de.sovity.edc.ext.brokerserver.dao.ContractOfferQueries;
-import de.sovity.edc.ext.brokerserver.dao.DataOfferQueries;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import lombok.Value;
-import org.eclipse.edc.spi.system.configuration.Config;
-
-import static org.mockito.Mockito.mock;
-
-@Value
-class DataOfferWriterTestDydi {
- Config config = mock(Config.class);
- BrokerServerSettings brokerServerSettings = mock(BrokerServerSettings.class);
- DataOfferQueries dataOfferQueries = new DataOfferQueries();
- ContractOfferQueries contractOfferQueries = new ContractOfferQueries();
- ContractOfferRecordUpdater contractOfferRecordUpdater = new ContractOfferRecordUpdater();
- DataOfferRecordUpdater dataOfferRecordUpdater = new DataOfferRecordUpdater();
- DataOfferPatchBuilder dataOfferPatchBuilder = new DataOfferPatchBuilder(
- contractOfferQueries,
- dataOfferQueries,
- dataOfferRecordUpdater,
- contractOfferRecordUpdater
- );
- DataOfferPatchApplier dataOfferPatchApplier = new DataOfferPatchApplier();
- DataOfferWriter dataOfferWriter = new DataOfferWriter(dataOfferPatchBuilder, dataOfferPatchApplier);
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/schedules/OfflineConnectorRemovalJobTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/schedules/OfflineConnectorRemovalJobTest.java
deleted file mode 100644
index feb69efc4..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/services/schedules/OfflineConnectorRemovalJobTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.services.schedules;
-
-import de.sovity.edc.ext.brokerserver.dao.ConnectorQueries;
-import de.sovity.edc.ext.brokerserver.db.FlywayTestUtils;
-import de.sovity.edc.ext.brokerserver.db.TestDatabase;
-import de.sovity.edc.ext.brokerserver.db.TestDatabaseFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.services.ConnectorCleaner;
-import de.sovity.edc.ext.brokerserver.services.ConnectorKiller;
-import de.sovity.edc.ext.brokerserver.services.OfflineConnectorKiller;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
-import org.jooq.DSLContext;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.time.Duration;
-import java.time.OffsetDateTime;
-
-import static de.sovity.edc.ext.brokerserver.db.jooq.tables.Connector.CONNECTOR;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-class OfflineConnectorRemovalJobTest {
-
- @RegisterExtension
- private static final TestDatabase TEST_DATABASE = TestDatabaseFactory.getTestDatabase();
-
- BrokerServerSettings brokerServerSettings;
- OfflineConnectorKiller offlineConnectorKiller;
-
- @BeforeAll
- static void beforeAll() {
- FlywayTestUtils.migrate(TEST_DATABASE);
- }
-
- @BeforeEach
- void beforeEach() {
- brokerServerSettings = mock(BrokerServerSettings.class);
- offlineConnectorKiller = new OfflineConnectorKiller(
- brokerServerSettings,
- new ConnectorQueries(),
- new BrokerEventLogger(),
- new ConnectorKiller(),
- new ConnectorCleaner()
- );
- }
-
- @Test
- void test_offlineConnectorKiller_should_be_dead() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- when(brokerServerSettings.getKillOfflineConnectorsAfter()).thenReturn(Duration.ofDays(5));
- createConnector(dsl, 6);
-
- // act
- offlineConnectorKiller.killIfOfflineTooLong(dsl);
-
- // assert
- dsl.select().from(CONNECTOR).fetch().forEach(record -> {
- assertThat(record.get(CONNECTOR.ENDPOINT)).isEqualTo("https://my-connector/api/dsp");
- assertThat(record.get(CONNECTOR.ONLINE_STATUS)).isEqualTo(ConnectorOnlineStatus.DEAD);
- });
- });
- }
-
- @Test
- void test_offlineConnectorKiller_should_not_be_dead() {
- TEST_DATABASE.testTransaction(dsl -> {
- // arrange
- when(brokerServerSettings.getKillOfflineConnectorsAfter()).thenReturn(Duration.ofDays(5));
- createConnector(dsl, 2);
-
- // act
- offlineConnectorKiller.killIfOfflineTooLong(dsl);
-
- // assert
- dsl.select().from(CONNECTOR).fetch().forEach(record -> {
- assertThat(record.get(CONNECTOR.ENDPOINT)).isEqualTo("https://my-connector/api/dsp");
- assertThat(record.get(CONNECTOR.ONLINE_STATUS)).isNotEqualTo(ConnectorOnlineStatus.DEAD);
- });
- });
- }
-
- private static void createConnector(DSLContext dsl, int createdDaysAgo) {
- dsl.insertInto(CONNECTOR)
- .set(CONNECTOR.ENDPOINT, "https://my-connector/api/dsp")
- .set(CONNECTOR.PARTICIPANT_ID, "my-connector")
- .set(CONNECTOR.ONLINE_STATUS, ConnectorOnlineStatus.OFFLINE)
- .set(CONNECTOR.LAST_SUCCESSFUL_REFRESH_AT, OffsetDateTime.now().minusDays(createdDaysAgo))
- .set(CONNECTOR.CREATED_AT, OffsetDateTime.now().minusDays(6))
- .set(CONNECTOR.DATA_OFFERS_EXCEEDED, ConnectorDataOffersExceeded.OK)
- .set(CONNECTOR.CONTRACT_OFFERS_EXCEEDED, ConnectorContractOffersExceeded.OK).execute();
- }
-}
diff --git a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/utils/UrlUtilsTest.java b/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/utils/UrlUtilsTest.java
deleted file mode 100644
index 72a223007..000000000
--- a/extensions/broker-server/src/test/java/de/sovity/edc/ext/brokerserver/utils/UrlUtilsTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2023 sovity GmbH
- *
- * This program and the accompanying materials are made available under the
- * terms of the Apache License, Version 2.0 which is available at
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- * Contributors:
- * sovity GmbH - initial API and implementation
- *
- */
-
-package de.sovity.edc.ext.brokerserver.utils;
-
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-class UrlUtilsTest {
- @Test
- void test_urlUtils() {
- assertTrue(UrlUtils.isValidUrl("http://localhost:8080"));
- assertTrue(UrlUtils.isValidUrl(" http://localhost:8080"));
-
- assertFalse(UrlUtils.isValidUrl("test"));
- assertFalse(UrlUtils.isValidUrl(""));
- assertFalse(UrlUtils.isValidUrl(" "));
- assertFalse(UrlUtils.isValidUrl(null));
- }
-}
diff --git a/extensions/catalog-crawler/README.md b/extensions/catalog-crawler/README.md
new file mode 100644
index 000000000..28de136de
--- /dev/null
+++ b/extensions/catalog-crawler/README.md
@@ -0,0 +1,39 @@
+
+
+
+
+## About this Extension
+
+The catalog crawler is a deployment unit depending on an existing Authority Portal's database:
+
+- It periodically checks the Authority Portal's connector list for its environment.
+- It crawls the given connectors in regular intervals.
+- It writes the data offers and connector statuses back into the Authority Portal DB.
+- Each Environment configured in the Authority Portal requires its own Catalog Crawler with credentials for that environment's DAPS.
+
+## Why does this component exist?
+
+The Authority Portal uses a non-EDC stack, and the EDC stack cannot handle multiple sources of authority at once.
+
+With the `DB -> UI` part of the broker having been moved to the Authority Portal, only the `Catalog -> DB` part remains as the Catalog Crawler,
+as it requires Connector-to-Connector IAM within the given Dataspace.
+
+## License
+
+Apache License 2.0 - see [LICENSE](../../LICENSE)
+
+## Contact
+
+sovity GmbH - contact@sovity.de
diff --git a/extensions/broker-server-postgres-flyway-jooq/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-db/build.gradle.kts
similarity index 78%
rename from extensions/broker-server-postgres-flyway-jooq/build.gradle.kts
rename to extensions/catalog-crawler/catalog-crawler-db/build.gradle.kts
index 9320b4c62..5c97eb9c4 100644
--- a/extensions/broker-server-postgres-flyway-jooq/build.gradle.kts
+++ b/extensions/catalog-crawler/catalog-crawler-db/build.gradle.kts
@@ -7,9 +7,8 @@ val jooqDbType = "org.jooq.meta.postgres.PostgresDatabase"
val jdbcDriver = "org.postgresql.Driver"
val postgresContainer = libs.versions.postgresDbImage.get()
-val migrationsDir = "src/main/resources/db/migration"
-val testDataDir = "src/main/resources/db/testdata"
-val jooqTargetPackage = "de.sovity.edc.ext.brokerserver.db.jooq"
+val migrationsDir = "src/main/resources/db-crawler/migration"
+val jooqTargetPackage = "de.sovity.edc.ext.catalog.crawler.db.jooq"
val jooqTargetSourceRoot = "build/generated/jooq"
val jooqTargetDir = jooqTargetSourceRoot + "/" + jooqTargetPackage.replace(".", "/")
@@ -34,20 +33,9 @@ dependencies {
jooqGenerator(libs.postgres)
flywayMigration(libs.postgres)
- implementation(libs.hikari)
-
- annotationProcessor(libs.lombok)
- compileOnly(libs.lombok)
- implementation(libs.apache.commonsLang)
-
- implementation(libs.edc.coreSpi)
// Adds Database-Related EDC-Extensions (EDC-SQL-Stores, JDBC-Driver, Pool and Transactions)
implementation(libs.postgres)
-
- api(libs.flyway.core)
-
- testImplementation(libs.edc.junit)
}
sourceSets {
@@ -71,20 +59,20 @@ fun isTestcontainersEnabled(): Boolean {
fun jdbcUrl(): String {
return container?.jdbcUrl
- ?: System.getenv()[jdbcUrlEnvVarName]
- ?: error("Need $jdbcUrlEnvVarName since $skipTestcontainersEnvVarName=true")
+ ?: System.getenv()[jdbcUrlEnvVarName]
+ ?: error("Need $jdbcUrlEnvVarName since $skipTestcontainersEnvVarName=true")
}
fun jdbcUser(): String {
return container?.username
- ?: System.getenv()[jdbcUserEnvVarName]
- ?: error("Need $jdbcUserEnvVarName since $skipTestcontainersEnvVarName=true")
+ ?: System.getenv()[jdbcUserEnvVarName]
+ ?: error("Need $jdbcUserEnvVarName since $skipTestcontainersEnvVarName=true")
}
fun jdbcPassword(): String {
return container?.password
- ?: System.getenv()[jdbcPasswordEnvVarName]
- ?: error("Need $jdbcPasswordEnvVarName since $skipTestcontainersEnvVarName=true")
+ ?: System.getenv()[jdbcPasswordEnvVarName]
+ ?: error("Need $jdbcPasswordEnvVarName since $skipTestcontainersEnvVarName=true")
}
@@ -107,8 +95,9 @@ flyway {
cleanDisabled = false
cleanOnValidationError = true
baselineOnMigrate = true
- locations = arrayOf("filesystem:${migrationsDir}", "filesystem:${testDataDir}")
+ locations = arrayOf("filesystem:${migrationsDir}")
configurations = arrayOf("flywayMigration")
+ failOnMissingLocations = true
mixed = true
}
@@ -159,15 +148,15 @@ jooq {
tasks.withType {
dependsOn.add("flywayMigrate")
inputs.files(fileTree(migrationsDir))
- .withPropertyName("migrations")
- .withPathSensitivity(PathSensitivity.RELATIVE)
+ .withPropertyName("migrations")
+ .withPathSensitivity(PathSensitivity.RELATIVE)
allInputsDeclared.set(true)
outputs.cacheIf { true }
doFirst {
require(this is nu.studer.gradle.jooq.JooqGenerate)
val jooqConfiguration = nu.studer.gradle.jooq.JooqGenerate::class.java.getDeclaredField("jooqConfiguration")
- .also { it.isAccessible = true }.get(this) as org.jooq.meta.jaxb.Configuration
+ .also { it.isAccessible = true }.get(this) as org.jooq.meta.jaxb.Configuration
jooqConfiguration.jdbc.apply {
url = jdbcUrl()
@@ -180,7 +169,7 @@ tasks.withType {
}
}
-group = libs.versions.sovityBrokerServerGroup.get()
+group = libs.versions.sovityCatalogCrawlerGroup.get()
publishing {
publications {
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration-test-utils/V9999__Make_Columns_Nullable.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration-test-utils/V9999__Make_Columns_Nullable.sql
new file mode 100644
index 000000000..bc4f9fd8c
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration-test-utils/V9999__Make_Columns_Nullable.sql
@@ -0,0 +1,25 @@
+do
+$$
+ declare
+ r record;
+ begin
+ for r in (select 'alter table "' || c.table_schema || '"."' || c.table_name || '" alter column "' || c.column_name ||
+ '" drop not null;' as command
+ from information_schema.columns c
+ where c.table_schema not in ('pg_catalog', 'information_schema') -- exclude system schemas
+ and c.table_name in ('connector', 'organization', 'user') -- only selected AP tables
+ and c.is_nullable = 'NO'
+ and not exists (SELECT tc.constraint_type
+ FROM information_schema.table_constraints AS tc
+ JOIN information_schema.key_column_usage AS kcu
+ ON tc.constraint_name = kcu.constraint_name
+ AND tc.table_schema = kcu.table_schema
+ WHERE tc.table_schema = c.table_schema
+ and tc.table_name = c.table_name
+ AND kcu.column_name = c.column_name
+ AND tc.constraint_type = 'PRIMARY KEY')) -- exclude primary keys
+ loop
+ execute r.command;
+ end loop;
+ end
+$$;
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V1__Initial_Schema.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V1__Initial_Schema.sql
new file mode 100644
index 000000000..bd792737a
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V1__Initial_Schema.sql
@@ -0,0 +1,50 @@
+-- User
+create type user_registration_status as enum ('FIRST_USER', 'CREATED', 'PENDING', 'APPROVED', 'REJECTED');
+
+create table "user"
+(
+ id text not null primary key,
+ organization_mds_id text,
+ registration_status user_registration_status not null
+);
+
+-- Organization
+create type organization_registration_status as enum ('PENDING', 'APPROVED', 'REJECTED');
+
+create table "organization"
+(
+ mds_id text not null primary key,
+ name text not null,
+ address text not null,
+ duns text not null,
+ url text not null,
+ security_email text not null,
+ created_by text not null,
+ registration_status organization_registration_status not null,
+ constraint fk_organization_created_by foreign key (created_by) references "user" (id)
+);
+
+alter table "user"
+ add constraint fk_user_organization_id
+ foreign key (organization_mds_id) references "organization" (mds_id);
+
+-- Connector
+create type connector_type as enum ('OWN', 'PROVIDED', 'CAAS');
+
+create table "connector"
+(
+ connector_id text not null primary key,
+ mds_id text not null,
+ provider_mds_id text not null,
+ type connector_type not null,
+ environment text not null,
+ client_id text not null,
+ name text not null,
+ location text not null,
+ url text not null,
+ created_by text not null,
+ created_at timestamp with time zone not null,
+ constraint fk_connector_organization_id foreign key (mds_id) references "organization" (mds_id),
+ constraint fk_connector_provider_id foreign key (provider_mds_id) references "organization" (mds_id),
+ constraint fk_connector_created_by foreign key (created_by) references "user" (id)
+);
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V2__Schema_Extension.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V2__Schema_Extension.sql
new file mode 100644
index 000000000..599d64773
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V2__Schema_Extension.sql
@@ -0,0 +1,12 @@
+-- User
+alter type user_registration_status add value 'INVITED' before 'CREATED';
+alter type user_registration_status add value 'DEACTIVATED' after 'REJECTED';
+alter type user_registration_status rename value 'APPROVED' to 'ACTIVE';
+
+alter table "user" add column created_at timestamp with time zone not null default now();
+
+-- Organization
+alter type organization_registration_status add value 'INVITED' before 'PENDING';
+alter type organization_registration_status rename value 'APPROVED' to 'ACTIVE';
+
+alter table "organization" add column created_at timestamp with time zone not null default now();
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V3__Schema_Extension.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V3__Schema_Extension.sql
new file mode 100644
index 000000000..e9aafee70
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V3__Schema_Extension.sql
@@ -0,0 +1,28 @@
+-- User
+ALTER TABLE "user"
+ ADD COLUMN email text,
+ ADD COLUMN first_name text,
+ ADD COLUMN last_name text,
+ ADD COLUMN job_title text,
+ ADD COLUMN phone text;
+
+-- Organization
+ALTER TABLE "organization"
+ ADD COLUMN business_unit text,
+ ADD COLUMN billing_address text,
+ ADD COLUMN tax_id text,
+ ADD COLUMN commerce_register_number text,
+ ADD COLUMN commerce_register_location text,
+ ADD COLUMN main_contact_name text,
+ ADD COLUMN main_contact_email text,
+ ADD COLUMN main_contact_phone text,
+ ADD COLUMN tech_contact_name text,
+ ADD COLUMN tech_contact_email text,
+ ADD COLUMN tech_contact_phone text;
+
+UPDATE "organization" SET main_contact_email = security_email;
+
+ALTER TABLE "organization"
+ DROP COLUMN duns,
+ DROP COLUMN security_email;
+
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V4__Schema_Extension.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V4__Schema_Extension.sql
new file mode 100644
index 000000000..bfe8162fa
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V4__Schema_Extension.sql
@@ -0,0 +1,14 @@
+-- Components
+create table "component" (
+ id text primary key,
+ mds_id text not null,
+ name text not null,
+ homepage_url text,
+ endpoint_url text not null,
+ environment text not null,
+ client_id text not null,
+ created_by text not null,
+ created_at timestamp with time zone not null default now(),
+ constraint fk_component_created_by foreign key (created_by) references "user" (id),
+ constraint fk_component_organization_id foreign key (mds_id) references "organization" (mds_id)
+);
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V5__Schema_Extension.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V5__Schema_Extension.sql
new file mode 100644
index 000000000..ec89fa6bb
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V5__Schema_Extension.sql
@@ -0,0 +1,46 @@
+-- User
+create type user_onboarding_type as enum ('INVITATION', 'SELF_REGISTRATION');
+
+alter table "user"
+ add column onboarding_type user_onboarding_type,
+ add column invited_by text,
+ add constraint fk_invited_by foreign key (invited_by) references "user" (id);
+
+update "user" set onboarding_type = 'SELF_REGISTRATION' where onboarding_type is null;
+
+-- Connector
+create type connector_broker_registration_status as enum ('REGISTERED', 'UNREGISTERED');
+create type caas_status as enum ('INIT', 'PROVISIONING', 'AWAITING_RUNNING', 'RUNNING', 'DEPROVISIONING', 'AWAITING_STOPPED', 'STOPPED', 'ERROR', 'NOT_FOUND');
+
+alter table "connector"
+ add column broker_registration_status connector_broker_registration_status not null default 'UNREGISTERED',
+ add column management_url text,
+ add column endpoint_url text,
+ add column jwks_url text,
+ add column caas_status caas_status,
+ alter column provider_mds_id drop not null,
+ alter column url drop not null;
+
+alter table "connector"
+ rename column url to frontend_url;
+
+-- Fallback in case someone tries to migrate from 0.x to 1.0
+update "connector"
+set management_url = frontend_url || '/api/management' where management_url is null;
+
+update "connector"
+set endpoint_url = frontend_url || '/api/dsp' where endpoint_url is null;
+
+-- Organization id type
+create type organization_legal_id_type as enum ('TAX_ID', 'COMMERCE_REGISTER_INFO');
+alter table "organization"
+ add column legal_id_type organization_legal_id_type,
+ add column description text,
+ alter column address drop not null,
+ alter column url drop not null;
+update organization set legal_id_type = 'TAX_ID' where organization.tax_id is not null;
+update organization set legal_id_type = 'COMMERCE_REGISTER_INFO' where organization.commerce_register_number is not null and tax_id is null;
+
+-- New registration flow
+alter type user_registration_status add value 'ONBOARDING' after 'CREATED';
+alter type organization_registration_status add value 'ONBOARDING' after 'INVITED';
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V6__AP_Release_1_0_0.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V6__AP_Release_1_0_0.sql
new file mode 100644
index 000000000..1aef70850
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V6__AP_Release_1_0_0.sql
@@ -0,0 +1,39 @@
+-- Edit Enum: Remove UserRegistrationStatus.CREATED, UserRegistrationStatus.FIRST_USER
+delete
+from "user"
+where registration_status = 'CREATED'
+ or registration_status = 'FIRST_USER';
+
+create type tmp_enum AS ENUM ('INVITED', 'ONBOARDING', 'PENDING', 'ACTIVE', 'REJECTED', 'DEACTIVATED');
+
+alter table "user"
+ alter column registration_status type tmp_enum
+ using (registration_status::text::tmp_enum);
+
+drop type user_registration_status;
+
+alter type tmp_enum rename to user_registration_status;
+
+-- Component & Connector online status
+create type component_type as enum ('BROKER', 'DAPS', 'LOGGING_HOUSE');
+create type component_online_status as enum ('UP', 'DOWN', 'PENDING', 'MAINTENANCE');
+create table "component_downtimes"
+(
+ component component_type not null,
+ status component_online_status not null,
+ environment text not null,
+ time_stamp timestamp with time zone not null,
+ primary key (component, environment, time_stamp)
+);
+create index component_downtimes_time_stamp_index on "component_downtimes" (time_stamp);
+
+create type connector_uptime_status as enum ('UP', 'DOWN', 'DEAD');
+create table "connector_downtimes"
+(
+ connector_id text not null,
+ status connector_uptime_status not null,
+ environment text not null,
+ time_stamp timestamp with time zone not null,
+ primary key (connector_id, time_stamp)
+);
+create index connector_downtimes_time_stamp_index on "connector_downtimes" (time_stamp);
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V7__Schema_Extension.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V7__Schema_Extension.sql
new file mode 100644
index 000000000..0818d9c03
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V7__Schema_Extension.sql
@@ -0,0 +1,3 @@
+-- Organization
+alter table "organization"
+ add column industry text;
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V8__Schema_Extension.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V8__Schema_Extension.sql
new file mode 100644
index 000000000..4a9e7b810
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V8__Schema_Extension.sql
@@ -0,0 +1,4 @@
+-- Fix invalid DB state
+delete
+from "user"
+where organization_mds_id is null;
diff --git a/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V9__Broker_Integration.sql b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V9__Broker_Integration.sql
new file mode 100644
index 000000000..a93987578
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-db/src/main/resources/db-crawler/migration/V9__Broker_Integration.sql
@@ -0,0 +1,107 @@
+create collation if not exists alphanumeric_with_natural_sort (provider = icu, locale = 'en-u-kn-true');
+
+-- Creating missing enums
+create type connector_online_status as enum ('ONLINE', 'OFFLINE', 'DEAD');
+create type connector_data_offers_exceeded as enum ('OK', 'EXCEEDED');
+create type connector_contract_offers_exceeded as enum ('OK', 'EXCEEDED');
+create type measurement_type as enum ('CONNECTOR_REFRESH');
+create type measurement_error_status as enum ('ERROR', 'OK');
+create type crawler_event_status as enum ('OK', 'ERROR');
+create type crawler_event_type as enum (
+ 'CONNECTOR_UPDATED',
+ 'CONNECTOR_STATUS_CHANGE_ONLINE',
+ 'CONNECTOR_STATUS_CHANGE_OFFLINE',
+ 'CONNECTOR_STATUS_CHANGE_FORCE_DELETED',
+ 'CONTRACT_OFFER_UPDATED',
+ 'CONTRACT_OFFER_CLICK',
+ 'CONNECTOR_DATA_OFFER_LIMIT_EXCEEDED',
+ 'CONNECTOR_DATA_OFFER_LIMIT_OK',
+ 'CONNECTOR_CONTRACT_OFFER_LIMIT_EXCEEDED',
+ 'CONNECTOR_CONTRACT_OFFER_LIMIT_OK',
+ 'CONNECTOR_KILLED_DUE_TO_OFFLINE_FOR_TOO_LONG',
+ 'CONNECTOR_DELETED');
+
+-- Adding missing columns from the crawler to the connector table
+alter table connector
+ add column last_refresh_attempt_at timestamp with time zone,
+ add column last_successful_refresh_at timestamp with time zone,
+ add column online_status connector_online_status not null default 'OFFLINE',
+ add column data_offers_exceeded connector_data_offers_exceeded not null default 'OK',
+ add column contract_offers_exceeded connector_contract_offers_exceeded not null default 'OK';
+
+-- Data offers, additionally keyed by env ID
+create table data_offer
+(
+ connector_id text not null,
+ asset_id text not null,
+ ui_asset_json jsonb not null,
+ created_at timestamp with time zone not null,
+ updated_at timestamp with time zone,
+ asset_title text collate alphanumeric_with_natural_sort not null,
+ description_no_markdown text not null default ''::text,
+ short_description_no_markdown text not null default ''::text,
+ data_category text not null default ''::text,
+ data_subcategory text not null default ''::text,
+ data_model text not null default ''::text,
+ transport_mode text not null default ''::text,
+ geo_reference_method text not null default ''::text,
+ keywords text[] not null default '{}'::text[],
+ keywords_comma_joined text not null default ''::text,
+ version text not null default ''::text,
+ primary key (connector_id, asset_id),
+ constraint data_offer_connector_fkey foreign key (connector_id) references connector (connector_id)
+);
+
+-- Data offer Viewcount
+create table data_offer_view_count
+(
+ id serial primary key,
+ connector_id text not null,
+ asset_id text not null,
+ date timestamp with time zone not null
+);
+
+create index data_offer_view_count_speedup on data_offer_view_count (connector_id, asset_id);
+
+-- Contract offers, additionally keyed by env ID
+create table contract_offer
+(
+ contract_offer_id text not null,
+ connector_id text not null,
+ asset_id text not null,
+ ui_policy_json jsonb not null,
+ created_at timestamp with time zone not null,
+ updated_at timestamp with time zone,
+ primary key (contract_offer_id, connector_id, asset_id),
+ constraint contract_offer_connector_fkey foreign key (connector_id) references connector (connector_id),
+ constraint contract_offer_data_offer_fkey foreign key (connector_id, asset_id) references data_offer (connector_id, asset_id)
+);
+
+-- Event Log
+create table crawler_event_log
+(
+ id uuid not null primary key,
+ environment text not null,
+ created_at timestamp with time zone not null,
+ user_message text not null,
+ event crawler_event_type not null,
+ event_status crawler_event_status not null,
+ connector_id text,
+ asset_id text,
+ error_stack text
+);
+create index crawler_event_log_speedup on crawler_event_log (environment, connector_id, asset_id, event_status);
+
+-- Crawling exec time measurements
+create table crawler_execution_time_measurement
+(
+ id uuid not null primary key,
+ environment text not null,
+ connector_id text not null,
+ created_at timestamp with time zone not null,
+ duration_in_ms bigint not null,
+ type measurement_type not null,
+ error_status measurement_error_status not null
+);
+
+
diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts
new file mode 100644
index 000000000..6cc40c67d
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/build.gradle.kts
@@ -0,0 +1,44 @@
+plugins {
+ `java-library`
+}
+
+dependencies {
+ compileOnly(project(":launchers:connectors:catalog-crawler-dev"))
+ compileOnly(project(":launchers:connectors:sovity-dev"))
+
+ testAnnotationProcessor(libs.lombok)
+ testCompileOnly(libs.lombok)
+
+ testImplementation(project(":utils:versions"))
+ testImplementation(project(":utils:test-connector-remote"))
+ testImplementation(project(":utils:json-and-jsonld-utils"))
+ testImplementation(project(":extensions:catalog-crawler:catalog-crawler-db"))
+ testImplementation(project(":extensions:wrapper:clients:java-client"))
+ testImplementation(project(":extensions:catalog-crawler:catalog-crawler"))
+
+ testImplementation(libs.assertj.core)
+ testImplementation(libs.mockito.core)
+ testImplementation(libs.mockito.inline)
+ testImplementation(libs.edc.junit)
+ testImplementation(libs.restAssured.restAssured)
+ testImplementation(libs.testcontainers.testcontainers)
+ testImplementation(libs.flyway.core)
+ testImplementation(libs.testcontainers.junitJupiter)
+ testImplementation(libs.testcontainers.postgresql)
+ testImplementation(libs.junit.api)
+ testImplementation(libs.jsonAssert)
+ testRuntimeOnly(libs.junit.engine)
+}
+
+tasks.getByName("test") {
+ useJUnitPlatform()
+ maxParallelForks = 1
+}
+
+publishing {
+ publications {
+ create(project.name) {
+ from(components["java"])
+ }
+ }
+}
diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java
new file mode 100644
index 000000000..3fedd0961
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerE2eTest.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - init
+ */
+
+package de.sovity.edc.ext.catalog.crawler;
+
+import de.sovity.edc.client.EdcClient;
+import de.sovity.edc.client.gen.model.ContractDefinitionRequest;
+import de.sovity.edc.client.gen.model.DataSourceType;
+import de.sovity.edc.client.gen.model.OperatorDto;
+import de.sovity.edc.client.gen.model.PolicyDefinitionCreateRequest;
+import de.sovity.edc.client.gen.model.UiAssetCreateRequest;
+import de.sovity.edc.client.gen.model.UiCriterion;
+import de.sovity.edc.client.gen.model.UiCriterionLiteral;
+import de.sovity.edc.client.gen.model.UiCriterionLiteralType;
+import de.sovity.edc.client.gen.model.UiCriterionOperator;
+import de.sovity.edc.client.gen.model.UiDataSource;
+import de.sovity.edc.client.gen.model.UiDataSourceHttpData;
+import de.sovity.edc.client.gen.model.UiPolicyConstraint;
+import de.sovity.edc.client.gen.model.UiPolicyCreateRequest;
+import de.sovity.edc.client.gen.model.UiPolicyLiteral;
+import de.sovity.edc.client.gen.model.UiPolicyLiteralType;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus;
+import de.sovity.edc.ext.catalog.crawler.utils.CrawlerDbAccess;
+import de.sovity.edc.ext.catalog.crawler.utils.TestData;
+import de.sovity.edc.extension.e2e.connector.config.ConnectorConfig;
+import de.sovity.edc.extension.e2e.db.EdcRuntimeExtensionWithTestDatabase;
+import de.sovity.edc.utils.jsonld.vocab.Prop;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ThrowingRunnable;
+import org.jooq.DSLContext;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import java.time.OffsetDateTime;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import static de.sovity.edc.extension.e2e.connector.config.ConnectorConfigFactory.forTestDatabase;
+import static de.sovity.edc.extension.e2e.connector.config.ConnectorConfigFactory.getFreePortRange;
+import static de.sovity.edc.extension.e2e.connector.config.api.EdcApiConfigFactory.configureApi;
+import static org.assertj.core.api.Assertions.assertThat;
+
+class CrawlerE2eTest {
+ private static ConnectorConfig connectorConfig;
+ private static EdcClient connectorClient;
+
+ @RegisterExtension
+ static EdcRuntimeExtensionWithTestDatabase providerExtension = new EdcRuntimeExtensionWithTestDatabase(
+ ":launchers:connectors:sovity-dev",
+ "provider",
+ testDatabase -> {
+ connectorConfig = forTestDatabase("MDSL1234XX.C1234XX", testDatabase);
+ connectorClient = EdcClient.builder()
+ .managementApiUrl(connectorConfig.getManagementEndpoint().getUri().toString())
+ .managementApiKey(connectorConfig.getProperties().get("edc.api.auth.key"))
+ .build();
+ return connectorConfig.getProperties();
+ }
+ );
+
+ @RegisterExtension
+ static EdcRuntimeExtensionWithTestDatabase crawlerExtension = new EdcRuntimeExtensionWithTestDatabase(
+ ":launchers:connectors:catalog-crawler-dev",
+ "crawler",
+ testDatabase -> {
+ var firstPort = getFreePortRange(5);
+
+ var props = new HashMap();
+ props.put("edc.participant.id", "broker");
+ props.put(CrawlerExtension.EXTENSION_ENABLED, "true");
+ props.put(CrawlerExtension.ENVIRONMENT_ID, "test");
+ props.put(CrawlerExtension.JDBC_URL, testDatabase.getJdbcCredentials().jdbcUrl());
+ props.put(CrawlerExtension.JDBC_USER, testDatabase.getJdbcCredentials().jdbcUser());
+ props.put(CrawlerExtension.JDBC_PASSWORD, testDatabase.getJdbcCredentials().jdbcPassword());
+ props.put(CrawlerExtension.DB_CONNECTION_POOL_SIZE, "30");
+ props.put(CrawlerExtension.DB_CONNECTION_TIMEOUT_IN_MS, "1000");
+ props.put(CrawlerExtension.DB_MIGRATE, "true");
+ props.put(CrawlerExtension.DB_CLEAN, "true");
+ props.put(CrawlerExtension.DB_CLEAN_ENABLED, "true");
+ props.put(CrawlerExtension.DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS, "classpath:db-crawler/migration-test-utils");
+ props.put(CrawlerExtension.NUM_THREADS, "2");
+ props.put(CrawlerExtension.MAX_DATA_OFFERS_PER_CONNECTOR, "100");
+ props.put(CrawlerExtension.MAX_CONTRACT_OFFERS_PER_DATA_OFFER, "100");
+ props.putAll(configureApi(firstPort, "managementApiKey").getProperties());
+
+ var everySeconds = "* * * * * ?";
+ props.put(CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH, everySeconds);
+ props.put(CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH, everySeconds);
+ props.put(CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, everySeconds);
+ props.put(CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, everySeconds);
+ props.put(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, "P1D");
+
+ return props;
+ }
+ );
+
+ private final String dataOfferId = "my-data-offer";
+
+ @Test
+ void crawlSingleDataOffer() {
+ // arrange
+ createPolicy();
+ createAsset();
+ createContractDefinition();
+
+ var connectorRef = new ConnectorRef(
+ "MDSL1234XX.C1234XX",
+ "test",
+ "My Org",
+ "MDSL1234XX",
+ connectorConfig.getProtocolEndpoint().getUri().toString()
+ );
+
+ crawlerTransaction(dsl -> {
+ TestData.insertConnector(dsl, connectorRef, connectorRecord -> {
+ connectorRecord.setOnlineStatus(ConnectorOnlineStatus.OFFLINE);
+ connectorRecord.setCreatedAt(OffsetDateTime.now());
+ });
+ });
+
+ // act / await crawl
+ Awaitility.await().atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(mapExceptionsToAssertionError(() ->
+ crawlerTransaction(dsl -> assertDataOfferInCatalog(dsl, connectorRef))));
+ }
+
+ private void assertDataOfferInCatalog(DSLContext dsl, ConnectorRef connectorRef) {
+ var c = Tables.CONNECTOR;
+ var connector = dsl.fetchOne(c, c.CONNECTOR_ID.eq(connectorRef.getConnectorId()));
+ assertThat(connector).isNotNull();
+ assertThat(connector.getOnlineStatus()).isEqualTo(ConnectorOnlineStatus.ONLINE);
+
+ var d = Tables.DATA_OFFER;
+ var dataOffers = dsl.fetch(d, d.CONNECTOR_ID.eq(connectorRef.getConnectorId()));
+ assertThat(dataOffers).hasSize(1);
+ assertThat(dataOffers.get(0).getAssetId()).isEqualTo(dataOfferId);
+ assertThat(dataOffers.get(0).getAssetTitle()).isEqualTo("My Data Offer");
+ }
+
+ private void createAsset() {
+ var asset = UiAssetCreateRequest.builder()
+ .id(dataOfferId)
+ .title("My Data Offer")
+ .description("Example Data Offer.")
+ .version("2023-11")
+ .language("EN")
+ .publisherHomepage("https://my-department.my-org.com/my-data-offer")
+ .licenseUrl("https://my-department.my-org.com/my-data-offer#license")
+ .dataSource(UiDataSource.builder()
+ .type(DataSourceType.HTTP_DATA)
+ .httpData(UiDataSourceHttpData.builder()
+ .baseUrl("http://0.0.0.0")
+ .build())
+ .build())
+ .build();
+
+ connectorClient.uiApi().createAsset(asset);
+ }
+
+ private void createPolicy() {
+ var afterYesterday = UiPolicyConstraint.builder()
+ .left("POLICY_EVALUATION_TIME")
+ .operator(OperatorDto.GT)
+ .right(UiPolicyLiteral.builder()
+ .type(UiPolicyLiteralType.STRING)
+ .value(OffsetDateTime.now().minusDays(1).toString())
+ .build())
+ .build();
+
+ var beforeTomorrow = UiPolicyConstraint.builder()
+ .left("POLICY_EVALUATION_TIME")
+ .operator(OperatorDto.LT)
+ .right(UiPolicyLiteral.builder()
+ .type(UiPolicyLiteralType.STRING)
+ .value(OffsetDateTime.now().plusDays(1).toString())
+ .build())
+ .build();
+
+ var policyDefinition = PolicyDefinitionCreateRequest.builder()
+ .policyDefinitionId(dataOfferId)
+ .policy(UiPolicyCreateRequest.builder()
+ .constraints(List.of(afterYesterday, beforeTomorrow))
+ .build())
+ .build();
+
+ connectorClient.uiApi().createPolicyDefinition(policyDefinition);
+ }
+
+ private void createContractDefinition() {
+ var contractDefinition = ContractDefinitionRequest.builder()
+ .contractDefinitionId(dataOfferId)
+ .accessPolicyId(dataOfferId)
+ .contractPolicyId(dataOfferId)
+ .assetSelector(List.of(UiCriterion.builder()
+ .operandLeft(Prop.Edc.ID)
+ .operator(UiCriterionOperator.EQ)
+ .operandRight(UiCriterionLiteral.builder()
+ .type(UiCriterionLiteralType.VALUE)
+ .value(dataOfferId)
+ .build())
+ .build()))
+ .build();
+
+ connectorClient.uiApi().createContractDefinition(contractDefinition);
+ }
+
+ private void crawlerTransaction(Consumer withDsl) {
+ CrawlerDbAccess.transaction(crawlerExtension.getTestDatabase(), withDsl);
+ }
+
+ private ThrowingRunnable mapExceptionsToAssertionError(ThrowingRunnable runnable) {
+ return () -> {
+ try {
+ runnable.run();
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ };
+ }
+}
diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CrawlerDbAccess.java b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CrawlerDbAccess.java
new file mode 100644
index 000000000..3dd2751db
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CrawlerDbAccess.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.utils;
+
+import de.sovity.edc.extension.e2e.db.TestDatabase;
+import lombok.experimental.UtilityClass;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+
+import java.util.function.Consumer;
+
+@UtilityClass
+public class CrawlerDbAccess {
+
+ public static void transaction(TestDatabase testDatabase, Consumer consumer) {
+ var credentials = testDatabase.getJdbcCredentials();
+ try (var dslContext = DSL.using(credentials.jdbcUrl(), credentials.jdbcUser(), credentials.jdbcPassword())) {
+ dslContext.transaction(configuration -> {
+ var dsl = DSL.using(configuration);
+ consumer.accept(dsl);
+ });
+ }
+ }
+}
diff --git a/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/TestData.java b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/TestData.java
new file mode 100644
index 000000000..77e056a98
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/TestData.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.utils;
+
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord;
+import lombok.experimental.UtilityClass;
+import org.jooq.DSLContext;
+
+import java.time.OffsetDateTime;
+import java.util.function.Consumer;
+
+@UtilityClass
+public class TestData {
+
+ public static void insertConnector(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ Consumer applier
+ ) {
+ var organization = dsl.newRecord(Tables.ORGANIZATION);
+ organization.setMdsId(connectorRef.getOrganizationId());
+ organization.setName(connectorRef.getOrganizationLegalName());
+ organization.insert();
+
+ var connector = dsl.newRecord(Tables.CONNECTOR);
+ connector.setEnvironment(connectorRef.getEnvironmentId());
+ connector.setMdsId(connectorRef.getOrganizationId());
+ connector.setConnectorId(connectorRef.getConnectorId());
+ connector.setName(connectorRef.getConnectorId() + " Name");
+ connector.setEndpointUrl(connectorRef.getEndpoint());
+ connector.setOnlineStatus(ConnectorOnlineStatus.OFFLINE);
+ connector.setLastRefreshAttemptAt(null);
+ connector.setLastSuccessfulRefreshAt(null);
+ connector.setCreatedAt(OffsetDateTime.now());
+ connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
+ connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
+ applier.accept(connector);
+ connector.insert();
+ }
+}
diff --git a/extensions/broker-server/src/test/resources/logging.properties b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/resources/logging.properties
similarity index 88%
rename from extensions/broker-server/src/test/resources/logging.properties
rename to extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/resources/logging.properties
index d2212b2a2..471bd20d6 100644
--- a/extensions/broker-server/src/test/resources/logging.properties
+++ b/extensions/catalog-crawler/catalog-crawler-e2e-test/src/test/resources/logging.properties
@@ -3,4 +3,4 @@ org.eclipse.edc.level=ALL
handlers=java.util.logging.ConsoleHandler
java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
java.util.logging.ConsoleHandler.level=ALL
-java.util.logging.SimpleFormatter.format=[%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS] [%4$-7s] %5$s%6$s%n
\ No newline at end of file
+java.util.logging.SimpleFormatter.format=[%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS] [%4$-7s] %5$s%6$s%n
diff --git a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts
new file mode 100644
index 000000000..2c376a085
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts
@@ -0,0 +1,19 @@
+plugins {
+ `java-library`
+}
+
+dependencies {
+ // A minimal EDC that can request catalogs
+ api(libs.edc.controlPlaneCore)
+ api(libs.edc.dataPlaneSelectorCore)
+ api(libs.edc.configurationFilesystem)
+ api(libs.edc.controlPlaneAggregateServices)
+ api(libs.edc.http)
+ api(libs.edc.dsp)
+ api(libs.edc.jsonLd)
+
+ // Data Catalog Crawler
+ api(project(":extensions:catalog-crawler:catalog-crawler"))
+}
+
+group = libs.versions.sovityEdcGroup.get()
diff --git a/extensions/broker-server/build.gradle.kts b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts
similarity index 55%
rename from extensions/broker-server/build.gradle.kts
rename to extensions/catalog-crawler/catalog-crawler/build.gradle.kts
index c53445e15..b63b7004f 100644
--- a/extensions/broker-server/build.gradle.kts
+++ b/extensions/catalog-crawler/catalog-crawler/build.gradle.kts
@@ -2,65 +2,44 @@ plugins {
`java-library`
}
-configurations.all {
- resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
-}
-
dependencies {
annotationProcessor(libs.lombok)
compileOnly(libs.lombok)
- implementation(libs.apache.commonsLang)
-
- api(project(":utils:catalog-parser"))
- api(project(":utils:json-and-jsonld-utils"))
- api(project(":extensions:wrapper:wrapper-common-mappers"))
-
implementation(libs.edc.controlPlaneSpi)
implementation(libs.edc.managementApiConfiguration)
- api(project(":extensions:broker-server-postgres-flyway-jooq"))
- implementation(project(":extensions:broker-server-api:api"))
+ implementation(libs.quartz.quartz)
+ implementation(libs.apache.commonsLang)
implementation(project(":utils:versions"))
- implementation(libs.okhttp.okhttp)
+ api(project(":utils:catalog-parser"))
+ api(project(":utils:json-and-jsonld-utils"))
+ api(project(":extensions:wrapper:wrapper-common-mappers"))
+ api(project(":extensions:catalog-crawler:catalog-crawler-db"))
+ api(project(":extensions:postgres-flyway-core"))
testAnnotationProcessor(libs.lombok)
testCompileOnly(libs.lombok)
- testImplementation(project(":extensions:wrapper:clients:java-client"))
- testImplementation(project(":extensions:sovity-edc-extensions-package"))
+ testImplementation(project(":utils:test-connector-remote"))
testImplementation(libs.assertj.core)
testImplementation(libs.mockito.core)
testImplementation(libs.mockito.inline)
- testImplementation(libs.edc.controlPlaneCore)
- testImplementation(libs.edc.dataPlaneSelectorCore)
- testImplementation(libs.edc.junit)
- testImplementation(libs.edc.http)
- testImplementation(libs.edc.iamMock)
- testImplementation(libs.edc.dsp)
- testImplementation(libs.edc.jsonLd)
- testImplementation(libs.edc.monitorJdkLogger)
- testImplementation(libs.edc.configurationFilesystem)
- testImplementation(project(":extensions:broker-server-api:client"))
testImplementation(libs.restAssured.restAssured)
testImplementation(libs.testcontainers.testcontainers)
+ testImplementation(libs.flyway.core)
testImplementation(libs.testcontainers.junitJupiter)
testImplementation(libs.testcontainers.postgresql)
testImplementation(libs.junit.api)
testImplementation(libs.jsonAssert)
testRuntimeOnly(libs.junit.engine)
-
- implementation(libs.quartz.quartz)
}
tasks.getByName("test") {
useJUnitPlatform()
+ maxParallelForks = 1
}
-tasks.register("prepareKotlinBuildScriptModel") {}
-
-group = libs.versions.sovityBrokerServerGroup.get()
-
publishing {
publications {
create(project.name) {
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java
new file mode 100644
index 000000000..290c5c2c3
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler;
+
+import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry;
+import org.eclipse.edc.connector.spi.catalog.CatalogService;
+import org.eclipse.edc.jsonld.spi.JsonLd;
+import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.runtime.metamodel.annotation.Provides;
+import org.eclipse.edc.runtime.metamodel.annotation.Setting;
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+import org.eclipse.edc.spi.types.TypeManager;
+
+import static de.sovity.edc.ext.catalog.crawler.orchestration.config.EdcConfigPropertyUtils.toEdcProp;
+
+@Provides({CrawlerExtensionContext.class})
+public class CrawlerExtension implements ServiceExtension {
+
+ public static final String EXTENSION_NAME = "Authority Portal Data Catalog Crawler";
+
+ @Setting(required = true)
+ public static final String EXTENSION_ENABLED = toEdcProp("CRAWLER_EXTENSION_ENABLED");
+
+ @Setting(required = true)
+ public static final String ENVIRONMENT_ID = toEdcProp("CRAWLER_ENVIRONMENT_ID");
+
+ @Setting(required = true)
+ public static final String JDBC_URL = toEdcProp("CRAWLER_DB_JDBC_URL");
+
+ @Setting(required = true)
+ public static final String JDBC_USER = toEdcProp("CRAWLER_DB_JDBC_USER");
+
+ @Setting(required = true)
+ public static final String JDBC_PASSWORD = toEdcProp("CRAWLER_DB_JDBC_PASSWORD");
+
+ @Setting
+ public static final String DB_CONNECTION_POOL_SIZE = toEdcProp("CRAWLER_DB_CONNECTION_POOL_SIZE");
+
+ @Setting
+ public static final String DB_CONNECTION_TIMEOUT_IN_MS = toEdcProp("CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS");
+
+ @Setting
+ public static final String DB_MIGRATE = toEdcProp("CRAWLER_DB_MIGRATE");
+
+ @Setting
+ public static final String DB_CLEAN = toEdcProp("CRAWLER_DB_CLEAN");
+
+ @Setting
+ public static final String DB_CLEAN_ENABLED = toEdcProp("CRAWLER_DB_CLEAN_ENABLED");
+
+ @Setting
+ public static final String DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS = toEdcProp("CRAWLER_DB_ADDITIONAL_FLYWAY_LOCATIONS");
+
+ @Setting
+ public static final String NUM_THREADS = toEdcProp("CRAWLER_NUM_THREADS");
+
+ @Setting
+ public static final String MAX_DATA_OFFERS_PER_CONNECTOR = toEdcProp("CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR");
+
+ @Setting
+ public static final String MAX_CONTRACT_OFFERS_PER_DATA_OFFER = toEdcProp("CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER");
+
+ @Setting
+ public static final String CRON_ONLINE_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH");
+
+ @Setting
+ public static final String CRON_OFFLINE_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH");
+
+ @Setting
+ public static final String CRON_DEAD_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_DEAD_CONNECTOR_REFRESH");
+
+ @Setting
+ public static final String SCHEDULED_KILL_OFFLINE_CONNECTORS = toEdcProp("CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS");
+ @Setting
+ public static final String KILL_OFFLINE_CONNECTORS_AFTER = toEdcProp("CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER");
+
+ @Inject
+ private TypeManager typeManager;
+
+ @Inject
+ private ManagementApiTypeTransformerRegistry typeTransformerRegistry;
+
+ @Inject
+ private JsonLd jsonLd;
+
+ @Inject
+ private CatalogService catalogService;
+
+ /**
+ * Manual Dependency Injection Result
+ */
+ private CrawlerExtensionContext services;
+
+ @Override
+ public String name() {
+ return EXTENSION_NAME;
+ }
+
+ @Override
+ public void initialize(ServiceExtensionContext context) {
+ if (!Boolean.TRUE.equals(context.getConfig().getBoolean(EXTENSION_ENABLED, false))) {
+ context.getMonitor().info("Crawler extension is disabled.");
+ return;
+ }
+
+ services = CrawlerExtensionContextBuilder.buildContext(
+ context.getConfig(),
+ context.getMonitor(),
+ typeManager,
+ typeTransformerRegistry,
+ jsonLd,
+ catalogService
+ );
+
+ // Provide access for the tests
+ context.registerService(CrawlerExtensionContext.class, services);
+ }
+
+ @Override
+ public void start() {
+ if (services == null) {
+ return;
+ }
+ services.crawlerInitializer().onStartup();
+ }
+
+ @Override
+ public void shutdown() {
+ if (services == null) {
+ return;
+ }
+ services.dataSource().close();
+ }
+}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java
new file mode 100644
index 000000000..264831fe8
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler;
+
+import com.zaxxer.hikari.HikariDataSource;
+import de.sovity.edc.ext.catalog.crawler.crawling.ConnectorCrawler;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogBuilder;
+import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory;
+import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater;
+import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
+
+
+/**
+ * Manual Dependency Injection result
+ *
+ * @param crawlerInitializer Startup Logic
+ */
+public record CrawlerExtensionContext(
+ CrawlerInitializer crawlerInitializer,
+ // Required for stopping connections on closing
+ HikariDataSource dataSource,
+ DslContextFactory dslContextFactory,
+
+ // Required for Integration Tests
+ ConnectorCrawler connectorCrawler,
+ PolicyMapper policyMapper,
+ FetchedCatalogBuilder catalogPatchBuilder,
+ DataOfferRecordUpdater dataOfferRecordUpdater
+) {
+}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java
new file mode 100644
index 000000000..2437dd751
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java
@@ -0,0 +1,313 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import de.sovity.edc.ext.catalog.crawler.crawling.ConnectorCrawler;
+import de.sovity.edc.ext.catalog.crawler.crawling.OfflineConnectorCleaner;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogBuilder;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogMappingUtils;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogService;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerExecutionTimeLogger;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.CatalogPatchBuilder;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateCatalogWriter;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateFailureWriter;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateSuccessWriter;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.DataOfferLimitsEnforcer;
+import de.sovity.edc.ext.catalog.crawler.dao.CatalogCleaner;
+import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier;
+import de.sovity.edc.ext.catalog.crawler.dao.config.DataSourceFactory;
+import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory;
+import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater;
+import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferRecordUpdater;
+import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater;
+import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfigFactory;
+import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueue;
+import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller;
+import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ThreadPool;
+import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ThreadPoolTaskQueue;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.DeadConnectorRefreshJob;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.OfflineConnectorCleanerJob;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.OfflineConnectorRefreshJob;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.OnlineConnectorRefreshJob;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.utils.CronJobRef;
+import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetEditRequestMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdBuilder;
+import de.sovity.edc.ext.wrapper.api.common.mappers.asset.AssetJsonLdParser;
+import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.AssetJsonLdUtils;
+import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.EdcPropertyUtils;
+import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder;
+import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.DataSourceMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpDataSourceMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.dataaddress.http.HttpHeaderMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.policy.AtomicConstraintMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.policy.ConstraintExtractor;
+import de.sovity.edc.ext.wrapper.api.common.mappers.policy.LiteralMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.policy.OperatorMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.policy.PolicyValidator;
+import de.sovity.edc.utils.catalog.DspCatalogService;
+import de.sovity.edc.utils.catalog.mapper.DspDataOfferBuilder;
+import lombok.NoArgsConstructor;
+import org.eclipse.edc.connector.spi.catalog.CatalogService;
+import org.eclipse.edc.jsonld.spi.JsonLd;
+import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.spi.CoreConstants;
+import org.eclipse.edc.spi.monitor.Monitor;
+import org.eclipse.edc.spi.system.configuration.Config;
+import org.eclipse.edc.spi.types.TypeManager;
+import org.eclipse.edc.transform.spi.TypeTransformerRegistry;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+
+
+/**
+ * Manual Dependency Injection (DYDI).
+ *
+ * We want to develop as Java Backend Development is done, but we have
+ * no CDI / DI Framework to rely on.
+ *
+ * EDC {@link Inject} only works in {@link CrawlerExtension}.
+ */
+@NoArgsConstructor(access = lombok.AccessLevel.PRIVATE)
+public class CrawlerExtensionContextBuilder {
+
+ public static CrawlerExtensionContext buildContext(
+ Config config,
+ Monitor monitor,
+ TypeManager typeManager,
+ TypeTransformerRegistry typeTransformerRegistry,
+ JsonLd jsonLd,
+ CatalogService catalogService
+ ) {
+ // Config
+ var crawlerConfigFactory = new CrawlerConfigFactory(config);
+ var crawlerConfig = crawlerConfigFactory.buildCrawlerConfig();
+
+ // DB
+ var dataSourceFactory = new DataSourceFactory(config);
+ var dataSource = dataSourceFactory.newDataSource();
+ var flywayService = new FlywayService(config, monitor, dataSource);
+ flywayService.validateOrMigrateInTests();
+
+ // Dao
+ var dataOfferQueries = new DataOfferQueries();
+ var dslContextFactory = new DslContextFactory(dataSource);
+ var connectorQueries = new ConnectorQueries(crawlerConfig);
+
+ // Services
+ var objectMapperJsonLd = getJsonLdObjectMapper(typeManager);
+ var assetMapper = newAssetMapper(typeTransformerRegistry, jsonLd);
+ var policyMapper = newPolicyMapper(typeTransformerRegistry, objectMapperJsonLd);
+ var crawlerEventLogger = new CrawlerEventLogger();
+ var crawlerExecutionTimeLogger = new CrawlerExecutionTimeLogger();
+ var dataOfferMappingUtils = new FetchedCatalogMappingUtils(
+ policyMapper,
+ assetMapper,
+ objectMapperJsonLd
+ );
+ var contractOfferRecordUpdater = new ContractOfferRecordUpdater();
+ var shortDescriptionBuilder = new ShortDescriptionBuilder();
+ var dataOfferRecordUpdater = new DataOfferRecordUpdater(shortDescriptionBuilder);
+ var contractOfferQueries = new ContractOfferQueries();
+ var dataOfferLimitsEnforcer = new DataOfferLimitsEnforcer(crawlerConfig, crawlerEventLogger);
+ var dataOfferPatchBuilder = new CatalogPatchBuilder(
+ contractOfferQueries,
+ dataOfferQueries,
+ dataOfferRecordUpdater,
+ contractOfferRecordUpdater
+ );
+ var dataOfferPatchApplier = new CatalogPatchApplier();
+ var dataOfferWriter = new ConnectorUpdateCatalogWriter(dataOfferPatchBuilder, dataOfferPatchApplier);
+ var connectorUpdateSuccessWriter = new ConnectorUpdateSuccessWriter(
+ crawlerEventLogger,
+ dataOfferWriter,
+ dataOfferLimitsEnforcer
+ );
+ var fetchedDataOfferBuilder = new FetchedCatalogBuilder(dataOfferMappingUtils);
+ var dspDataOfferBuilder = new DspDataOfferBuilder(jsonLd);
+ var dspCatalogService = new DspCatalogService(
+ catalogService,
+ dspDataOfferBuilder
+ );
+ var dataOfferFetcher = new FetchedCatalogService(dspCatalogService, fetchedDataOfferBuilder);
+ var connectorUpdateFailureWriter = new ConnectorUpdateFailureWriter(crawlerEventLogger, monitor);
+ var connectorUpdater = new ConnectorCrawler(
+ dataOfferFetcher,
+ connectorUpdateSuccessWriter,
+ connectorUpdateFailureWriter,
+ connectorQueries,
+ dslContextFactory,
+ monitor,
+ crawlerExecutionTimeLogger
+ );
+
+ var threadPoolTaskQueue = new ThreadPoolTaskQueue();
+ var threadPool = new ThreadPool(threadPoolTaskQueue, crawlerConfig, monitor);
+ var connectorQueue = new ConnectorQueue(connectorUpdater, threadPool);
+ var connectorQueueFiller = new ConnectorQueueFiller(connectorQueue, connectorQueries);
+ var connectorStatusUpdater = new ConnectorStatusUpdater();
+ var catalogCleaner = new CatalogCleaner();
+ var offlineConnectorCleaner = new OfflineConnectorCleaner(
+ crawlerConfig,
+ connectorQueries,
+ crawlerEventLogger,
+ connectorStatusUpdater,
+ catalogCleaner
+ );
+
+ // Schedules
+ List> jobs = List.of(
+ getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller),
+ getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller),
+ getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller),
+ getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner)
+ );
+
+ // Startup
+ var quartzScheduleInitializer = new QuartzScheduleInitializer(config, monitor, jobs);
+ var crawlerInitializer = new CrawlerInitializer(quartzScheduleInitializer);
+
+ return new CrawlerExtensionContext(
+ crawlerInitializer,
+ dataSource,
+ dslContextFactory,
+ connectorUpdater,
+ policyMapper,
+ fetchedDataOfferBuilder,
+ dataOfferRecordUpdater
+ );
+ }
+
+ @NotNull
+ private static PolicyMapper newPolicyMapper(TypeTransformerRegistry typeTransformerRegistry, ObjectMapper objectMapperJsonLd) {
+ var operatorMapper = new OperatorMapper();
+ var literalMapper = new LiteralMapper(
+ objectMapperJsonLd
+ );
+ var atomicConstraintMapper = new AtomicConstraintMapper(
+ literalMapper,
+ operatorMapper
+ );
+ var policyValidator = new PolicyValidator();
+ var constraintExtractor = new ConstraintExtractor(
+ policyValidator,
+ atomicConstraintMapper
+ );
+ return new PolicyMapper(
+ constraintExtractor,
+ atomicConstraintMapper,
+ typeTransformerRegistry
+ );
+ }
+
+ @NotNull
+ private static AssetMapper newAssetMapper(
+ TypeTransformerRegistry typeTransformerRegistry,
+ JsonLd jsonLd
+ ) {
+ var edcPropertyUtils = new EdcPropertyUtils();
+ var assetJsonLdUtils = new AssetJsonLdUtils();
+ var assetEditRequestMapper = new AssetEditRequestMapper();
+ var shortDescriptionBuilder = new ShortDescriptionBuilder();
+ var assetJsonLdParser = new AssetJsonLdParser(
+ assetJsonLdUtils,
+ shortDescriptionBuilder,
+ endpoint -> false
+ );
+ var httpHeaderMapper = new HttpHeaderMapper();
+ var httpDataSourceMapper = new HttpDataSourceMapper(httpHeaderMapper);
+ var dataSourceMapper = new DataSourceMapper(
+ edcPropertyUtils,
+ httpDataSourceMapper
+ );
+ var assetJsonLdBuilder = new AssetJsonLdBuilder(
+ dataSourceMapper,
+ assetJsonLdParser,
+ assetEditRequestMapper
+ );
+ return new AssetMapper(
+ typeTransformerRegistry,
+ assetJsonLdBuilder,
+ assetJsonLdParser,
+ jsonLd
+ );
+ }
+
+ @NotNull
+ private static CronJobRef getOfflineConnectorCleanerCronJob(DslContextFactory dslContextFactory,
+ OfflineConnectorCleaner offlineConnectorCleaner) {
+ return new CronJobRef<>(
+ CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS,
+ OfflineConnectorCleanerJob.class,
+ () -> new OfflineConnectorCleanerJob(dslContextFactory, offlineConnectorCleaner)
+ );
+ }
+
+ @NotNull
+ private static CronJobRef getOnlineConnectorRefreshCronJob(
+ DslContextFactory dslContextFactory,
+ ConnectorQueueFiller connectorQueueFiller
+ ) {
+ return new CronJobRef<>(
+ CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH,
+ OnlineConnectorRefreshJob.class,
+ () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller)
+ );
+ }
+
+ @NotNull
+ private static CronJobRef getOfflineConnectorRefreshCronJob(
+ DslContextFactory dslContextFactory,
+ ConnectorQueueFiller connectorQueueFiller
+ ) {
+ return new CronJobRef<>(
+ CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH,
+ OfflineConnectorRefreshJob.class,
+ () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller)
+ );
+ }
+
+ @NotNull
+ private static CronJobRef getDeadConnectorRefreshCronJob(DslContextFactory dslContextFactory,
+ ConnectorQueueFiller connectorQueueFiller) {
+ return new CronJobRef<>(
+ CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH,
+ DeadConnectorRefreshJob.class,
+ () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller)
+ );
+ }
+
+ private static ObjectMapper getJsonLdObjectMapper(TypeManager typeManager) {
+ var objectMapper = typeManager.getMapper(CoreConstants.JSON_LD);
+
+ // Fixes Dates in JSON-LD Object Mapper
+ // The Core EDC uses longs over OffsetDateTime, so they never fixed the date format
+ objectMapper.registerModule(new JavaTimeModule());
+ objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+
+ return objectMapper;
+ }
+}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/BrokerServerInitializer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java
similarity index 56%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/BrokerServerInitializer.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java
index 4f1ca59f2..719ace1c3 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/BrokerServerInitializer.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java
@@ -12,20 +12,16 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services;
+package de.sovity.edc.ext.catalog.crawler;
-import de.sovity.edc.ext.brokerserver.db.DslContextFactory;
-import de.sovity.edc.ext.brokerserver.services.schedules.QuartzScheduleInitializer;
+import de.sovity.edc.ext.catalog.crawler.orchestration.schedules.QuartzScheduleInitializer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
-public class BrokerServerInitializer {
- private final DslContextFactory dslContextFactory;
- private final KnownConnectorsInitializer knownConnectorsInitializer;
+public class CrawlerInitializer {
private final QuartzScheduleInitializer quartzScheduleInitializer;
public void onStartup() {
- dslContextFactory.transaction(knownConnectorsInitializer::addKnownConnectorsOnStartup);
quartzScheduleInitializer.startSchedules();
}
}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdater.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java
similarity index 53%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdater.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java
index 5a759fab0..dca422308 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdater.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java
@@ -12,14 +12,16 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing;
+package de.sovity.edc.ext.catalog.crawler.crawling;
-import de.sovity.edc.ext.brokerserver.dao.ConnectorQueries;
-import de.sovity.edc.ext.brokerserver.db.DslContextFactory;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.MeasurementErrorStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerExecutionTimeLogger;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.CatalogFetcher;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogService;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerExecutionTimeLogger;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateFailureWriter;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateSuccessWriter;
+import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementErrorStatus;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.time.StopWatch;
import org.eclipse.edc.spi.monitor.Monitor;
@@ -30,41 +32,41 @@
* Updates a single connector.
*/
@RequiredArgsConstructor
-public class ConnectorUpdater {
- private final CatalogFetcher catalogFetcher;
+public class ConnectorCrawler {
+ private final FetchedCatalogService fetchedCatalogService;
private final ConnectorUpdateSuccessWriter connectorUpdateSuccessWriter;
private final ConnectorUpdateFailureWriter connectorUpdateFailureWriter;
private final ConnectorQueries connectorQueries;
private final DslContextFactory dslContextFactory;
private final Monitor monitor;
- private final BrokerExecutionTimeLogger brokerExecutionTimeLogger;
+ private final CrawlerExecutionTimeLogger crawlerExecutionTimeLogger;
/**
* Updates single connector.
*
- * @param connectorEndpoint connector endpoint
+ * @param connectorRef connector
*/
- public void updateConnector(String connectorEndpoint) {
+ public void crawlConnector(ConnectorRef connectorRef) {
var executionTime = StopWatch.createStarted();
var failed = false;
try {
- monitor.info("Updating connector: " + connectorEndpoint);
+ monitor.info("Updating connector: " + connectorRef);
- var catalog = catalogFetcher.fetchCatalog(connectorEndpoint);
+ var catalog = fetchedCatalogService.fetchCatalog(connectorRef);
// Update connector in a single transaction
dslContextFactory.transaction(dsl -> {
- ConnectorRecord connectorRecord = connectorQueries.findByEndpoint(dsl, connectorEndpoint);
- connectorUpdateSuccessWriter.handleConnectorOnline(dsl, connectorRecord, catalog);
+ var connectorRecord = connectorQueries.findByConnectorId(dsl, connectorRef.getConnectorId());
+ connectorUpdateSuccessWriter.handleConnectorOnline(dsl, connectorRef, connectorRecord, catalog);
});
} catch (Exception e) {
failed = true;
try {
// Update connector in a single transaction
dslContextFactory.transaction(dsl -> {
- ConnectorRecord connectorRecord = connectorQueries.findByEndpoint(dsl, connectorEndpoint);
- connectorUpdateFailureWriter.handleConnectorOffline(dsl, connectorRecord, e);
+ var connectorRecord = connectorQueries.findByConnectorId(dsl, connectorRef.getConnectorId());
+ connectorUpdateFailureWriter.handleConnectorOffline(dsl, connectorRef, connectorRecord, e);
});
} catch (Exception e1) {
e1.addSuppressed(e);
@@ -74,9 +76,12 @@ public void updateConnector(String connectorEndpoint) {
executionTime.stop();
try {
var status = failed ? MeasurementErrorStatus.ERROR : MeasurementErrorStatus.OK;
- dslContextFactory.transaction(dsl -> {
- brokerExecutionTimeLogger.logExecutionTime(dsl, connectorEndpoint, executionTime.getTime(TimeUnit.MILLISECONDS), status);
- });
+ dslContextFactory.transaction(dsl -> crawlerExecutionTimeLogger.logExecutionTime(
+ dsl,
+ connectorRef,
+ executionTime.getTime(TimeUnit.MILLISECONDS),
+ status
+ ));
} catch (Exception e) {
monitor.severe("Failed logging connector update execution time.", e);
}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java
new file mode 100644
index 000000000..51f5c9bd4
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling;
+
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger;
+import de.sovity.edc.ext.catalog.crawler.dao.CatalogCleaner;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater;
+import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig;
+import lombok.RequiredArgsConstructor;
+import org.jooq.DSLContext;
+
+@RequiredArgsConstructor
+public class OfflineConnectorCleaner {
+ private final CrawlerConfig crawlerConfig;
+ private final ConnectorQueries connectorQueries;
+ private final CrawlerEventLogger crawlerEventLogger;
+ private final ConnectorStatusUpdater connectorStatusUpdater;
+ private final CatalogCleaner catalogCleaner;
+
+ public void cleanConnectorsIfOfflineTooLong(DSLContext dsl) {
+ var killOfflineConnectorsAfter = crawlerConfig.getKillOfflineConnectorsAfter();
+ var connectorsToKill = connectorQueries.findAllConnectorsForKilling(dsl, killOfflineConnectorsAfter);
+
+ catalogCleaner.removeCatalogByConnectors(dsl, connectorsToKill);
+ connectorStatusUpdater.markAsDead(dsl, connectorsToKill);
+
+ crawlerEventLogger.addKilledDueToOfflineTooLongMessages(dsl, connectorsToKill);
+ }
+}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java
new file mode 100644
index 000000000..2ef73ffa2
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling.fetching;
+
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.utils.catalog.model.DspCatalog;
+import de.sovity.edc.utils.catalog.model.DspContractOffer;
+import de.sovity.edc.utils.catalog.model.DspDataOffer;
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.Validate;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+
+@RequiredArgsConstructor
+public class FetchedCatalogBuilder {
+ private final FetchedCatalogMappingUtils fetchedCatalogMappingUtils;
+
+ public FetchedCatalog buildFetchedCatalog(DspCatalog catalog, ConnectorRef connectorRef) {
+ assertEqualEndpoint(catalog, connectorRef);
+ assertEqualParticipantId(catalog, connectorRef);
+
+ var fetchedDataOffers = catalog.getDataOffers().stream()
+ .map(dspDataOffer -> buildFetchedDataOffer(dspDataOffer, connectorRef))
+ .toList();
+
+ var fetchedCatalog = new FetchedCatalog();
+ fetchedCatalog.setConnectorRef(connectorRef);
+ fetchedCatalog.setDataOffers(fetchedDataOffers);
+ return fetchedCatalog;
+ }
+
+ private void assertEqualParticipantId(DspCatalog catalog, ConnectorRef connectorRef) {
+ Validate.isTrue(
+ connectorRef.getConnectorId().equals(catalog.getParticipantId()),
+ String.format(
+ "Connector connectorId does not match the participantId: connectorId %s, participantId %s",
+ connectorRef.getConnectorId(),
+ catalog.getParticipantId()
+ )
+ );
+ }
+
+ private void assertEqualEndpoint(DspCatalog catalog, ConnectorRef connectorRef) {
+ Validate.isTrue(
+ connectorRef.getEndpoint().equals(catalog.getEndpoint()),
+ String.format(
+ "Connector endpoint mismatch: expected %s, got %s",
+ connectorRef.getEndpoint(),
+ catalog.getEndpoint()
+ )
+ );
+ }
+
+ @NotNull
+ private FetchedDataOffer buildFetchedDataOffer(
+ DspDataOffer dspDataOffer,
+ ConnectorRef connectorRef
+ ) {
+ var uiAsset = fetchedCatalogMappingUtils.buildUiAsset(dspDataOffer, connectorRef);
+ var uiAssetJson = fetchedCatalogMappingUtils.buildUiAssetJson(uiAsset);
+
+ var fetchedDataOffer = new FetchedDataOffer();
+ fetchedDataOffer.setAssetId(uiAsset.getAssetId());
+ fetchedDataOffer.setUiAsset(uiAsset);
+ fetchedDataOffer.setUiAssetJson(uiAssetJson);
+ fetchedDataOffer.setContractOffers(buildFetchedContractOffers(dspDataOffer.getContractOffers()));
+ return fetchedDataOffer;
+ }
+
+ @NotNull
+ private List buildFetchedContractOffers(List offers) {
+ return offers.stream()
+ .map(this::buildFetchedContractOffer)
+ .toList();
+ }
+
+ @NotNull
+ private FetchedContractOffer buildFetchedContractOffer(DspContractOffer offer) {
+ var uiPolicyJson = fetchedCatalogMappingUtils.buildUiPolicyJson(offer.getPolicyJsonLd());
+ var contractOffer = new FetchedContractOffer();
+ contractOffer.setContractOfferId(offer.getContractOfferId());
+ contractOffer.setUiPolicyJson(uiPolicyJson);
+ return contractOffer;
+ }
+
+}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java
new file mode 100644
index 000000000..1ed02ef3e
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling.fetching;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.wrapper.api.common.mappers.AssetMapper;
+import de.sovity.edc.ext.wrapper.api.common.mappers.PolicyMapper;
+import de.sovity.edc.ext.wrapper.api.common.model.UiAsset;
+import de.sovity.edc.utils.catalog.model.DspDataOffer;
+import jakarta.json.JsonObject;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+
+@RequiredArgsConstructor
+public class FetchedCatalogMappingUtils {
+ private final PolicyMapper policyMapper;
+ private final AssetMapper assetMapper;
+ private final ObjectMapper objectMapper;
+
+ public UiAsset buildUiAsset(
+ DspDataOffer dspDataOffer,
+ ConnectorRef connectorRef
+ ) {
+ var assetJsonLd = assetMapper.buildAssetJsonLdFromDatasetProperties(dspDataOffer.getAssetPropertiesJsonLd());
+ var asset = assetMapper.buildAsset(assetJsonLd);
+ var uiAsset = assetMapper.buildUiAsset(asset, connectorRef.getEndpoint(), connectorRef.getConnectorId());
+ uiAsset.setCreatorOrganizationName(connectorRef.getOrganizationLegalName());
+ uiAsset.setParticipantId(connectorRef.getConnectorId());
+ return uiAsset;
+ }
+
+ @SneakyThrows
+ public String buildUiAssetJson(UiAsset uiAsset) {
+ return objectMapper.writeValueAsString(uiAsset);
+ }
+
+ @SneakyThrows
+ public String buildUiPolicyJson(JsonObject policyJsonLd) {
+ var policy = policyMapper.buildPolicy(policyJsonLd);
+ var uiPolicy = policyMapper.buildUiPolicy(policy);
+ return objectMapper.writeValueAsString(uiPolicy);
+ }
+}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/CatalogFetcher.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java
similarity index 59%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/CatalogFetcher.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java
index 2b7da851c..c55973bbe 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/CatalogFetcher.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java
@@ -12,29 +12,30 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
+package de.sovity.edc.ext.catalog.crawler.crawling.fetching;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedCatalog;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
import de.sovity.edc.utils.catalog.DspCatalogService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.eclipse.edc.connector.contract.spi.types.offer.ContractOffer;
@RequiredArgsConstructor
-public class CatalogFetcher {
+public class FetchedCatalogService {
private final DspCatalogService dspCatalogService;
- private final FetchedCatalogBuilder fetchedCatalogBuilder;
+ private final FetchedCatalogBuilder catalogPatchBuilder;
/**
* Fetches {@link ContractOffer}s and de-duplicates them into {@link FetchedDataOffer}s.
*
- * @param connectorEndpoint connector endpoint
+ * @param connectorRef connector
* @return updated connector db row
*/
@SneakyThrows
- public FetchedCatalog fetchCatalog(String connectorEndpoint) {
- var dspCatalog = dspCatalogService.fetchDataOffers(connectorEndpoint);
- return fetchedCatalogBuilder.buildFetchedCatalog(dspCatalog);
+ public FetchedCatalog fetchCatalog(ConnectorRef connectorRef) {
+ var dspCatalog = dspCatalogService.fetchDataOffers(connectorRef.getEndpoint());
+ return catalogPatchBuilder.buildFetchedCatalog(dspCatalog, connectorRef);
}
}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedCatalog.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java
similarity index 68%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedCatalog.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java
index 550dd0657..027d2c8b3 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedCatalog.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java
@@ -12,10 +12,14 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers.model;
+package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Getter;
+import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.experimental.FieldDefaults;
@@ -26,8 +30,11 @@
*/
@Getter
@Setter
+@Builder
+@RequiredArgsConstructor
+@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class FetchedCatalog {
- String participantId;
+ ConnectorRef connectorRef;
List dataOffers;
}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedContractOffer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java
similarity index 72%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedContractOffer.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java
index b2d566f70..2b1993749 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedContractOffer.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java
@@ -12,17 +12,23 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers.model;
+package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model;
import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.FieldDefaults;
@Getter
@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class FetchedContractOffer {
String contractOfferId;
- String policyJson;
+ String uiPolicyJson;
}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedDataOffer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java
similarity index 68%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedDataOffer.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java
index d93306613..a2e6fe99b 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/model/FetchedDataOffer.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java
@@ -12,10 +12,14 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers.model;
+package de.sovity.edc.ext.catalog.crawler.crawling.fetching.model;
+import de.sovity.edc.ext.wrapper.api.common.model.UiAsset;
import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.FieldDefaults;
@@ -26,18 +30,13 @@
*/
@Getter
@Setter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class FetchedDataOffer {
String assetId;
- String assetTitle;
- String description;
- String curatorOrganizationName;
- String dataCategory;
- String dataSubcategory;
- String dataModel;
- String transportMode;
- String geoReferenceMethod;
- List keywords;
- String assetJsonLd;
+ UiAsset uiAsset;
+ String uiAssetJson;
List contractOffers;
}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/ConnectorChangeTracker.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java
similarity index 96%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/ConnectorChangeTracker.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java
index 1513c1f5e..e87eabb01 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/ConnectorChangeTracker.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java
@@ -12,7 +12,7 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.logging;
+package de.sovity.edc.ext.catalog.crawler.crawling.logging;
import lombok.Getter;
import lombok.Setter;
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventErrorMessage.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java
similarity index 57%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventErrorMessage.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java
index b3746df1f..9d7ca039a 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/logging/BrokerEventErrorMessage.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java
@@ -12,32 +12,32 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.logging;
+package de.sovity.edc.ext.catalog.crawler.crawling.logging;
-import de.sovity.edc.ext.brokerserver.utils.StringUtils2;
+import de.sovity.edc.ext.catalog.crawler.utils.StringUtils2;
import lombok.NonNull;
import org.apache.commons.lang3.exception.ExceptionUtils;
/**
* Helper Dto that contains User Message + Error Stack Trace to be written into
- * {@link de.sovity.edc.ext.brokerserver.db.jooq.tables.BrokerEventLog}.
+ * {@link de.sovity.edc.ext.catalog.crawler.db.jooq.tables.CrawlerEventLog}.
*
* This class exists so that logging exceptions has a consistent format.
*
- * @param message message
+ * @param message message
* @param stackTraceOrNull stack trace
*/
-public record BrokerEventErrorMessage(String message, String stackTraceOrNull) {
+public record CrawlerEventErrorMessage(String message, String stackTraceOrNull) {
- public static BrokerEventErrorMessage ofMessage(@NonNull String message) {
- return new BrokerEventErrorMessage(message, null);
+ public static CrawlerEventErrorMessage ofMessage(@NonNull String message) {
+ return new CrawlerEventErrorMessage(message, null);
}
- public static BrokerEventErrorMessage ofStackTrace(@NonNull String baseMessage, @NonNull Throwable cause) {
+ public static CrawlerEventErrorMessage ofStackTrace(@NonNull String baseMessage, @NonNull Throwable cause) {
var message = baseMessage;
message = StringUtils2.removeSuffix(message, ".");
message = StringUtils2.removeSuffix(message, ":");
message = "%s: %s".formatted(message, cause.getClass().getName());
- return new BrokerEventErrorMessage(message, ExceptionUtils.getStackTrace(cause));
+ return new CrawlerEventErrorMessage(message, ExceptionUtils.getStackTrace(cause));
}
}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java
new file mode 100644
index 000000000..396e2095e
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling.logging;
+
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.CrawlerEventStatus;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.CrawlerEventType;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.CrawlerEventLogRecord;
+import lombok.RequiredArgsConstructor;
+import org.jooq.DSLContext;
+
+import java.time.OffsetDateTime;
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Updates a single connector.
+ */
+@RequiredArgsConstructor
+public class CrawlerEventLogger {
+
+ public void logConnectorUpdated(DSLContext dsl, ConnectorRef connectorRef, ConnectorChangeTracker changes) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_UPDATED);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage(changes.toString());
+ logEntry.insert();
+ }
+
+ public void logConnectorOffline(DSLContext dsl, ConnectorRef connectorRef, CrawlerEventErrorMessage errorMessage) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_STATUS_CHANGE_OFFLINE);
+ logEntry.setEventStatus(CrawlerEventStatus.ERROR);
+ logEntry.setUserMessage("Connector is offline.");
+ logEntry.setErrorStack(errorMessage.stackTraceOrNull());
+ logEntry.insert();
+ }
+
+ public void logConnectorOnline(DSLContext dsl, ConnectorRef connectorRef) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_STATUS_CHANGE_ONLINE);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage("Connector is online.");
+ logEntry.insert();
+ }
+
+ public void logConnectorUpdateDataOfferLimitExceeded(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ Integer maxDataOffersPerConnector
+ ) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_DATA_OFFER_LIMIT_EXCEEDED);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage(
+ "Connector has more than %d data offers. Exceeding data offers will be ignored.".formatted(maxDataOffersPerConnector));
+ logEntry.insert();
+ }
+
+ public void logConnectorUpdateDataOfferLimitOk(DSLContext dsl, ConnectorRef connectorRef) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_DATA_OFFER_LIMIT_OK);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage("Connector is not exceeding the maximum number of data offers limit anymore.");
+ logEntry.insert();
+ }
+
+ public void logConnectorUpdateContractOfferLimitExceeded(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ Integer maxContractOffersPerConnector
+ ) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_CONTRACT_OFFER_LIMIT_EXCEEDED);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage(String.format(
+ "Some data offers have more than %d contract offers. Exceeding contract offers will be ignored.: ",
+ maxContractOffersPerConnector
+ ));
+ logEntry.insert();
+ }
+
+ public void logConnectorUpdateContractOfferLimitOk(DSLContext dsl, ConnectorRef connectorRef) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_CONTRACT_OFFER_LIMIT_OK);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage("Connector is not exceeding the maximum number of contract offers per data offer limit anymore.");
+ logEntry.insert();
+ }
+
+ public void addKilledDueToOfflineTooLongMessages(DSLContext dsl, Collection connectorRefs) {
+ var logEntries = connectorRefs.stream()
+ .map(connectorRef -> buildKilledDueToOfflineTooLongMessage(dsl, connectorRef))
+ .toList();
+ dsl.batchInsert(logEntries).execute();
+ }
+
+ private CrawlerEventLogRecord buildKilledDueToOfflineTooLongMessage(DSLContext dsl, ConnectorRef connectorRef) {
+ var logEntry = newLogEntry(dsl, connectorRef);
+ logEntry.setEvent(CrawlerEventType.CONNECTOR_KILLED_DUE_TO_OFFLINE_FOR_TOO_LONG);
+ logEntry.setEventStatus(CrawlerEventStatus.OK);
+ logEntry.setUserMessage("Connector was marked as dead for being offline too long.");
+ return logEntry;
+ }
+
+ private CrawlerEventLogRecord newLogEntry(DSLContext dsl, ConnectorRef connectorRef) {
+ var logEntry = dsl.newRecord(Tables.CRAWLER_EVENT_LOG);
+ logEntry.setId(UUID.randomUUID());
+ logEntry.setEnvironment(connectorRef.getEnvironmentId());
+ logEntry.setConnectorId(connectorRef.getConnectorId());
+ logEntry.setCreatedAt(OffsetDateTime.now());
+ return logEntry;
+ }
+}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java
new file mode 100644
index 000000000..b04e757f4
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling.logging;
+
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementErrorStatus;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementType;
+import lombok.RequiredArgsConstructor;
+import org.jooq.DSLContext;
+
+import java.time.OffsetDateTime;
+import java.util.UUID;
+
+/**
+ * Updates a single connector.
+ */
+@RequiredArgsConstructor
+public class CrawlerExecutionTimeLogger {
+ public void logExecutionTime(DSLContext dsl, ConnectorRef connectorRef, long executionTimeInMs, MeasurementErrorStatus errorStatus) {
+ var logEntry = dsl.newRecord(Tables.CRAWLER_EXECUTION_TIME_MEASUREMENT);
+ logEntry.setId(UUID.randomUUID());
+ logEntry.setEnvironment(connectorRef.getEnvironmentId());
+ logEntry.setConnectorId(connectorRef.getConnectorId());
+ logEntry.setDurationInMs(executionTimeInMs);
+ logEntry.setType(MeasurementType.CONNECTOR_REFRESH);
+ logEntry.setErrorStatus(errorStatus);
+ logEntry.setCreatedAt(OffsetDateTime.now());
+ logEntry.insert();
+ }
+}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferPatchBuilder.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java
similarity index 67%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferPatchBuilder.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java
index 8b7dcc6be..309e8d8fc 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferPatchBuilder.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java
@@ -12,15 +12,19 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
-
-import de.sovity.edc.ext.brokerserver.dao.ContractOfferQueries;
-import de.sovity.edc.ext.brokerserver.dao.DataOfferQueries;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ContractOfferRecord;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.DataOfferRecord;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.DataOfferPatch;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedContractOffer;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
+package de.sovity.edc.ext.catalog.crawler.crawling.writing;
+
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.DiffUtils;
+import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatch;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferRecordUpdater;
+import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries;
+import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord;
import lombok.RequiredArgsConstructor;
import org.jooq.DSLContext;
@@ -31,7 +35,7 @@
import static java.util.stream.Collectors.groupingBy;
@RequiredArgsConstructor
-public class DataOfferPatchBuilder {
+public class CatalogPatchBuilder {
private final ContractOfferQueries contractOfferQueries;
private final DataOfferQueries dataOfferQueries;
private final DataOfferRecordUpdater dataOfferRecordUpdater;
@@ -40,19 +44,19 @@ public class DataOfferPatchBuilder {
/**
* Fetches existing data offers of given connector endpoint and compares them with fetched data offers.
*
- * @param dsl dsl
- * @param connectorEndpoint connector endpoint
+ * @param dsl dsl
+ * @param connectorRef connector
* @param fetchedDataOffers fetched data offers
* @return change list / patch
*/
- public DataOfferPatch buildDataOfferPatch(
+ public CatalogPatch buildDataOfferPatch(
DSLContext dsl,
- String connectorEndpoint,
+ ConnectorRef connectorRef,
Collection fetchedDataOffers
) {
- var patch = new DataOfferPatch();
- var dataOffers = dataOfferQueries.findByConnectorEndpoint(dsl, connectorEndpoint);
- var contractOffersByAssetId = contractOfferQueries.findByConnectorEndpoint(dsl, connectorEndpoint)
+ var patch = new CatalogPatch();
+ var dataOffers = dataOfferQueries.findByConnectorId(dsl, connectorRef.getConnectorId());
+ var contractOffersByAssetId = contractOfferQueries.findByConnectorId(dsl, connectorRef.getConnectorId())
.stream()
.collect(groupingBy(ContractOfferRecord::getAssetId));
@@ -64,8 +68,8 @@ public DataOfferPatch buildDataOfferPatch(
);
diff.added().forEach(fetched -> {
- var newRecord = dataOfferRecordUpdater.newDataOffer(connectorEndpoint, fetched);
- patch.insertDataOffer(newRecord);
+ var newRecord = dataOfferRecordUpdater.newDataOffer(connectorRef, fetched);
+ patch.dataOffers().insert(newRecord);
patchContractOffers(patch, newRecord, List.of(), fetched.getContractOffers());
});
@@ -81,21 +85,21 @@ public DataOfferPatch buildDataOfferPatch(
changed = dataOfferRecordUpdater.updateDataOffer(existing, fetched, changed);
if (changed) {
- patch.updateDataOffer(existing);
+ patch.dataOffers().update(existing);
}
});
diff.removed().forEach(dataOffer -> {
- patch.deleteDataOffer(dataOffer);
+ patch.dataOffers().delete(dataOffer);
var contractOffers = contractOffersByAssetId.getOrDefault(dataOffer.getAssetId(), List.of());
- contractOffers.forEach(patch::deleteContractOffer);
+ contractOffers.forEach(it -> patch.contractOffers().delete(it));
});
return patch;
}
private boolean patchContractOffers(
- DataOfferPatch patch,
+ CatalogPatch patch,
DataOfferRecord dataOffer,
Collection contractOffers,
Collection fetchedContractOffers
@@ -111,7 +115,7 @@ private boolean patchContractOffers(
diff.added().forEach(fetched -> {
var newRecord = contractOfferRecordUpdater.newContractOffer(dataOffer, fetched);
- patch.insertContractOffer(newRecord);
+ patch.contractOffers().insert(newRecord);
hasUpdates.set(true);
});
@@ -120,13 +124,13 @@ private boolean patchContractOffers(
var fetched = match.fetched();
if (contractOfferRecordUpdater.updateContractOffer(existing, fetched)) {
- patch.updateContractOffer(existing);
+ patch.contractOffers().update(existing);
hasUpdates.set(true);
}
});
diff.removed().forEach(existing -> {
- patch.deleteContractOffer(existing);
+ patch.contractOffers().delete(existing);
hasUpdates.set(true);
});
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java
new file mode 100644
index 000000000..ec67dd2a2
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling.writing;
+
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker;
+import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import org.jooq.DSLContext;
+
+import java.util.Collection;
+
+@RequiredArgsConstructor
+public class ConnectorUpdateCatalogWriter {
+ private final CatalogPatchBuilder catalogPatchBuilder;
+ private final CatalogPatchApplier catalogPatchApplier;
+
+ /**
+ * Updates a connector's data offers with given {@link FetchedDataOffer}s.
+ *
+ * @param dsl dsl
+ * @param connectorRef connector
+ * @param fetchedDataOffers fetched data offers
+ * @param changes change tracker for log message
+ */
+ @SneakyThrows
+ public void updateDataOffers(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ Collection fetchedDataOffers,
+ ConnectorChangeTracker changes
+ ) {
+ var patch = catalogPatchBuilder.buildDataOfferPatch(dsl, connectorRef, fetchedDataOffers);
+ changes.setNumOffersAdded(patch.dataOffers().getInsertions().size());
+ changes.setNumOffersUpdated(patch.dataOffers().getUpdates().size());
+ changes.setNumOffersDeleted(patch.dataOffers().getDeletions().size());
+ catalogPatchApplier.applyDbUpdatesBatched(dsl, patch);
+ }
+}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdateFailureWriter.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java
similarity index 51%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdateFailureWriter.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java
index 87dd61389..05b7f1bb1 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/ConnectorUpdateFailureWriter.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java
@@ -12,12 +12,13 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing;
+package de.sovity.edc.ext.catalog.crawler.crawling.writing;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorOnlineStatus;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventErrorMessage;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventErrorMessage;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord;
import lombok.RequiredArgsConstructor;
import org.eclipse.edc.spi.monitor.Monitor;
import org.jooq.DSLContext;
@@ -26,14 +27,19 @@
@RequiredArgsConstructor
public class ConnectorUpdateFailureWriter {
- private final BrokerEventLogger brokerEventLogger;
+ private final CrawlerEventLogger crawlerEventLogger;
private final Monitor monitor;
- public void handleConnectorOffline(DSLContext dsl, ConnectorRecord connector, Throwable e) {
+ public void handleConnectorOffline(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ ConnectorRecord connector,
+ Throwable e
+ ) {
// Log Status Change and set status to offline if necessary
if (connector.getOnlineStatus() == ConnectorOnlineStatus.ONLINE || connector.getLastRefreshAttemptAt() == null) {
- monitor.info("Connector is offline: " + connector.getEndpoint(), e);
- brokerEventLogger.logConnectorOffline(dsl, connector.getEndpoint(), getFailureMessage(e));
+ monitor.info("Connector is offline: " + connector.getEndpointUrl(), e);
+ crawlerEventLogger.logConnectorOffline(dsl, connectorRef, getFailureMessage(e));
connector.setOnlineStatus(ConnectorOnlineStatus.OFFLINE);
}
@@ -41,7 +47,7 @@ public void handleConnectorOffline(DSLContext dsl, ConnectorRecord connector, Th
connector.update();
}
- public BrokerEventErrorMessage getFailureMessage(Throwable e) {
- return BrokerEventErrorMessage.ofStackTrace("Unexpected exception during connector update.", e);
+ public CrawlerEventErrorMessage getFailureMessage(Throwable e) {
+ return CrawlerEventErrorMessage.ofStackTrace("Unexpected exception during connector update.", e);
}
}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java
new file mode 100644
index 000000000..32b5ebe5e
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2023 sovity GmbH
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Contributors:
+ * sovity GmbH - initial API and implementation
+ *
+ */
+
+package de.sovity.edc.ext.catalog.crawler.crawling.writing;
+
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord;
+import lombok.RequiredArgsConstructor;
+import org.jooq.DSLContext;
+
+import java.time.OffsetDateTime;
+
+@RequiredArgsConstructor
+public class ConnectorUpdateSuccessWriter {
+ private final CrawlerEventLogger crawlerEventLogger;
+ private final ConnectorUpdateCatalogWriter connectorUpdateCatalogWriter;
+ private final DataOfferLimitsEnforcer dataOfferLimitsEnforcer;
+
+ public void handleConnectorOnline(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ ConnectorRecord connector,
+ FetchedCatalog catalog
+ ) {
+ // Limit data offers and log limitation if necessary
+ var limitedDataOffers = dataOfferLimitsEnforcer.enforceLimits(catalog.getDataOffers());
+ dataOfferLimitsEnforcer.logEnforcedLimitsIfChanged(dsl, connectorRef, connector, limitedDataOffers);
+
+ // Log Status Change and set status to online if necessary
+ if (connector.getOnlineStatus() != ConnectorOnlineStatus.ONLINE || connector.getLastRefreshAttemptAt() == null) {
+ crawlerEventLogger.logConnectorOnline(dsl, connectorRef);
+ connector.setOnlineStatus(ConnectorOnlineStatus.ONLINE);
+ }
+
+ // Track changes for final log message
+ var changes = new ConnectorChangeTracker();
+ var now = OffsetDateTime.now();
+ connector.setLastSuccessfulRefreshAt(now);
+ connector.setLastRefreshAttemptAt(now);
+ connector.update();
+
+ // Update data offers
+ connectorUpdateCatalogWriter.updateDataOffers(
+ dsl,
+ connectorRef,
+ limitedDataOffers.abbreviatedDataOffers(),
+ changes
+ );
+
+ // Log event if changes are present
+ if (!changes.isEmpty()) {
+ crawlerEventLogger.logConnectorUpdated(dsl, connectorRef, changes);
+ }
+ }
+
+}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferLimitsEnforcer.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java
similarity index 62%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferLimitsEnforcer.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java
index b4d067881..0583b93b3 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DataOfferLimitsEnforcer.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java
@@ -12,14 +12,15 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
+package de.sovity.edc.ext.catalog.crawler.crawling.writing;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorContractOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.enums.ConnectorDataOffersExceeded;
-import de.sovity.edc.ext.brokerserver.db.jooq.tables.records.ConnectorRecord;
-import de.sovity.edc.ext.brokerserver.services.config.BrokerServerSettings;
-import de.sovity.edc.ext.brokerserver.services.logging.BrokerEventLogger;
-import de.sovity.edc.ext.brokerserver.services.refreshing.offers.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer;
+import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger;
+import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded;
+import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord;
+import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig;
import lombok.RequiredArgsConstructor;
import org.jooq.DSLContext;
@@ -29,8 +30,8 @@
@RequiredArgsConstructor
public class DataOfferLimitsEnforcer {
- private final BrokerServerSettings brokerServerSettings;
- private final BrokerEventLogger brokerEventLogger;
+ private final CrawlerConfig crawlerConfig;
+ private final CrawlerEventLogger crawlerEventLogger;
public record DataOfferLimitsEnforced(
Collection abbreviatedDataOffers,
@@ -41,8 +42,8 @@ public record DataOfferLimitsEnforced(
public DataOfferLimitsEnforced enforceLimits(Collection dataOffers) {
// Get limits from config
- var maxDataOffers = brokerServerSettings.getMaxDataOffersPerConnector();
- var maxContractOffers = brokerServerSettings.getMaxContractOffersPerDataOffer();
+ var maxDataOffers = crawlerConfig.getMaxDataOffersPerConnector();
+ var maxContractOffers = crawlerConfig.getMaxContractOffersPerDataOffer();
List offerList = new ArrayList<>(dataOffers);
// No limits set
@@ -70,26 +71,31 @@ public DataOfferLimitsEnforced enforceLimits(Collection dataOf
return new DataOfferLimitsEnforced(offerList, dataOfferLimitsExceeded, contractOfferLimitsExceeded);
}
- public void logEnforcedLimitsIfChanged(DSLContext dsl, ConnectorRecord connector, DataOfferLimitsEnforced enforcedLimits) {
- String endpoint = connector.getEndpoint();
+ public void logEnforcedLimitsIfChanged(
+ DSLContext dsl,
+ ConnectorRef connectorRef,
+ ConnectorRecord connector,
+ DataOfferLimitsEnforced enforcedLimits
+ ) {
// DataOffer
if (enforcedLimits.dataOfferLimitsExceeded() && connector.getDataOffersExceeded() == ConnectorDataOffersExceeded.OK) {
- var maxDataOffers = brokerServerSettings.getMaxDataOffersPerConnector();
- brokerEventLogger.logConnectorUpdateDataOfferLimitExceeded(dsl, maxDataOffers, endpoint);
+ var maxDataOffers = crawlerConfig.getMaxDataOffersPerConnector();
+ crawlerEventLogger.logConnectorUpdateDataOfferLimitExceeded(dsl, connectorRef, maxDataOffers);
connector.setDataOffersExceeded(ConnectorDataOffersExceeded.EXCEEDED);
} else if (!enforcedLimits.dataOfferLimitsExceeded() && connector.getDataOffersExceeded() == ConnectorDataOffersExceeded.EXCEEDED) {
- brokerEventLogger.logConnectorUpdateDataOfferLimitOk(dsl, endpoint);
+ crawlerEventLogger.logConnectorUpdateDataOfferLimitOk(dsl, connectorRef);
connector.setDataOffersExceeded(ConnectorDataOffersExceeded.OK);
}
// ContractOffer
if (enforcedLimits.contractOfferLimitsExceeded() && connector.getContractOffersExceeded() == ConnectorContractOffersExceeded.OK) {
- var maxContractOffers = brokerServerSettings.getMaxContractOffersPerDataOffer();
- brokerEventLogger.logConnectorUpdateContractOfferLimitExceeded(dsl, maxContractOffers, endpoint);
+ var maxContractOffers = crawlerConfig.getMaxContractOffersPerDataOffer();
+ crawlerEventLogger.logConnectorUpdateContractOfferLimitExceeded(dsl, connectorRef, maxContractOffers);
connector.setContractOffersExceeded(ConnectorContractOffersExceeded.EXCEEDED);
- } else if (!enforcedLimits.contractOfferLimitsExceeded() && connector.getContractOffersExceeded() == ConnectorContractOffersExceeded.EXCEEDED) {
- brokerEventLogger.logConnectorUpdateContractOfferLimitOk(dsl, endpoint);
+ } else if (!enforcedLimits.contractOfferLimitsExceeded() &&
+ connector.getContractOffersExceeded() == ConnectorContractOffersExceeded.EXCEEDED) {
+ crawlerEventLogger.logConnectorUpdateContractOfferLimitOk(dsl, connectorRef);
connector.setContractOffersExceeded(ConnectorContractOffersExceeded.OK);
}
}
diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java
new file mode 100644
index 000000000..dad7f9cae
--- /dev/null
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java
@@ -0,0 +1,36 @@
+package de.sovity.edc.ext.catalog.crawler.crawling.writing.utils;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.util.Objects;
+import java.util.function.BiPredicate;
+import java.util.function.Consumer;
+
+@Getter
+@NoArgsConstructor
+@AllArgsConstructor
+public class ChangeTracker {
+ private boolean changed = false;
+
+ public void setIfChanged(
+ T existing,
+ T fetched,
+ Consumer setter
+ ) {
+ setIfChanged(existing, fetched, setter, Objects::equals);
+ }
+
+ public void setIfChanged(
+ T existing,
+ T fetched,
+ Consumer setter,
+ BiPredicate equalityChecker
+ ) {
+ if (!equalityChecker.test(existing, fetched)) {
+ setter.accept(fetched);
+ changed = true;
+ }
+ }
+}
diff --git a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DiffUtils.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java
similarity index 72%
rename from extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DiffUtils.java
rename to extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java
index 413ba77a2..7db2dac4f 100644
--- a/extensions/broker-server/src/main/java/de/sovity/edc/ext/brokerserver/services/refreshing/offers/DiffUtils.java
+++ b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java
@@ -12,9 +12,9 @@
*
*/
-package de.sovity.edc.ext.brokerserver.services.refreshing.offers;
+package de.sovity.edc.ext.catalog.crawler.crawling.writing.utils;
-import de.sovity.edc.ext.brokerserver.utils.MapUtils;
+import de.sovity.edc.ext.catalog.crawler.utils.MapUtils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
@@ -30,13 +30,13 @@ public class DiffUtils {
/**
* Tries to match two collections by a key, then collects planned change sets as {@link DiffResult}.
*
- * @param existing list of existing elements
+ * @param existing list of existing elements
* @param existingKeyFn existing elements key extractor
- * @param fetched list of fetched elements
- * @param fetchedKeyFn fetched elements key extractor
- * @param first collection type
- * @param second collection type
- * @param key type
+ * @param fetched list of fetched elements
+ * @param fetchedKeyFn fetched elements key extractor
+ * @param first collection type
+ * @param second collection type
+ * @param key type
*/
public static DiffResult compareLists(
Collection existing,
@@ -71,13 +71,13 @@ public static DiffResult compareLists(
/**
* Result of comparing two collections by keys.
*
- * @param added elements that are present in fetched collection but not in existing
+ * @param added elements that are present in fetched collection but not in existing
* @param updated elements that are present in both collections
* @param removed elements that are present in existing collection but not in fetched
- * @param existing item type
- * @param fetched item type
+ * @param existing item type
+ * @param fetched item type
*/
- record DiffResult(List added, List> updated, List removed) {
+ public record DiffResult(List added, List> updated, List removed) {
DiffResult() {
this(new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
}
@@ -87,10 +87,10 @@ record DiffResult(List added, List> updated, List
* Pair of elements that are present in both collections.
*
* @param existing existing item
- * @param fetched fetched item
- * @param existing item type
- * @param fetched item type
+ * @param fetched fetched item
+ * @param existing item type
+ * @param
-
-
-