Merge branch 'main' into mds-2.2-combine-policies

.dockerignore

@@ -39,7 +39,7 @@ build
 
 .env
 !launchers/.env.connector
-!launchers/.env.broker
+!launchers/.env.catalog-crawler
 
 # Log files
 *.log

.editorconfig

@@ -22,6 +22,17 @@ indent_size = 2
 [*.{yml,yaml}]
 indent_size = 2
 
+[*.java]
+max_line_length = 140
+ij_continuation_indent_size = 8
+ij_java_blank_lines_after_imports = 1
+ij_java_blank_lines_after_package = 1
+ij_java_line_comment_add_space = true
+ij_java_imports_layout = *,|,java.**,javax.**,|,$*
+ij_java_class_count_to_use_import_on_demand = 999
+ij_java_doc_align_param_comments = false
+
+
 [{*.kt,*.kts}]
 ij_kotlin_imports_layout = *, java.**, javax.**, kotlin.**, ^
 ij_kotlin_packages_to_use_import_on_demand = explicitly-none

.env

@@ -3,4 +3,3 @@ EDC_IMAGE=ghcr.io/sovity/edc-dev:8.1.0
 TEST_BACKEND_IMAGE=ghcr.io/sovity/test-backend:8.1.0
 EDC_UI_IMAGE=ghcr.io/sovity/edc-ui:3.2.2
 EDC_UI_ACTIVE_PROFILE=sovity-open-source
-BROKER_IMAGE=ghcr.io/sovity/broker-server-dev:8.1.0

.env.dev

@@ -3,4 +3,3 @@ EDC_IMAGE=ghcr.io/sovity/edc-dev:latest
 TEST_BACKEND_IMAGE=ghcr.io/sovity/test-backend:latest
 EDC_UI_IMAGE=ghcr.io/sovity/edc-ui:latest
 EDC_UI_ACTIVE_PROFILE=sovity-open-source
-BROKER_IMAGE=ghcr.io/sovity/broker-server-dev:latest

.github/ISSUE_TEMPLATE/release.md

@@ -46,8 +46,6 @@ Feel free to edit this release checklist in-progress depending on what tasks nee
         the [docker-compose's .env file](https://github.com/sovity/edc-ce/blob/main/.env).
   - [ ] Set the version for `TEST_BACKEND_IMAGE` of
         the [docker-compose's .env file](https://github.com/sovity/edc-ce/blob/main/.env).
-  - [ ] Set the version for `BROKER_IMAGE` of
-    the [docker-compose's .env file](https://github.com/sovity/edc-ce/blob/main/.env).
   - [ ] Set the UI release version for `EDC_UI_IMAGE` of
         the [docker-compose's .env file](https://github.com/sovity/edc-ce/blob/main/.env).
   - [ ] If the Eclipse EDC version changed, update
@@ -58,13 +56,10 @@ Feel free to edit this release checklist in-progress depending on what tasks nee
 - [ ] Validate the image
   - [ ] Pull the latest edc-dev image: `docker image pull ghcr.io/sovity/edc-dev:latest`.
   - [ ] Check that your image was built recently `docker image ls | grep ghcr.io/sovity/edc-dev`.
-  - [ ] Test the release `docker-compose.yaml` with `EDC_IMAGE=ghcr.io/sovity/edc-dev:latest` and `BROKER_IMAGE=ghcr.io/sovity/broker-server-dev:latest` (at minimum execute a transfer between the two connectors).
+  - [ ] Test the release `docker-compose.yaml` with `EDC_IMAGE=ghcr.io/sovity/edc-dev:latest` (at minimum execute a transfer between the two connectors).
     - EDC
       - [ ] Test with `EDC_UI_ACTIVE_PROFILE=sovity-open-source`
       - [ ] Test with `EDC_UI_ACTIVE_PROFILE=mds-open-source`
-    - Broker
-      - [ ] Validate that the EDC is scanned.
-      - [ ] Validate that the index is searchable.
   - [ ] Ensure with a `docker ps -a` that all containers are healthy, and not `healthy: starting` or `healthy: unhealthy`.
 - [ ] [Create a release](https://github.com/sovity/edc-ce/releases/new)
   - [ ] In `Choose the tag`, type your new release version in the format `vx.y.z` (for instance `v1.2.3`) then click `+Create new tag vx.y.z on release`.

.github/actions/build-connector-image/action.yml

@@ -21,7 +21,7 @@ inputs:
     description: "EDC Connector Name in launchers/connectors/{connector-name}"
   deployment-type:
     required: true
-    description: "Type of deployment: 'connector' or 'broker'"
+    description: "Type of deployment: 'connector' or 'catalog-crawler'"
   title:
     required: true
     description: "Docker Image Title"

.github/markdown-link-checker-config.jq

@@ -7,7 +7,6 @@
     {"pattern": "^https://www\\.linkedin\\.com"},
     {"pattern": "https://(.*?)\\.azure\\.sovity\\.io"},
     {"pattern": "http://edc2?:"},
-    {"pattern": "^https?://broker:"},
     {"pattern": "^https?://connector:"}
   ],
   "replacementPatterns": [

.github/workflows/ci.yml

@@ -104,30 +104,30 @@ jobs:
           deployment-type: "connector"
           title: "Test Data Source / Data Sink"
           description: "Provides a minimal data source / data sink for E2E tests."
-      - name: "Docker Image: broker-server-dev"
+      - name: "Docker Image: catalog-crawler-dev"
         uses: ./.github/actions/build-connector-image
         with:
           registry-url: ${{ env.REGISTRY_URL }}
           registry-user: ${{ env.REGISTRY_USER }}
           registry-password: ${{ secrets.GITHUB_TOKEN }}
           image-base-name: ${{ env.IMAGE_BASE_NAME }}
-          image-name: "broker-server-dev"
-          connector-name: "broker-server-dev"
-          deployment-type: "broker"
-          title: "Broker Server (Dev)"
-          description: "sovity EDC Broker Server. This dev version contains no auth and can be used to quickly start a locally running Broker Server + Broker UI."
-      - name: "Docker Image: broker-server-ce"
+          image-name: "catalog-crawler-dev"
+          connector-name: "catalog-crawler-dev"
+          deployment-type: "catalog-crawler"
+          title: "Catalog Crawler (Dev)"
+          description: "sovity CE Catalog crawler for the sovity CE Authority Portal. This dev version contains no auth and can be used to quickly start a locally running Catalog Crawler."
+      - name: "Docker Image: catalog-crawler-ce"
         uses: ./.github/actions/build-connector-image
         with:
           registry-url: ${{ env.REGISTRY_URL }}
           registry-user: ${{ env.REGISTRY_USER }}
           registry-password: ${{ secrets.GITHUB_TOKEN }}
           image-base-name: ${{ env.IMAGE_BASE_NAME }}
-          image-name: "broker-server-ce"
-          connector-name: "broker-server-ce"
-          deployment-type: "broker"
-          title: "Broker Server (Community Edition)"
-          description: "sovity EDC Broker Server. Requires dataspace credentials to join an existing dataspace."
+          image-name: "catalog-crawler-ce"
+          connector-name: "catalog-crawler-ce"
+          deployment-type: "catalog-crawler"
+          title: "Catalog Crawler (Community Edition, DAPS)"
+          description: "sovity CE Catalog crawler for the sovity CE Authority Portal. Requires DAPS dataspace credentials to join an existing dataspace."
   ts-api-client-library:
     name: TS API Client Library
     runs-on: ubuntu-latest
@@ -181,57 +181,3 @@ jobs:
         env:
           NODE_USER: richardtreier-sovity
           NODE_AUTH_TOKEN: ${{ secrets.SOVITY_EDC_CLIENT_NPM_AUTH }}
-  ts-broker-api-client-library:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - uses: FranzDiebold/github-env-vars-action@v2
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-          cache: 'gradle'
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 16
-          cache: 'npm'
-          registry-url: 'https://registry.npmjs.org'
-          cache-dependency-path: ./extensions/broker-server-api/client-ts/package-lock.json
-      - name: Generate openapi.yaml & Client Code
-        run: |
-          ./gradlew :extensions:broker-server-api:api:clean :extensions:broker-server-api:api:build -x test --no-daemon
-        env:
-          USERNAME: ${{ github.actor }}
-          TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: NPM Package Dist Tag & Version
-        working-directory: ./extensions/broker-server-api/client-ts
-        run: |
-          if [ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]; then
-            # Full Release
-            VERSION="${GITHUB_REF#refs/tags/v}"
-            DIST_TAG=latest
-          else
-            VERSION="0.$(date '+%Y%m%d.%H%M%S')-main-$CI_SHA_SHORT"
-            DIST_TAG=main
-          fi
-          npm version $VERSION
-          echo "DIST_TAG=$DIST_TAG" >> $GITHUB_ENV
-      - name: Build NPM Library
-        working-directory: ./extensions/broker-server-api/client-ts
-        run: |
-          npm ci && npm run build
-      - name: Publish NPM Library
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/tags/v*'
-        working-directory: ./extensions/broker-server-api/client-ts
-        run: |
-          npm set //registry.npmjs.org/:_authToken $NODE_AUTH_TOKEN
-          npm set //registry.npmjs.org/:username $NODE_USER
-          npm publish --access public --tag "${{ env.DIST_TAG }}"
-        env:
-          NODE_USER: richardtreier-sovity
-          NODE_AUTH_TOKEN: ${{ secrets.SOVITY_BROKER_CLIENT_NPM_AUTH }}

.github/workflows/license_scan.yml

@@ -31,8 +31,6 @@ jobs:
         run: cd extensions/wrapper/clients/typescript-client && npm clean-install
       - name: npm install (typescript-client-example)
         run: cd extensions/wrapper/clients/typescript-client-example && npm clean-install
-      - name: npm install (client-ts)
-        run: cd extensions/broker-server-api/client-ts && npm clean-install
       - name: Run license scanner
         uses: aquasecurity/trivy-action@master
         with:

.gitignore

@@ -40,7 +40,7 @@ build
 **/.env
 !.env
 !launchers/.env.connector
-!launchers/.env.broker
+!launchers/.env.catalog-crawler
 
 # Log files
 *.log

CHANGELOG.md

@@ -7,12 +7,14 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md).
 
 ### Overview
 
-### EDC UI
-
-### EDC Extensions and Broker
+### Detailed Changes
 
 #### Major Changes
 
+- The Broker has been removed in favor of the Authority Portal and the new Deployment Unit, the "Data Catalog Crawler":
+  - Each "Data Catalog Crawler" connects to an existing Authority Portal Deployment's DB.
+  - Each "Data Catalog Crawler" is responsible for crawling exactly one environment.
+  - The Data Catalog functionality of the Broker has been integrated into the Authority Portal.
 - API Wrapper UI API: Moved to well-typed data sources, breaking changes to the asset model and API.
 
 #### Minor Changes
@@ -25,8 +27,14 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md).
 
 ### Deployment Migration Notes
 
-The database migration system has been moved from multiple migration history tables to a single one. Although this
-process has been extensively tested in the enterprise edition already, it should be tested once more on a copy of a productive connector.
+- Connector:
+  - The database migration system has been moved from multiple migration history tables to a single one. Although this
+    process has been extensively tested in the enterprise edition already, it should be tested once more on a copy of a productive connector.
+- Broker:
+  - The broker has been removed. For Authority Portal users, please check out the new 
+    [Data Catalog Crawler Productive Deployment Guide](docs/deployment-guide/goals/catalog-crawler-production/README.md). 
+  - Any previous broker deployment's database is not required anymore.
+  - Please care that only some environment variables look similar. It is recommended to create fresh deployments.
 
 ## [8.1.0] - 2024-06-14
 
@@ -42,7 +50,7 @@ Support for Multiplicity Constraints in the API Wrapper.
 
 #### Minor Changes
 
-- API Wrapper 
+- API Wrapper
   - Support for Multiplicity Constraints (https://github.com/sovity/edc-ce/issues/968)
   - Providing `Prop` class from `json-and-jsonld-utils` to the java-client to make relevant Constants available
 
@@ -88,7 +96,6 @@ The functionalities of each part, Broker and Extensions, on this release, is the
   - Broker CE: `ghcr.io/sovity/broker-server-ce:8.0.0`
 - Connector UI Docker Image: `ghcr.io/sovity/edc-ui:3.2.2`
 
-
 ## [7.5.0] - 2024-05-16
 
 ### Overview
@@ -114,9 +121,9 @@ Security updates
 #### Compatible Versions
 
 - Connector Backend Docker Images:
-    - Dev EDC: `ghcr.io/sovity/edc-dev:7.5.0`
-    - sovity EDC CE: `ghcr.io/sovity/edc-ce:7.5.0`
-    - MDS EDC CE: `ghcr.io/sovity/edc-ce-mds:7.5.0`
+  - Dev EDC: `ghcr.io/sovity/edc-dev:7.5.0`
+  - sovity EDC CE: `ghcr.io/sovity/edc-ce:7.5.0`
+  - MDS EDC CE: `ghcr.io/sovity/edc-ce-mds:7.5.0`
 - Connector UI Docker Image: `ghcr.io/sovity/edc-ui:3.2.2`
 - Connector UI Release: https://github.com/sovity/edc-ui/releases/tag/v3.2.2
 
@@ -146,9 +153,9 @@ Contains DB migrations, DB backups advised.
 #### Compatible Versions
 
 - Connector Backend Docker Images:
-    - Dev EDC: `ghcr.io/sovity/edc-dev:7.4.2`
-    - sovity EDC CE: `ghcr.io/sovity/edc-ce:7.4.2`
-    - MDS EDC CE: `ghcr.io/sovity/edc-ce-mds:7.4.2`
+  - Dev EDC: `ghcr.io/sovity/edc-dev:7.4.2`
+  - sovity EDC CE: `ghcr.io/sovity/edc-ce:7.4.2`
+  - MDS EDC CE: `ghcr.io/sovity/edc-ce-mds:7.4.2`
 - Connector UI Docker Image: `ghcr.io/sovity/edc-ui:3.2.2`
 - Connector UI Release: https://github.com/sovity/edc-ui/releases/tag/v3.2.2