From 94bc98c530c8f5d0a9853ea317282fa57c9ae455 Mon Sep 17 00:00:00 2001 From: Andrea Cosentino Date: Fri, 25 Jun 2021 13:05:33 +0200 Subject: [PATCH 1/3] MongoDB Source Kamelet: Added persistentTailTracking support --- mongodb-source.kamelet.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/mongodb-source.kamelet.yaml b/mongodb-source.kamelet.yaml index 83e850106..3738e3ed3 100644 --- a/mongodb-source.kamelet.yaml +++ b/mongodb-source.kamelet.yaml @@ -14,7 +14,11 @@ spec: definition: title: "MongoDB Source" description: |- - Consume documents from MongoDB + Consume documents from MongoDB. + + If the persistentTailTracking option will be enabled, the consumer will keep track of the last consumed message and on the next restart, the consumption will restart from that message. In case of persistentTailTracking enabled, the tailTrackIncreasingField must be provided (by default it is optional). + + If the persistentTailTracking option won't be enabled, the consumer will consume the whole collection and wait in idle for new documents to consume. required: - hosts - collection @@ -46,6 +50,17 @@ spec: title: MongoDB Database description: Sets the name of the MongoDB database to target. type: string + persistentTailTracking: + title: MongoDB Persistent Tail Tracking + description: Enable persistent tail tracking, which is a mechanism to keep track of the last consumed message across system restarts. The next time the system is up, the endpoint will recover the cursor from the point where it last stopped slurping records. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false + tailTrackIncreasingField: + title: MongoDB Tail Track Increasing Field + description: Correlation field in the incoming record which is of increasing nature and will be used to position the tailing cursor every time it is generated. + type: string dependencies: - "camel:kamelet" - "camel:mongodb" @@ -62,6 +77,8 @@ spec: password: "{{password}}" username: "{{username}}" database: "{{database}}" + persistentTailTracking: "{{persistentTailTracking}}" + tailTrackIncreasingField: "{{?tailTrackIncreasingField}}" steps: - marshal: json: {} From c37867da9df5a2bc82bf736ac4e378095805e36d Mon Sep 17 00:00:00 2001 From: Andrea Cosentino Date: Fri, 25 Jun 2021 13:06:21 +0200 Subject: [PATCH 2/3] MongoDB Source Kamelet: Added persistentTailTracking support --- .../kamelets/mongodb-source.kamelet.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/library/camel-kamelets-catalog/src/main/resources/kamelets/mongodb-source.kamelet.yaml b/library/camel-kamelets-catalog/src/main/resources/kamelets/mongodb-source.kamelet.yaml index 83e850106..3738e3ed3 100644 --- a/library/camel-kamelets-catalog/src/main/resources/kamelets/mongodb-source.kamelet.yaml +++ b/library/camel-kamelets-catalog/src/main/resources/kamelets/mongodb-source.kamelet.yaml @@ -14,7 +14,11 @@ spec: definition: title: "MongoDB Source" description: |- - Consume documents from MongoDB + Consume documents from MongoDB. + + If the persistentTailTracking option will be enabled, the consumer will keep track of the last consumed message and on the next restart, the consumption will restart from that message. In case of persistentTailTracking enabled, the tailTrackIncreasingField must be provided (by default it is optional). + + If the persistentTailTracking option won't be enabled, the consumer will consume the whole collection and wait in idle for new documents to consume. required: - hosts - collection @@ -46,6 +50,17 @@ spec: title: MongoDB Database description: Sets the name of the MongoDB database to target. type: string + persistentTailTracking: + title: MongoDB Persistent Tail Tracking + description: Enable persistent tail tracking, which is a mechanism to keep track of the last consumed message across system restarts. The next time the system is up, the endpoint will recover the cursor from the point where it last stopped slurping records. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false + tailTrackIncreasingField: + title: MongoDB Tail Track Increasing Field + description: Correlation field in the incoming record which is of increasing nature and will be used to position the tailing cursor every time it is generated. + type: string dependencies: - "camel:kamelet" - "camel:mongodb" @@ -62,6 +77,8 @@ spec: password: "{{password}}" username: "{{username}}" database: "{{database}}" + persistentTailTracking: "{{persistentTailTracking}}" + tailTrackIncreasingField: "{{?tailTrackIncreasingField}}" steps: - marshal: json: {} From d8d5c8dd0f54b68b9153521bb65589fbd1ae902b Mon Sep 17 00:00:00 2001 From: Andrea Cosentino Date: Fri, 25 Jun 2021 13:06:53 +0200 Subject: [PATCH 3/3] MongoDB Source Kamelet: Added persistentTailTracking support --- docs/modules/ROOT/pages/mongodb-source.adoc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/mongodb-source.adoc b/docs/modules/ROOT/pages/mongodb-source.adoc index a49b9a17f..358ecf5b2 100644 --- a/docs/modules/ROOT/pages/mongodb-source.adoc +++ b/docs/modules/ROOT/pages/mongodb-source.adoc @@ -5,7 +5,11 @@ *Support Level for this Kamelet is: "Preview"* -Consume documents from MongoDB +Consume documents from MongoDB. + +If the persistentTailTracking option will be enabled, the consumer will keep track of the last consumed message and on the next restart, the consumption will restart from that message. In case of persistentTailTracking enabled, the tailTrackIncreasingField must be provided (by default it is optional). + +If the persistentTailTracking option won't be enabled, the consumer will consume the whole collection and wait in idle for new documents to consume. == Configuration Options @@ -18,6 +22,8 @@ The following table summarizes the configuration options available for the `mong | *hosts {empty}* *| MongoDB Hosts| Comma separated list of MongoDB Host Addresses in host:port format.| string| | | *password {empty}* *| MongoDB Password| User password for accessing MongoDB.| string| | | *username {empty}* *| MongoDB Username| Username for accessing MongoDB.| string| | +| persistentTailTracking| MongoDB Persistent Tail Tracking| Enable persistent tail tracking, which is a mechanism to keep track of the last consumed message across system restarts. The next time the system is up, the endpoint will recover the cursor from the point where it last stopped slurping records.| boolean| `false`| +| tailTrackIncreasingField| MongoDB Tail Track Increasing Field| Correlation field in the incoming record which is of increasing nature and will be used to position the tailing cursor every time it is generated.| string| | |=== NOTE: Fields marked with ({empty}*) are mandatory.