integration.yml

apiVersion: camel.apache.org/v1
kind: Integration
metadata:
  name: s3-to-s3
spec:
  dependencies:
    - "camel:camel-csv"
  configuration:
    - type: configmap
      value: rss-to-s3-configmap
    - type: secret
      value: aws
  flows:
  - from:
      uri: aws-s3://hume-datasets
      parameters:
        prefix: articles
        accessKey: "{{ACCESS_KEY}}"
        secretKey: "{{SECRET_KEY}}"
        region: "{{REGION}}"
        deleteAfterRead: false
      steps:
        - unmarshal:
            csv:
              use-maps: true
              allow-missing-column-names: true
        - split:
            simple: "${body}"
            streaming: true
# Remove the headers on the exchange, otherwise it will try to write on the same bucket but also the original content length and target content length are different
        - remove-headers:
            pattern: "*"
        - set-header:
            name: CamelAwsS3Key
            simple: "articles/converted/${body[id]}.json"
        - marshal:
            json: {}
        - to:
            uri: aws-s3://{{BUCKET}}
            parameters:
                accessKey: "{{ACCESS_KEY}}"
                secretKey: "{{SECRET_KEY}}"
                region: "{{REGION}}"