quix.yaml

# Quix Project Descriptor
# This file describes the data pipeline and configuration of resources of a Quix Project.

metadata:
  version: 1.0

# This section describes the Deployments of the data pipeline
deployments:
  - name: Kinesis on localstack
    application: localstack kinesis
    version: latest
    deploymentType: Service
    resources:
      cpu: 1000
      memory: 1000
      replicas: 1
    state:
      enabled: true
      size: 3
    network:
      serviceName: kinesis
      ports:
        - port: 4566
          targetPort: 4566
    variables: []
  - name: Publish to Kinesis
    application: publish-kinesis
    version: latest
    deploymentType: Service
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    variables:
      - name: aws_region_name
        inputType: FreeText
        description: ''
        required: false
        value: us-east-1
      - name: aws_secret_access_key
        inputType: Secret
        description: ''
        required: false
        secretKey: aws_secret_access_key
      - name: aws_access_key_id
        inputType: Secret
        description: ''
        required: false
        secretKey: aws_access_key_id
      - name: kinesis_stream_name
        inputType: FreeText
        description: ''
        required: false
        value: kinesis_data
  - name: Kinesis Source
    application: kinesis-source
    version: latest
    deploymentType: Service
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    variables:
      - name: output
        inputType: OutputTopic
        description: Name of the output topic to write into
        required: true
        value: kinesis_data
      - name: kinesis_stream
        inputType: FreeText
        description: ''
        required: false
        value: kinesis_data
      - name: aws_region_name
        inputType: FreeText
        description: ''
        required: false
        value: us-east-1
      - name: aws_secret_access_key
        inputType: Secret
        description: ''
        required: false
        secretKey: aws_secret_access_key
      - name: aws_access_key_id
        inputType: Secret
        description: ''
        required: false
        secretKey: aws_access_key_id
  - name: Transformation
    application: transformation
    version: latest
    deploymentType: Service
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    variables:
      - name: input
        inputType: InputTopic
        description: Name of the input topic to listen to.
        required: false
        value: kinesis_data
      - name: output
        inputType: OutputTopic
        description: Name of the output topic to write to.
        required: false
        value: transformed
  - name: BigQuery Sink
    application: BigQuery Sink
    version: latest
    deploymentType: Service
    resources:
      cpu: 200
      memory: 500
      replicas: 1
    variables:
      - name: input
        inputType: InputTopic
        description: This is the input topic
        required: true
        value: transformed
      - name: PROJECT_ID
        inputType: FreeText
        description: The BigQuery GCP Project ID
        required: true
        value: steve-438814
      - name: DATASET_ID
        inputType: FreeText
        description: The target BigQuery dataset ID
        required: true
        value: steve-438814.demo
      - name: DATASET_LOCATION
        inputType: FreeText
        description: Location of BigQuery dataset
        required: true
        value: EU
      - name: TABLE_NAME
        inputType: FreeText
        description: Name of the destination table.
        required: true
        value: demo2
      - name: CONSUMER_GROUP
        inputType: FreeText
        description: Kafka consumer group.
        required: true
        value: big-query-sink-v1
      - name: BIGQUERY_SERVICE_ACCOUNT_JSON
        inputType: Secret
        description: JSON string of the service account file for the BigQuery GCP project
        required: true
        secretKey: google_demo_service_account
      - name: MAX_QUEUE_SIZE
        inputType: FreeText
        description: Max queue size for the sink ingestion
        required: true
        value: 50

# This section describes the Topics of the data pipeline
topics:
  - name: kinesis_data
    configuration:
      replicationFactor: 2
      retentionInMinutes: 1440
      retentionInBytes: 52428800
  - name: transformed
    configuration:
      replicationFactor: 2
      retentionInMinutes: 1440
      retentionInBytes: 52428800
    dataTier: Silver