testflow-sink-connector-lightweight.yml

name: TestFlows Tests - Sink Connector(Light-weight)

on:
  push:
    branches: [ main, develop]
  pull_request:
    branches: [ main , develop]
  workflow_dispatch:
    inputs:
      sink_version:
        description: "sink connector version"
        required: true
        type: string
        default: 2023-08-11

env:
#  SINK_CONNECTOR_VERSION: "${{ inputs.sink_version }}"
#  SINK_CONNECTOR_VERSION: "2023-08-11"
  SINK_CONNECTOR_VERSION: "latest"

jobs:
  testflows_lightweight:
    runs-on: [self-hosted, on-demand, type-cpx51, image-x86-app-docker-ce]

    steps:
      - uses: actions/checkout@v2

      - name: Runner ssh command
        working-directory: sink-connector/tests/integration
        run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
      
      - name: Install docker-compose
        working-directory: sink-connector-lightweight/tests/integration
        run: pip3 install docker-compose==1.29.2
      
      - name: Install testflows
        working-directory: sink-connector-lightweight/tests/integration
        run: pip3 install testflows

      - name: Install awscli
        working-directory: sink-connector-lightweight/tests/integration
        run: pip3 install awscli

      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y-%m-%d_%H%M%S')" >> $GITHUB_OUTPUT

      - name: Add ~./local/bin to the PATH
        if: always()
        working-directory: sink-connector-lightweight/tests/integration
        run: echo ~/.local/bin >> $GITHUB_PATH

      - name: Run testflows tests
        working-directory: sink-connector-lightweight/tests/integration
        run: python3 -u regression.py --only "/mysql to clickhouse replication/mysql to clickhouse replication auto/*" --clickhouse-binary-path=docker://clickhouse/clickhouse-server:23.3 --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log

      - name: Create tfs results report
        if: always()
        working-directory: sink-connector-lightweight/tests/integration/logs
        run: cat raw.log | tfs report results | tfs document convert > report.html

      - name: Create tfs coverage report
        if: always()
        working-directory: sink-connector-lightweight/tests/integration/logs
        run: cat raw.log | tfs report coverage ../requirements/requirements.py | tfs document convert > coverage.html

      - name: Upload artifacts to Altinity Test Reports S3 bucket
        if: always()
        working-directory: sink-connector-lightweight/tests/integration/logs
        run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html"
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: 'eu-west-2'

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: testflows-artifacts-${{ steps.date.outputs.date }}
          path: |
            sink-connector-lightweight/tests/integration/logs/*.log
          if-no-files-found: error
          retention-days: 60