diff --git a/.env.dist b/.env.dist new file mode 100644 index 00000000..c0695317 --- /dev/null +++ b/.env.dist @@ -0,0 +1 @@ +MSSQL_VERSION=2022 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a75a388a..5c6d53f4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,33 +9,96 @@ env: DOCKERHUB_USER: "keboolabot" DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} KBC_TEST_PROJECT_URL: "https://connection.keboola.com/admin/projects/2703/dashboard" + KBC_TEST_PROJECT_CONFIGS: "287615945 287628364" KBC_STORAGE_TOKEN: ${{ secrets.KBC_STORAGE_TOKEN }} jobs: - Build: + build: runs-on: ubuntu-latest + outputs: + app_image_tag: ${{ steps.tag.outputs.app_image_tag }} + is_semantic_tag: ${{ steps.tag.outputs.is_semantic_tag }} steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Print Docker version + run: docker -v + - name: Docker login + if: env.DOCKERHUB_TOKEN + run: docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN" + - name: Build image + run: docker build -t $APP_IMAGE . + - name: Set image tag + id: tag run: | - docker -v - - name: Build image and run tests - run: | - docker build -t $APP_IMAGE . - docker compose run --rm wait - docker compose run --rm app + TAG="${GITHUB_REF##*/}" + IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) + echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" + echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT + echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT - name: Push image to ECR + uses: keboola/action-push-to-ecr@master + with: + vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} + app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }} + username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} + password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} + tag: ${{ steps.tag.outputs.app_image_tag }} + push_latest: ${{ steps.tag.outputs.is_semantic_tag }} + source_image: ${{ env.APP_IMAGE}} + + tests: + needs: + - build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + mssql: [ 2022, 2019 ] + steps: + - name: Check out the repo + uses: actions/checkout@v4 + - name: Pull image from ECR + uses: keboola/action-pull-from-ecr@master + with: + vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} + app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }} + username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} + password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} + tag: ${{ needs.build.outputs.app_image_tag }} + target_image: ${{ env.APP_IMAGE}} + tag_as_latest: true + - name: Run tests run: | - docker pull quay.io/keboola/developer-portal-cli-v2:latest - export REPOSITORY=`docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-repository $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP` - docker tag $APP_IMAGE:latest $REPOSITORY:test - eval $(docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-login $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP) - docker push $REPOSITORY:test - docker pull quay.io/keboola/syrup-cli:latest + export MSSQL_VERSION="${{ matrix.mssql }}" + docker compose run --rm app + + tests-in-kbc: + needs: + - build + runs-on: ubuntu-latest + steps: - name: Run KBC test jobs - run: | - docker run --rm -e KBC_STORAGE_TOKEN quay.io/keboola/syrup-cli:latest run-job keboola.ex-db-mssql 287615945 test - docker run --rm -e KBC_STORAGE_TOKEN quay.io/keboola/syrup-cli:latest run-job keboola.ex-db-mssql 287628364 test - - name: Deploy - if: startsWith(github.ref, 'refs/tags/') - run: ./deploy.sh + if: env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS + uses: keboola/action-run-configs-parallel@master + with: + token: ${{ env.KBC_STORAGE_TOKEN }} + componentId: ${{ env.KBC_DEVELOPERPORTAL_APP }} + tag: ${{ needs.build.outputs.app_image_tag }} + configs: ${{ env.KBC_TEST_PROJECT_CONFIGS }} + + deploy: + needs: + - build + - tests + - tests-in-kbc + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') && needs.build.outputs.is_semantic_tag == 'true' + steps: + - name: Set tag in the Developer Portal + uses: keboola/action-set-tag-developer-portal@master + with: + vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} + app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }} + username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} + password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} + tag: ${{ needs.build.outputs.app_image_tag }} diff --git a/.gitignore b/.gitignore index 4a1ae57a..1b92b07a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ vendor/ composer.phar .phpunit.result.cache /data/ +.env diff --git a/Dockerfile b/Dockerfile index ce0b1287..5fe35c61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,14 +20,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ wget \ libxml2-dev \ gnupg2 \ + unixodbc \ unixodbc-dev \ libgss3 \ && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list \ && apt-get update \ - && ACCEPT_EULA=Y apt-get install -y --no-install-recommends \ - msodbcsql17=17.7.1.1-1 \ - mssql-tools=17.7.1.1-1 \ + && wget https://packages.microsoft.com/debian/10/prod/pool/main/m/msodbcsql18/msodbcsql18_18.0.1.1-1_amd64.deb \ + && ACCEPT_EULA=Y dpkg -i msodbcsql18_18.0.1.1-1_amd64.deb \ + && ACCEPT_EULA=Y apt-get install -y --no-install-recommends mssql-tools \ && rm -r /var/lib/apt/lists/* \ && sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \ && locale-gen \ diff --git a/README.md b/README.md index 3f98ee96..82a8f347 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,14 @@ # MS SQL DB Extractor +## Supported MS SQL versions +The component is using Microsoft ODBC Driver for SQL Server in version `18.0.1.1-1` and supports following versions: +- SQL Server 2012 +- SQL Server 2014 +- SQL Server 2016 +- SQL Server 2017 +- SQL Server 2019 +- SQL Server 2022 + ## Configuration The configuration `config.json` contains following properties in `parameters` key: diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index 40875c8a..00000000 --- a/deploy.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e - -GITHUB_TAG=${GITHUB_REF/refs\/tags\//} - -# Obtain the component repository and log in -docker pull quay.io/keboola/developer-portal-cli-v2:latest -export REPOSITORY=`docker run --rm \ - -e KBC_DEVELOPERPORTAL_USERNAME \ - -e KBC_DEVELOPERPORTAL_PASSWORD \ - quay.io/keboola/developer-portal-cli-v2:latest \ - ecr:get-repository ${KBC_DEVELOPERPORTAL_VENDOR} ${KBC_DEVELOPERPORTAL_APP}` - -eval $(docker run --rm \ - -e KBC_DEVELOPERPORTAL_USERNAME \ - -e KBC_DEVELOPERPORTAL_PASSWORD \ - quay.io/keboola/developer-portal-cli-v2:latest \ - ecr:get-login ${KBC_DEVELOPERPORTAL_VENDOR} ${KBC_DEVELOPERPORTAL_APP}) - -# Push to the repository -docker tag ${APP_IMAGE}:latest ${REPOSITORY}:${GITHUB_TAG} -docker tag ${APP_IMAGE}:latest ${REPOSITORY}:latest -docker push ${REPOSITORY}:${GITHUB_TAG} -docker push ${REPOSITORY}:latest - -# Update the tag in Keboola Developer Portal -> Deploy to KBC -if echo ${GITHUB_TAG} | grep -c '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' -then - docker run --rm \ - -e KBC_DEVELOPERPORTAL_USERNAME \ - -e KBC_DEVELOPERPORTAL_PASSWORD \ - quay.io/keboola/developer-portal-cli-v2:latest \ - update-app-repository ${KBC_DEVELOPERPORTAL_VENDOR} ${KBC_DEVELOPERPORTAL_APP} ${GITHUB_TAG} ecr ${REPOSITORY} -else - echo "Skipping deployment to KBC, tag ${GITHUB_TAG} is not allowed." -fi diff --git a/docker-compose.yml b/docker-compose.yml index aa908493..53e57835 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,13 @@ services: - mssql - mssql-ssl - mssql-ssl-invalid-cn + depends_on: + mssql: + condition: service_healthy + mssql-ssl: + condition: service_healthy + mssql-ssl-invalid-cn: + condition: service_healthy dev: <<: *app @@ -33,11 +40,19 @@ services: build: context: . dockerfile: docker/mssql/Dockerfile + args: + MSSQL_VERSION: ${MSSQL_VERSION} ports: - "1433:1433" environment: ACCEPT_EULA: Y SA_PASSWORD: "yourStrong(!)Password" + healthcheck: + test: ["CMD-SHELL", "nc -z localhost 1433 || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 5s mssql-ssl: <<: *mssql @@ -47,6 +62,8 @@ services: build: context: . dockerfile: docker/mssql-ssl/Dockerfile + args: + MSSQL_VERSION: ${MSSQL_VERSION} mssql-ssl-invalid-cn: <<: *mssql @@ -56,6 +73,8 @@ services: build: context: . dockerfile: docker/mssql-ssl-invalid-cn/Dockerfile + args: + MSSQL_VERSION: ${MSSQL_VERSION} sshproxy: image: keboola/db-component-ssh-proxy:latest @@ -64,15 +83,5 @@ services: links: - mssql - wait: - image: waisbrot/wait - depends_on: - - mssql-ssl-invalid-cn - - mssql-ssl - - mssql - environment: - - TARGETS=mssql-ssl:1433,mssql:1433,mssql-ssl-invalid-cn:1433 - - TIMEOUT=200 - volumes: ssh-keys: diff --git a/docker/mssql-ssl-invalid-cn/Dockerfile b/docker/mssql-ssl-invalid-cn/Dockerfile index f168bac8..689838c9 100644 --- a/docker/mssql-ssl-invalid-cn/Dockerfile +++ b/docker/mssql-ssl-invalid-cn/Dockerfile @@ -1,4 +1,6 @@ -FROM mcr.microsoft.com/mssql/server:2019-latest +ARG MSSQL_VERSION=2022 + +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest USER root @@ -7,6 +9,8 @@ COPY tests/ssl-certs/mssql-invalidCn.key /etc/ssl/private/mssql.key RUN chmod 600 /etc/ssl/certs/mssql.crt /etc/ssl/private/mssql.key +RUN apt-get update && apt-get install -y netcat + RUN /opt/mssql/bin/mssql-conf set network.tlscert /etc/ssl/certs/mssql.crt \ && /opt/mssql/bin/mssql-conf set network.tlskey /etc/ssl/private/mssql.key \ && /opt/mssql/bin/mssql-conf set network.tlsprotocols 1.2 \ diff --git a/docker/mssql-ssl/Dockerfile b/docker/mssql-ssl/Dockerfile index 2ee15d32..701aeccc 100644 --- a/docker/mssql-ssl/Dockerfile +++ b/docker/mssql-ssl/Dockerfile @@ -1,4 +1,6 @@ -FROM mcr.microsoft.com/mssql/server:2019-latest +ARG MSSQL_VERSION=2022 + +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest USER root @@ -7,6 +9,8 @@ COPY tests/ssl-certs/mssql.key /etc/ssl/private/mssql.key RUN chmod 600 /etc/ssl/certs/mssql.crt /etc/ssl/private/mssql.key +RUN apt-get update && apt-get install -y netcat + RUN /opt/mssql/bin/mssql-conf set network.tlscert /etc/ssl/certs/mssql.crt \ && /opt/mssql/bin/mssql-conf set network.tlskey /etc/ssl/private/mssql.key \ && /opt/mssql/bin/mssql-conf set network.tlsprotocols 1.2 \ diff --git a/docker/mssql/Dockerfile b/docker/mssql/Dockerfile index 5a220dba..ac4ac63c 100644 --- a/docker/mssql/Dockerfile +++ b/docker/mssql/Dockerfile @@ -1,5 +1,9 @@ -FROM mcr.microsoft.com/mssql/server:2019-latest +ARG MSSQL_VERSION=2022 + +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest USER root +RUN apt-get update && apt-get install -y netcat + RUN /opt/mssql/bin/mssql-conf set sqlagent.enabled true diff --git a/tests/functional/bcp-retry/expected-stderr b/tests/functional/bcp-retry/expected-stderr index 2fe33497..7affa4f9 100644 --- a/tests/functional/bcp-retry/expected-stderr +++ b/tests/functional/bcp-retry/expected-stderr @@ -1,3 +1,3 @@ -Export by "BCP" adapter failed: Export process failed. Output: Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0002, NativeError = 208 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. BCP copy out failed . Error Output: . +Export by "BCP" adapter failed: Export process failed. Output: Starting copy... %s . Error Output: . Export by "PDO" adapter failed: [in.c-main.sales]: DB query failed: SQLSTATE[42S02]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. Tried 5 times. [in.c-main.sales]: DB query failed: SQLSTATE[42S02]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. Tried 5 times. diff --git a/tests/functional/bcp-retry/expected-stdout b/tests/functional/bcp-retry/expected-stdout index 22209f08..779c7528 100644 --- a/tests/functional/bcp-retry/expected-stdout +++ b/tests/functional/bcp-retry/expected-stdout @@ -1,11 +1,11 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM unexistsTable' queryout 'in.c-main.sales.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c -Export process failed. Output: Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0002, NativeError = 208 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. BCP copy out failed . Error Output: .. Retrying... [1x] +Export process failed. Output: Starting copy... %s . Error Output: .. Retrying... [1x] Executing BCP command: bcp 'SELECT * FROM unexistsTable' queryout 'in.c-main.sales.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c -Export process failed. Output: Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0002, NativeError = 208 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. BCP copy out failed . Error Output: .. Retrying... [2x] +Export process failed. Output: Starting copy... %s . Error Output: .. Retrying... [2x] Executing BCP command: bcp 'SELECT * FROM unexistsTable' queryout 'in.c-main.sales.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exporting by "PDO" adapter. Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' diff --git a/tests/functional/clause-with-single-quote/expected-stdout b/tests/functional/clause-with-single-quote/expected-stdout index b743940c..d1e16b09 100644 --- a/tests/functional/clause-with-single-quote/expected-stdout +++ b/tests/functional/clause-with-single-quote/expected-stdout @@ -1,6 +1,6 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "special" to "in.c-main.special". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT TOP 5 "usergender", "sku" FROM "sales" WHERE "usergender" LIKE '\''male'\''' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "5" rows to "in.c-main.special". diff --git a/tests/functional/different-quoting/expected-stdout b/tests/functional/different-quoting/expected-stdout index 7805a5f1..2e9c28bb 100644 --- a/tests/functional/different-quoting/expected-stdout +++ b/tests/functional/different-quoting/expected-stdout @@ -1,6 +1,6 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "special" to "in.c-main.special". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT usergender, [sku] FROM sales WHERE "usergender" LIKE '\''male'\''' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "40" rows to "in.c-main.special". diff --git a/tests/functional/error-invalid-query/expected-stderr b/tests/functional/error-invalid-query/expected-stderr index 9a4193de..cfd274a0 100644 --- a/tests/functional/error-invalid-query/expected-stderr +++ b/tests/functional/error-invalid-query/expected-stderr @@ -1,3 +1,3 @@ -Export by "BCP" adapter failed: Export process failed. Output: Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0022, NativeError = 207 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'SOMETHING'. BCP copy out failed . Error Output: . +Export by "BCP" adapter failed: Export process failed. Output: Starting copy... %s . Error Output: . Export by "PDO" adapter failed: [in.c-main.special]: DB query failed: SQLSTATE[42S22]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'SOMETHING'. Tried 5 times. [in.c-main.special]: DB query failed: SQLSTATE[42S22]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'SOMETHING'. Tried 5 times. diff --git a/tests/functional/error-invalid-query/expected-stdout b/tests/functional/error-invalid-query/expected-stdout index 3c185d99..d338cb39 100644 --- a/tests/functional/error-invalid-query/expected-stdout +++ b/tests/functional/error-invalid-query/expected-stdout @@ -1,7 +1,7 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "special" to "in.c-main.special". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT SOMETHING INVALID FROM "dbo"."special"' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exporting by "PDO" adapter. Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' diff --git a/tests/functional/pdo-fallback-disabled-config-row/expected-stdout b/tests/functional/pdo-fallback-disabled-config-row/expected-stdout index e41b644b..1b5c7356 100644 --- a/tests/functional/pdo-fallback-disabled-config-row/expected-stdout +++ b/tests/functional/pdo-fallback-disabled-config-row/expected-stdout @@ -1,7 +1,7 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "special" to "in.c-main.special". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM "special";' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exporting by "PDO" adapter. Adapter "PDO" skipped: Disabled in configuration. diff --git a/tests/functional/pdo-fallback-disabled/expected-stdout b/tests/functional/pdo-fallback-disabled/expected-stdout index e41b644b..1b5c7356 100644 --- a/tests/functional/pdo-fallback-disabled/expected-stdout +++ b/tests/functional/pdo-fallback-disabled/expected-stdout @@ -1,7 +1,7 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "special" to "in.c-main.special". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM "special";' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exporting by "PDO" adapter. Adapter "PDO" skipped: Disabled in configuration. diff --git a/tests/functional/pdo-fallback/expected-stdout b/tests/functional/pdo-fallback/expected-stdout index 977b9c9a..11b0308b 100644 --- a/tests/functional/pdo-fallback/expected-stdout +++ b/tests/functional/pdo-fallback/expected-stdout @@ -1,7 +1,7 @@ Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test' Exporting "special" to "in.c-main.special". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM "special";' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exporting by "PDO" adapter. Exported "7" rows to "in.c-main.special". diff --git a/tests/functional/simplified-pdo-fallback-query/expected-stderr b/tests/functional/simplified-pdo-fallback-query/expected-stderr index da7218a3..f3ce894e 100644 --- a/tests/functional/simplified-pdo-fallback-query/expected-stderr +++ b/tests/functional/simplified-pdo-fallback-query/expected-stderr @@ -1 +1 @@ -Export by "BCP" adapter failed: Export process failed. Output: Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = 37000, NativeError = 8116 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Argument data type sql_variant is invalid for argument 1 of replace function. BCP copy out failed . Error Output: . +Export by "BCP" adapter failed: Export process failed. Output: Starting copy... %s . Error Output: . diff --git a/tests/functional/ssl-not-verify-cert-missing-certificate/expected-stdout b/tests/functional/ssl-not-verify-cert-missing-certificate/expected-stdout index 345e215c..da9ffc38 100644 --- a/tests/functional/ssl-not-verify-cert-missing-certificate/expected-stdout +++ b/tests/functional/ssl-not-verify-cert-missing-certificate/expected-stdout @@ -2,6 +2,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl,1433;Database=test;Encrypt=true;Trust Using SSL connection Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "100" rows to "in.c-main.sales". diff --git a/tests/functional/ssl-not-verify-cert-valid-certificate/expected-stdout b/tests/functional/ssl-not-verify-cert-valid-certificate/expected-stdout index 345e215c..da9ffc38 100644 --- a/tests/functional/ssl-not-verify-cert-valid-certificate/expected-stdout +++ b/tests/functional/ssl-not-verify-cert-valid-certificate/expected-stdout @@ -2,6 +2,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl,1433;Database=test;Encrypt=true;Trust Using SSL connection Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "100" rows to "in.c-main.sales". diff --git a/tests/functional/ssl-verify-cert-ignore-invalid-cn-certificate/expected-stdout b/tests/functional/ssl-verify-cert-ignore-invalid-cn-certificate/expected-stdout index c909529e..8e4e340f 100644 --- a/tests/functional/ssl-verify-cert-ignore-invalid-cn-certificate/expected-stdout +++ b/tests/functional/ssl-verify-cert-ignore-invalid-cn-certificate/expected-stdout @@ -3,7 +3,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl-invalid-cn,1433;Database=test;Encrypt Using SSL connection Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl-invalid-cn,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "100" rows to "in.c-main.sales". - diff --git a/tests/functional/ssl-verify-cert-invalid-cn-certificate/expected-stdout b/tests/functional/ssl-verify-cert-invalid-cn-certificate/expected-stdout index 87088b29..8e4e340f 100644 --- a/tests/functional/ssl-verify-cert-invalid-cn-certificate/expected-stdout +++ b/tests/functional/ssl-verify-cert-invalid-cn-certificate/expected-stdout @@ -3,6 +3,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl-invalid-cn,1433;Database=test;Encrypt Using SSL connection Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl-invalid-cn,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "100" rows to "in.c-main.sales". diff --git a/tests/functional/ssl-verify-cert-valid-certificate/expected-stdout b/tests/functional/ssl-verify-cert-valid-certificate/expected-stdout index 1e7bb5d1..5af4c09e 100644 --- a/tests/functional/ssl-verify-cert-valid-certificate/expected-stdout +++ b/tests/functional/ssl-verify-cert-valid-certificate/expected-stdout @@ -2,6 +2,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl,1433;Database=test;Encrypt=true;Trust Using SSL connection Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "100" rows to "in.c-main.sales". diff --git a/tests/functional/ssl-without-ca/expected-stdout b/tests/functional/ssl-without-ca/expected-stdout index 4fa74e6e..446c2245 100644 --- a/tests/functional/ssl-without-ca/expected-stdout +++ b/tests/functional/ssl-without-ca/expected-stdout @@ -2,6 +2,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl,1433;Database=test;Encrypt=true;Trust Using SSL connection Exporting "sales" to "in.c-main.sales". Exporting by "BCP" adapter. -Found database server version: 15.%d.%d.%d +Found database server version: %d.%d.%d.%d Executing BCP command: bcp 'SELECT TOP 3 * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c Exported "3" rows to "in.c-main.sales". diff --git a/tests/traits/RemoveAllTablesTrait.php b/tests/traits/RemoveAllTablesTrait.php index b503c6df..7b1a6241 100644 --- a/tests/traits/RemoveAllTablesTrait.php +++ b/tests/traits/RemoveAllTablesTrait.php @@ -5,6 +5,7 @@ namespace Keboola\DbExtractor\TraitTests; use PDO; +use PDOException; trait RemoveAllTablesTrait { @@ -16,23 +17,60 @@ protected function removeAllTables(): void { $this->removeAllFkConstraints(); - // Delete all tables, except sys tables - $sql = 'SELECT * FROM information_schema.tables'; + // Delete all tables, excluding system tables (sys, cdc) + $sql = " + SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE + FROM information_schema.tables + WHERE TABLE_SCHEMA NOT IN ('sys', 'cdc') + AND TABLE_NAME NOT LIKE 'sys%' + AND TABLE_TYPE = 'BASE TABLE' + "; + /** @var \PDOStatement $stmt */ $stmt = $this->connection->query($sql); /** @var array $tables */ $tables = $stmt->fetchAll(PDO::FETCH_ASSOC); + foreach ($tables as $table) { - $statement = 'DROP TABLE'; - if ($table['TABLE_TYPE'] === 'VIEW') { - $statement = 'DROP VIEW'; + $schema = $this->quoteIdentifier($table['TABLE_SCHEMA']); + $tableName = $this->quoteIdentifier($table['TABLE_NAME']); + + // Check if CDC is enabled by querying the system CDC tables directly + $cdcEnabled = false; + $captureInstance = null; + try { + $cdcCheckSql = sprintf( + "SELECT capture_instance FROM cdc.change_tables WHERE source_object_id = OBJECT_ID('%s.%s')", + $table['TABLE_SCHEMA'], + $table['TABLE_NAME'], + ); + $cdcCheckStmt = $this->connection->query($cdcCheckSql); + $cdcRow = $cdcCheckStmt->fetch(PDO::FETCH_ASSOC); + if (is_array($cdcRow) && isset($cdcRow['capture_instance'])) { + $cdcEnabled = true; + $captureInstance = $cdcRow['capture_instance']; + } + } catch (PDOException $e) { + // Ignore error if CDC is not enabled + if (!str_contains($e->getMessage(), 'Invalid object name')) { + throw $e; + } } - $this->connection->query(sprintf( - '%s %s.%s', - $statement, - $this->quoteIdentifier($table['TABLE_SCHEMA']), - $this->quoteIdentifier($table['TABLE_NAME']), - )); + + // If CDC is enabled, disable it before dropping the table + if ($cdcEnabled && $captureInstance) { + $disableCdcSql = sprintf( + "EXEC sys.sp_cdc_disable_table @source_schema = '%s', @source_name = '%s', " . + "@capture_instance = '%s'", + $table['TABLE_SCHEMA'], + $table['TABLE_NAME'], + $captureInstance, + ); + $this->connection->exec($disableCdcSql); + } + + // Drop the table + $this->connection->query(sprintf('DROP TABLE %s.%s', $schema, $tableName)); } } diff --git a/wait-for-it.sh b/wait-for-it.sh deleted file mode 100755 index 12f10ee7..00000000 --- a/wait-for-it.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available -# https://github.com/vishnubob/wait-for-it - -cmdname=$(basename $0) - -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $TIMEOUT -gt 0 ]]; then - echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" - else - echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" - fi - start_ts=$(date +%s) - while : - do - if [[ $ISBUSY -eq 1 ]]; then - nc -z $HOST $PORT - result=$? - else - (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 - result=$? - fi - if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" - break - fi - sleep 1 - done - return $result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - hostport=(${1//:/ }) - HOST=${hostport[0]} - PORT=${hostport[1]} - shift 1 - ;; - --child) - CHILD=1 - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -s | --strict) - STRICT=1 - shift 1 - ;; - -h) - HOST="$2" - if [[ $HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - HOST="${1#*=}" - shift 1 - ;; - -p) - PORT="$2" - if [[ $PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - PORT="${1#*=}" - shift 1 - ;; - -t) - TIMEOUT="$2" - if [[ $TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$HOST" == "" || "$PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -TIMEOUT=${TIMEOUT:-15} -STRICT=${STRICT:-0} -CHILD=${CHILD:-0} -QUIET=${QUIET:-0} - -# check to see if timeout is from busybox? -# check to see if timeout is from busybox? -TIMEOUT_PATH=$(realpath $(which timeout)) -if [[ $TIMEOUT_PATH =~ "busybox" ]]; then - ISBUSY=1 - BUSYTIMEFLAG="-t" -else - ISBUSY=0 - BUSYTIMEFLAG="" -fi - -if [[ $CHILD -gt 0 ]]; then - wait_for - RESULT=$? - exit $RESULT -else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi -fi - -if [[ $CLI != "" ]]; then - if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then - echoerr "$cmdname: strict mode, refusing to execute subprocess" - exit $RESULT - fi - exec "${CLI[@]}" -else - exit $RESULT -fi