Merge pull request #239 from keboola/adamvyborny-mssql-drive-update

MSSQL driver update

.env.dist

@@ -0,0 +1 @@
+MSSQL_VERSION=2022

.github/workflows/push.yml

@@ -9,33 +9,96 @@ env:
   DOCKERHUB_USER: "keboolabot"
   DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
   KBC_TEST_PROJECT_URL: "https://connection.keboola.com/admin/projects/2703/dashboard"
+  KBC_TEST_PROJECT_CONFIGS: "287615945 287628364"
   KBC_STORAGE_TOKEN: ${{ secrets.KBC_STORAGE_TOKEN }}
 jobs:
-  Build:
+  build:
     runs-on: ubuntu-latest
+    outputs:
+      app_image_tag: ${{ steps.tag.outputs.app_image_tag }}
+      is_semantic_tag: ${{ steps.tag.outputs.is_semantic_tag }}
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Print Docker version
+        run: docker -v
+      - name: Docker login
+        if: env.DOCKERHUB_TOKEN
+        run: docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"
+      - name: Build image
+        run: docker build -t $APP_IMAGE .
+      - name: Set image tag
+        id: tag
         run: |
-          docker -v
-      - name: Build image and run tests
-        run: |
-          docker build -t $APP_IMAGE .
-          docker compose run --rm wait
-          docker compose run --rm app
+          TAG="${GITHUB_REF##*/}"
+          IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false)
+          echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'"
+          echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT
+          echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT
       - name: Push image to ECR
+        uses: keboola/action-push-to-ecr@master
+        with:
+          vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
+          app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
+          password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
+          tag: ${{ steps.tag.outputs.app_image_tag }}
+          push_latest: ${{ steps.tag.outputs.is_semantic_tag }}
+          source_image: ${{ env.APP_IMAGE}}
+
+  tests:
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        mssql: [ 2022, 2019 ]
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Pull image from ECR
+        uses: keboola/action-pull-from-ecr@master
+        with:
+          vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
+          app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
+          password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
+          tag: ${{ needs.build.outputs.app_image_tag }}
+          target_image: ${{ env.APP_IMAGE}}
+          tag_as_latest: true
+      - name: Run tests
         run: |
-          docker pull quay.io/keboola/developer-portal-cli-v2:latest
-          export REPOSITORY=`docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-repository $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP`
-          docker tag $APP_IMAGE:latest $REPOSITORY:test
-          eval $(docker run --rm -e KBC_DEVELOPERPORTAL_USERNAME -e KBC_DEVELOPERPORTAL_PASSWORD -e KBC_DEVELOPERPORTAL_URL quay.io/keboola/developer-portal-cli-v2:latest ecr:get-login $KBC_DEVELOPERPORTAL_VENDOR $KBC_DEVELOPERPORTAL_APP)
-          docker push $REPOSITORY:test
-          docker pull quay.io/keboola/syrup-cli:latest
+          export MSSQL_VERSION="${{ matrix.mssql }}"
+          docker compose run --rm app
+
+  tests-in-kbc:
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    steps:
       - name: Run KBC test jobs
-        run: |
-          docker run --rm -e KBC_STORAGE_TOKEN quay.io/keboola/syrup-cli:latest run-job keboola.ex-db-mssql 287615945 test
-          docker run --rm -e KBC_STORAGE_TOKEN quay.io/keboola/syrup-cli:latest run-job keboola.ex-db-mssql 287628364 test
-      - name: Deploy
-        if: startsWith(github.ref, 'refs/tags/')
-        run: ./deploy.sh
+        if: env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS
+        uses: keboola/action-run-configs-parallel@master
+        with:
+          token: ${{ env.KBC_STORAGE_TOKEN }}
+          componentId: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          tag: ${{ needs.build.outputs.app_image_tag }}
+          configs: ${{ env.KBC_TEST_PROJECT_CONFIGS }}
+
+  deploy:
+    needs:
+      - build
+      - tests
+      - tests-in-kbc
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/') && needs.build.outputs.is_semantic_tag == 'true'
+    steps:
+      - name: Set tag in the Developer Portal
+        uses: keboola/action-set-tag-developer-portal@master
+        with:
+          vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
+          app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
+          password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
+          tag: ${{ needs.build.outputs.app_image_tag }}

.gitignore

@@ -7,3 +7,4 @@ vendor/
 composer.phar
 .phpunit.result.cache
 /data/
+.env

Dockerfile

@@ -20,14 +20,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     wget \
     libxml2-dev \
     gnupg2 \
+    unixodbc \
     unixodbc-dev \
     libgss3 \
     && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
     && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list \
     && apt-get update \
-    && ACCEPT_EULA=Y apt-get install -y --no-install-recommends \
-    msodbcsql17=17.7.1.1-1 \
-    mssql-tools=17.7.1.1-1 \
+    && wget https://packages.microsoft.com/debian/10/prod/pool/main/m/msodbcsql18/msodbcsql18_18.0.1.1-1_amd64.deb \
+    && ACCEPT_EULA=Y dpkg -i msodbcsql18_18.0.1.1-1_amd64.deb \
+    && ACCEPT_EULA=Y apt-get install -y --no-install-recommends mssql-tools \
     && rm -r /var/lib/apt/lists/* \
     && sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \
     && locale-gen \

README.md

@@ -1,5 +1,14 @@
 # MS SQL DB Extractor
 
+## Supported MS SQL versions
+The component is using Microsoft ODBC Driver for SQL Server in version `18.0.1.1-1` and supports following versions:
+- SQL Server 2012
+- SQL Server 2014
+- SQL Server 2016
+- SQL Server 2017
+- SQL Server 2019
+- SQL Server 2022
+
 ## Configuration
 
 The configuration `config.json` contains following properties in `parameters` key:

docker-compose.yml

@@ -20,6 +20,13 @@ services:
       - mssql
       - mssql-ssl
       - mssql-ssl-invalid-cn
+    depends_on:
+      mssql:
+        condition: service_healthy
+      mssql-ssl:
+        condition: service_healthy
+      mssql-ssl-invalid-cn:
+        condition: service_healthy
 
   dev:
     <<: *app
@@ -33,11 +40,19 @@ services:
     build:
       context: .
       dockerfile: docker/mssql/Dockerfile
+      args:
+        MSSQL_VERSION: ${MSSQL_VERSION}
     ports:
       - "1433:1433"
     environment:
       ACCEPT_EULA: Y
       SA_PASSWORD: "yourStrong(!)Password"
+    healthcheck:
+      test: ["CMD-SHELL", "nc -z localhost 1433 || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 3
+      start_period: 5s
 
   mssql-ssl:
     <<: *mssql
@@ -47,6 +62,8 @@ services:
     build:
       context: .
       dockerfile: docker/mssql-ssl/Dockerfile
+      args:
+        MSSQL_VERSION: ${MSSQL_VERSION}
 
   mssql-ssl-invalid-cn:
     <<: *mssql
@@ -56,6 +73,8 @@ services:
     build:
       context: .
       dockerfile: docker/mssql-ssl-invalid-cn/Dockerfile
+      args:
+        MSSQL_VERSION: ${MSSQL_VERSION}
 
   sshproxy:
     image: keboola/db-component-ssh-proxy:latest
@@ -64,15 +83,5 @@ services:
     links:
       - mssql
 
-  wait:
-    image: waisbrot/wait
-    depends_on:
-      - mssql-ssl-invalid-cn
-      - mssql-ssl
-      - mssql
-    environment:
-      - TARGETS=mssql-ssl:1433,mssql:1433,mssql-ssl-invalid-cn:1433
-      - TIMEOUT=200
-
 volumes:
   ssh-keys:

docker/mssql-ssl-invalid-cn/Dockerfile

@@ -1,4 +1,6 @@
-FROM mcr.microsoft.com/mssql/server:2019-latest
+ARG MSSQL_VERSION=2022
+
+FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest
 
 USER root
 
@@ -7,6 +9,8 @@ COPY tests/ssl-certs/mssql-invalidCn.key /etc/ssl/private/mssql.key
 
 RUN chmod 600 /etc/ssl/certs/mssql.crt /etc/ssl/private/mssql.key
 
+RUN apt-get update && apt-get install -y netcat
+
 RUN /opt/mssql/bin/mssql-conf set network.tlscert /etc/ssl/certs/mssql.crt \
       && /opt/mssql/bin/mssql-conf set network.tlskey /etc/ssl/private/mssql.key \
       && /opt/mssql/bin/mssql-conf set network.tlsprotocols 1.2 \

docker/mssql-ssl/Dockerfile

@@ -1,4 +1,6 @@
-FROM mcr.microsoft.com/mssql/server:2019-latest
+ARG MSSQL_VERSION=2022
+
+FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest
 
 USER root
 
@@ -7,6 +9,8 @@ COPY tests/ssl-certs/mssql.key /etc/ssl/private/mssql.key
 
 RUN chmod 600 /etc/ssl/certs/mssql.crt /etc/ssl/private/mssql.key
 
+RUN apt-get update && apt-get install -y netcat
+
 RUN /opt/mssql/bin/mssql-conf set network.tlscert /etc/ssl/certs/mssql.crt \
       && /opt/mssql/bin/mssql-conf set network.tlskey /etc/ssl/private/mssql.key \
       && /opt/mssql/bin/mssql-conf set network.tlsprotocols 1.2 \

docker/mssql/Dockerfile

@@ -1,5 +1,9 @@
-FROM mcr.microsoft.com/mssql/server:2019-latest
+ARG MSSQL_VERSION=2022
+
+FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest
 
 USER root
 
+RUN apt-get update && apt-get install -y netcat
+
 RUN /opt/mssql/bin/mssql-conf set sqlagent.enabled true

tests/functional/bcp-retry/expected-stderr

@@ -1,3 +1,3 @@
-Export by "BCP" adapter failed: Export process failed. Output:  Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0002, NativeError = 208 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'.  BCP copy out failed .    Error Output: .
+Export by "BCP" adapter failed: Export process failed. Output:  Starting copy... %s .    Error Output: .
 Export by "PDO" adapter failed: [in.c-main.sales]: DB query failed: SQLSTATE[42S02]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. Tried 5 times.
 [in.c-main.sales]: DB query failed: SQLSTATE[42S02]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'. Tried 5 times.

tests/functional/bcp-retry/expected-stdout

@@ -1,11 +1,11 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "sales" to "in.c-main.sales".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT * FROM unexistsTable' queryout 'in.c-main.sales.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
-Export process failed. Output:  Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0002, NativeError = 208 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'.  BCP copy out failed .    Error Output: .. Retrying... [1x]
+Export process failed. Output:  Starting copy... %s .    Error Output: .. Retrying... [1x]
 Executing BCP command: bcp 'SELECT * FROM unexistsTable' queryout 'in.c-main.sales.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
-Export process failed. Output:  Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0002, NativeError = 208 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid object name 'unexistsTable'.  BCP copy out failed .    Error Output: .. Retrying... [2x]
+Export process failed. Output:  Starting copy... %s .    Error Output: .. Retrying... [2x]
 Executing BCP command: bcp 'SELECT * FROM unexistsTable' queryout 'in.c-main.sales.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exporting by "PDO" adapter.
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'

tests/functional/clause-with-single-quote/expected-stdout

@@ -1,6 +1,6 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "special" to "in.c-main.special".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT TOP 5 "usergender", "sku" FROM "sales" WHERE "usergender" LIKE '\''male'\''' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exported "5" rows to "in.c-main.special".

tests/functional/different-quoting/expected-stdout

@@ -1,6 +1,6 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "special" to "in.c-main.special".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT usergender, [sku]  FROM sales WHERE "usergender" LIKE '\''male'\''' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exported "40" rows to "in.c-main.special".

tests/functional/error-invalid-query/expected-stderr

@@ -1,3 +1,3 @@
-Export by "BCP" adapter failed: Export process failed. Output:  Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = S0022, NativeError = 207 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'SOMETHING'.  BCP copy out failed .    Error Output: .
+Export by "BCP" adapter failed: Export process failed. Output:  Starting copy... %s .    Error Output: .
 Export by "PDO" adapter failed: [in.c-main.special]: DB query failed: SQLSTATE[42S22]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'SOMETHING'. Tried 5 times.
 [in.c-main.special]: DB query failed: SQLSTATE[42S22]: [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Invalid column name 'SOMETHING'. Tried 5 times.

tests/functional/error-invalid-query/expected-stdout

@@ -1,7 +1,7 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "special" to "in.c-main.special".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT SOMETHING INVALID FROM "dbo"."special"' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exporting by "PDO" adapter.
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'

tests/functional/pdo-fallback-disabled-config-row/expected-stdout

@@ -1,7 +1,7 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "special" to "in.c-main.special".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT *  FROM "special";' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exporting by "PDO" adapter.
 Adapter "PDO" skipped: Disabled in configuration.

tests/functional/pdo-fallback-disabled/expected-stdout

@@ -1,7 +1,7 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "special" to "in.c-main.special".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT *  FROM "special";' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exporting by "PDO" adapter.
 Adapter "PDO" skipped: Disabled in configuration.

tests/functional/pdo-fallback/expected-stdout

@@ -1,7 +1,7 @@
 Connecting to DSN 'sqlsrv:Server=mssql,1433;Database=test'
 Exporting "special" to "in.c-main.special".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT *  FROM "special";' queryout 'in.c-main.special.csv' -S 'mssql,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exporting by "PDO" adapter.
 Exported "7" rows to "in.c-main.special".

tests/functional/simplified-pdo-fallback-query/expected-stderr

@@ -1 +1 @@
-Export by "BCP" adapter failed: Export process failed. Output:  Starting copy... SQLState = S1000, NativeError = 0 Error = [Microsoft][ODBC Driver 17 for SQL Server]Unable to resolve column level collations SQLState = 37000, NativeError = 8116 Error = [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Argument data type sql_variant is invalid for argument 1 of replace function.  BCP copy out failed .    Error Output: .
+Export by "BCP" adapter failed: Export process failed. Output:  Starting copy... %s .    Error Output: .

tests/functional/ssl-not-verify-cert-missing-certificate/expected-stdout

@@ -2,6 +2,6 @@ Connecting to DSN 'sqlsrv:Server=mssql-ssl,1433;Database=test;Encrypt=true;Trust
 Using SSL connection
 Exporting "sales" to "in.c-main.sales".
 Exporting by "BCP" adapter.
-Found database server version: 15.%d.%d.%d
+Found database server version: %d.%d.%d.%d
 Executing BCP command: bcp 'SELECT * FROM sales' queryout 'in.c-main.sales.csv' -S 'mssql-ssl,1433' -U 'sa' -P ***** -d 'test' -q -k -b 50000 -m 1 -t "," -r "\n" -c
 Exported "100" rows to "in.c-main.sales".