From 3a3f56c4e01a53969ed26251958aa29ebf241830 Mon Sep 17 00:00:00 2001 From: Chaitanya Baraskar Date: Wed, 5 Jan 2022 12:24:43 +0530 Subject: [PATCH] Added new Blueprint for MS SQL Server (#1165) * Added new blueprint for mssql backup/restore * removing commented code * Updated blueprint to delete backup files from sql server pod * Added README.md file with instructions * Updated image repo * Addressing review comments * updating variable name * added note on default storage class * address review comments * fixing config * push mssql-tools image Co-authored-by: Prasad Ghangal --- .goreleaser.yml | 5 + build/push_images.sh | 2 +- docker/mssql-tools/Dockerfile | 10 + examples/stable/mssql/README.md | 352 +++++++++++++++++++++ examples/stable/mssql/mssql-blueprint.yaml | 85 +++++ 5 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 docker/mssql-tools/Dockerfile create mode 100644 examples/stable/mssql/README.md create mode 100644 examples/stable/mssql/mssql-blueprint.yaml diff --git a/.goreleaser.yml b/.goreleaser.yml index dfc1993e33..68d6834544 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -127,6 +127,11 @@ dockers: dockerfile: 'docker/kafka-adobes3Connector/image/adobeSink.Dockerfile' extra_files: - 'docker/kafka-adobes3Connector/image/adobe-monitorsink.sh' +- ids: + - kando + image_templates: + - 'ghcr.io/kanisterio/mssql-tools:{{ .Tag }}' + dockerfile: 'docker/mssql-tools/Dockerfile' snapshot: name_template: '{{ .Tag }}' checksum: diff --git a/build/push_images.sh b/build/push_images.sh index cf9c04fa65..ae6af859a0 100755 --- a/build/push_images.sh +++ b/build/push_images.sh @@ -18,7 +18,7 @@ set -o errexit set -o nounset IMAGE_REGISTRY="ghcr.io/kanisterio" -IMAGES=("mysql-sidecar" "kafka-adobe-s3-sink-connector" "postgres-kanister-tools" "postgresql" "postgres-tools-9.6" "cassandra" "kanister-kubectl-1.18" "mongo-sidecar" "mongodb" "es-sidecar" "controller" "kanister-tools" "couchbase-tools" "kafka-adobe-s3-source-connector" "foundationdb") +IMAGES=("mysql-sidecar" "kafka-adobe-s3-sink-connector" "postgres-kanister-tools" "postgresql" "postgres-tools-9.6" "cassandra" "kanister-kubectl-1.18" "mongo-sidecar" "mongodb" "es-sidecar" "controller" "kanister-tools" "couchbase-tools" "kafka-adobe-s3-source-connector" "foundationdb" "mssql-tools") TAG=${1:-"v9.99.9-dev"} diff --git a/docker/mssql-tools/Dockerfile b/docker/mssql-tools/Dockerfile new file mode 100644 index 0000000000..3eef65976a --- /dev/null +++ b/docker/mssql-tools/Dockerfile @@ -0,0 +1,10 @@ +FROM mcr.microsoft.com/mssql-tools +ENV kubectl_version="v1.18.0" + +ADD kando /usr/local/bin/ + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/amd64/kubectl \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + +CMD [ "/usr/bin/tail", "-f", "/dev/null" ] diff --git a/examples/stable/mssql/README.md b/examples/stable/mssql/README.md new file mode 100644 index 0000000000..1cea51a146 --- /dev/null +++ b/examples/stable/mssql/README.md @@ -0,0 +1,352 @@ +# Microsoft SQL Server + +MS SQL Server is a relational database management system (RDBMS) developed by Microsoft. This product is built for the basic function of storing retrieving data as required by other applications. +It can be run either on the same computer or on another across a network. + +## Introduction +This document will cover how to install SQL Server and how to run backup/restore actions. + +## Prerequisites + +- Kubernetes 1.6+ with Beta APIs enabled +- PV provisioner support in the underlying infrastructure +- Kanister controller version 0.71.0 installed in your cluster, let's assume in Namespace `kanister` +- Kanctl CLI installed (https://docs.kanister.io/tooling.html#install-the-tools) + +## Installing Microsoft SQL Server + +Resources created as part of installation +- PVC +- Deployment +- Service + +### Create Namespace + +```bash +$ kubectl create ns sqlserver +``` + +### Create Password + +```bash +$ kubectl create secret generic mssql --from-literal=SA_PASSWORD="MyC0m9l&xP@ssw0rd" -n sqlserver +``` + +### Create storage +Execute following commands to create PVC for SQL Server installation. +Default storage class will be used to provision PVC. +```bash +$ cat < CREATE DATABASE TestDB +2> SELECT Name from sys.Databases +3> GO +Name +--------------------------------------------------------------------------- +master +tempdb +model +msdb +TestDB + +# Create table "Inventory" inside database "TestDB" +1> USE TestDB +2> CREATE TABLE Inventory (id INT, name NVARCHAR(50), quantity INT) +3> INSERT INTO Inventory VALUES (1, 'banana', 150); INSERT INTO Inventory VALUES (2, 'orange', 154); +4> GO + +(1 rows affected) + +(1 rows affected) + +# View data in "Inventory" table +1> SELECT * FROM Inventory; +2> GO +id name quantity +----------- -------------------------------------------------- ----------- + 1 banana 150 + 2 orange 154 + +``` +After following all given steps database named `TestDB` should have table called `Inventory` + +## Integrating with Kanister + +If you have deployed SQL Server with name other than `mssql-deployment` and namespace other than `sqlserver`, +you need to modify the commands(backup, restore and delete) used below to use the correct release name and namespace + +### Create Profile +Create Profile CR if not created already + +```bash +$ kanctl create profile s3compliant --access-key \ + --secret-key \ + --bucket \ + --region \ + --namespace kanister +``` +You can read more about the Profile custom Kanister resource [here](https://docs.kanister.io/architecture.html?highlight=profile#profiles). + +**NOTE:** + +The above command will configure a location where artifacts resulting from Kanister +data operations such as backup should go. This is stored as a `profiles.cr.kanister.io` +*CustomResource (CR)* which is then referenced in Kanister ActionSets. Every ActionSet +requires a Profile reference to complete the action. This CR (`profiles.cr.kanister.io`) +can be shared between Kanister-enabled application instances. + +### Create Blueprint + +Create Blueprint in the same namespace as the Kanister controller + +Execute following command to create the blueprint +```bash +$ kubectl create -f ./mssql-blueprint.yaml -n kanister +``` +Blueprint with name `mssql-blueprint` will be created in `kanister` namespace + +## Protect the Application + +You can now take a backup of the SQL Server data using an ActionSet defining backup for this application. +Create an ActionSet in the same namespace as the controller. + +```bash +# Find profile name +$ kubectl get profile -n kanister +NAME AGE +s3-profile-k4r8w 7d1h + +# Create Actionset +# Please make sure the value of profile and blueprint matches with the names of profile and blueprint that we have created already +$ kanctl create actionset --action backup --namespace kanister --blueprint mssql-blueprint --profile s3-profile-k4r8w --secrets mssql=sqlserver/mssql --deployment sqlserver/mssql-deployment +actionset backup-dzchc created + + +$ kubectl get actionsets -n kanister +NAME AGE +backup-dzchc 29s + +# View the status of the actionset +# Please make sure the name of the actionset here matches with the name of actionset that we have created above and make sure the status is complete. +$ kubectl describe actionset backup-dzchc -n kanister +``` + +### Disaster strikes! + +Let's say someone accidentally deleted the test database using the following command: + +```bash +# Connect to SQL Sever by running a shell inside mssql pod +$ kubectl exec -it -n sqlserver $(kubectl get pods --selector=app=mssql -o=jsonpath='{.items[0].metadata.name}' -n sqlserver) -- bash + +$ /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "MyC0m9l&xP@ssw0rd" + +1> SELECT Name from sys.Databases +2> GO +Name +--------------------------------------------------------------------------- +master +tempdb +model +msdb +TestDB + +# Drop database "TestDB" +1> DROP DATABASE TestDB +2> GO + +# View list of databases available +1> SELECT Name from sys.Databases +2> go +Name +-------------------------------------------------------------------------------------------------------------------------------- +master +tempdb +model +msdb + +``` + +### Restore the Application + +To restore the missing data, you should use the backup that you created before. An easy way to do this is to leverage `kanctl`, a command-line tool that helps create ActionSets that depend on other ActionSets: + +```bash +# Make sure to use correct backup actionset name here +$ kanctl create actionset --action restore --namespace kanister --from "backup-dzchc" +actionset restore-backup-dzchc-vqr5v created + +# View the status of the ActionSet +# Make sure to use correct restore actionset name here +$ kubectl describe actionset restore-backup-dzchc-vqr5v -n kanister +``` + +Once the ActionSet status is set to "complete", you can see that the data has been successfully restored to SQL Server. + +```bash +1> SELECT Name from sys.Databases +2> GO +Name +--------------------------------------------------------------------------- +master +tempdb +model +msdb +TestDB + +# View data in "Inventory" table +1> USE TestDB +2> SELECT * FROM Inventory; +3> GO +id name quantity +----------- -------------------------------------------------- ----------- + 1 banana 150 + 2 orange 154 + +``` + +### Delete the Artifacts + +The artifacts created by the backup action can be cleaned up using the following command: + +```bash +$ kanctl create actionset --action delete --namespace kanister --from "backup-dzchc" +actionset delete-backup-dzchc-kcvkg created + +# View the status of the ActionSet +$ kubectl describe actionset delete-backup-dzchc-kcvkg -n kanister +``` + +## Troubleshooting + +If you run into any issues with the above commands, you can check the logs of the controller using: + +```bash +$ kubectl --namespace kanister logs -l app=kanister-operator +``` + +You can also check events of the actionset + +```bash +$ kubectl describe actionset restore-backup-dzchc-vqr5v -n kanister +``` + +## Cleanup + +### Uninstalling the SQL Server + +To uninstall/delete the `mssql` deployment: + +```bash +# Delete deployment, service and pvc +$ kubectl delete service/mssql-deployment -n sqlserver +$ kubectl delete deployment.apps/mssql-deployment -n sqlserver +$ kubectl delete pvc mssql-data -n sqlserver +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +### Delete CRs +Remove Blueprint and Profile CR + +```bash +$ kubectl delete blueprint mssql-blueprint -n kanister +blueprint.cr.kanister.io "mssql-blueprint" deleted + +$ kubectl get profiles -n kanister +NAME AGE +s3-profile-k4r8w 122m + +$ kubectl delete profile s3-profile-k4r8w -n kanister +profile.cr.kanister.io "s3-profile-k4r8w" deleted +``` diff --git a/examples/stable/mssql/mssql-blueprint.yaml b/examples/stable/mssql/mssql-blueprint.yaml new file mode 100644 index 0000000000..6f32a1c67a --- /dev/null +++ b/examples/stable/mssql/mssql-blueprint.yaml @@ -0,0 +1,85 @@ +apiVersion: cr.kanister.io/v1alpha1 +kind: Blueprint +metadata: + name: mssql-blueprint + namespace: kanister +actions: + backup: + outputArtifacts: + mssqlCloudDump: + keyValue: + s3path: "{{ .Phases.dumpToObjectStore.Output.s3path }}" + secretNames: + - mssql + phases: + - func: KubeTask + name: dumpToObjectStore + args: + image: ghcr.io/kanisterio/mssql-tools:0.71.0 + command: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + s3_path="/mssql-backups/{{ index .Deployment.Name }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02T15-04-05" }}/dump.sql.gz" + root_password="{{ .Secrets.mssql.Data.SA_PASSWORD | toString }}" + root_uname="sa" + mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}" + server_name="{{ index .Deployment.Name }}.{{index .Deployment.Namespace}}.svc.cluster.local" + databases=$(/opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "SET NOCOUNT ON; SELECT name FROM sys.databases WHERE name NOT IN ('master','model','msdb','tempdb')" -b -s "," -h -1) + for database in $databases; do /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "backup database $database to disk = '/tmp/backup/$database.bak' with format;"; done + kubectl cp ${mssql_pod}:/tmp/backup /tmp/backup + cd /tmp/ + tar -zcvf backup.tar.gz backup/ + kando location push --profile '{{ toJson .Profile }}' --path ${s3_path} backup.tar.gz + kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup + kando output s3path ${s3_path} + restore: + inputArtifactNames: + - mssqlCloudDump + secretNames: + - mssql + phases: + - func: KubeTask + name: restoreFromObjectStore + args: + image: ghcr.io/kanisterio/mssql-tools:0.71.0 + command: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + s3_path="{{ .ArtifactsIn.mssqlCloudDump.KeyValue.s3path }}" + root_password="{{ .Secrets.mssql.Data.SA_PASSWORD | toString }}" + root_uname="sa" + mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}" + server_name="{{ index .Deployment.Name }}.{{index .Deployment.Namespace}}.svc.cluster.local" + kando location pull --profile '{{ toJson .Profile }}' --path ${s3_path} - | tar zxvf - -C /tmp/ + kubectl cp /tmp/backup ${mssql_pod}:/tmp/backup + backup_files=$(ls /tmp/backup) + for script in $backup_files; do database="$(cut -d'.' -f1 <<<"$script")"; /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "restore database $database from disk = '/tmp/backup/$script' with replace"; done + kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup + delete: + inputArtifactNames: + - mssqlCloudDump + phases: + - func: KubeTask + name: deleteFromBlobStore + args: + image: ghcr.io/kanisterio/mssql-tools:0.71.0 + command: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + s3_path="{{ .ArtifactsIn.mssqlCloudDump.KeyValue.s3path }}" + kando location delete --profile '{{ toJson .Profile }}' --path ${s3_path}