Support MySQL backend

Bug: T242055
wikimedia · Jul 27, 2020 · 460ee9a · 460ee9a
1 parent e578e79
commit 460ee9a
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 13 deletions.
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -18,7 +18,7 @@ jobs:
       matrix:
         node-version: [6.11, 10.x, 12.x]
         cassandra-version: [3.11.4]
-        test-target: [sqlite, cassandra]
+        test-target: [sqlite, cassandra, mysql]
         test-mode: [fs, fefs, febe]
 
     steps:
@@ -43,5 +43,12 @@ jobs:
         sed -i -e 's/^-XX:+UseNUMA/#-XX:+UseNUMA/' ../apache-cassandra-${{ matrix.cassandra-version }}/conf/jvm.options
         export PATH=${PATH}:../apache-cassandra-${{ matrix.cassandra-version }}/bin
         cassandra
+    - name: Setup MySql
+      if: matrix.test-target == 'mysql'
+      run: |
+        sudo systemctl start mysql.service
+        mysql -uroot -proot -e "CREATE DATABASE \`test_db\`;";
+        mysql -uroot -proot -e "CREATE USER 'mysql'@\`%\` IDENTIFIED BY 'mysql';";
+        mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON \`test_db\`.* TO 'mysql'@\`%\`;";
     - run: npm install
     - run: npm run lint && npm run coverage -- ${{ matrix.test-target }} ${{ matrix.test-mode }}
diff --git a/config.backend.test.yaml b/config.backend.test.yaml
@@ -28,14 +28,15 @@ default_project: &default_project
                               backend: '{env(RB_TEST_BACKEND, sqlite)}'
                               hosts: [localhost]
                               keyspace: system
-                              username: cassandra
-                              password: cassandra
+                              username: '{env(RB_TEST_BACKEND_USERNAME, cassandra)}'
+                              password: '{env(RB_TEST_BACKEND_PASSWORD, cassandra)}'
                               defaultConsistency: one # or 'localQuorum' for production
                               storage_groups:
                                 - name: test.group.local
                                   domains: /./
-                              # ignored in cassandra, but useful in SQLite testing
-                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}'
+                              host: localhost # ignored in cassandra, but useful in MySQL testing
+                              database: test_db # ignored in cassandra, but useful in MySQL testing
+                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}' # ignored in cassandra, but useful in SQLite testing
                     /action:
                       x-modules:
                         - path: sys/action.js

diff --git a/config.fullstack.test.yaml b/config.fullstack.test.yaml
@@ -68,14 +68,15 @@ default_project: &default_project
                               backend: '{env(RB_TEST_BACKEND, sqlite)}'
                               hosts: [localhost]
                               keyspace: system
-                              username: cassandra
-                              password: cassandra
+                              username: '{env(RB_TEST_BACKEND_USERNAME, cassandra)}'
+                              password: '{env(RB_TEST_BACKEND_PASSWORD, cassandra)}'
                               defaultConsistency: one # or 'localQuorum' for production
                               storage_groups:
                                 - name: test.group.local
                                   domains: /./
-                              # ignored in cassandra, but useful in SQLite testing
-                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}'
+                              host: localhost # ignored in cassandra, but useful in MySQL testing
+                              database: test_db # ignored in cassandra, but useful in MySQL testing
+                              dbname: '{env(RB_SQLITE_FILE, test.db.sqlite3)}' # ignored in cassandra, but useful in SQLite testing
 
 wikimedia_project: &wikimedia_project
   x-modules:

diff --git a/package.json b/package.json
@@ -25,6 +25,7 @@
     "tables",
     "queues",
     "cassandra",
+    "mysql",
     "kafka"
   ],
   "author": "Wikimedia Service Team <services@wikimedia.org>",
@@ -43,6 +44,7 @@
     "jsonwebtoken": "^8.5.1",
     "mediawiki-title": "^0.7.2",
     "restbase-mod-table-cassandra": "^1.2.1",
+    "restbase-mod-table-mysql": "^0.1.1",
     "semver": "^7.3.2",
     "service-runner": "^2.7.8",
     "uuid": "^7.0.3"

diff --git a/sys/table.js b/sys/table.js
@@ -9,8 +9,11 @@ module.exports = (options) => {
     options.conf.backend = options.conf.backend || 'cassandra';
     options.log = options.logger.log.bind(options.logger);
 
-    if (options.conf.backend !== 'cassandra' &&
-            options.conf.backend !== 'sqlite') {
+    if (
+        options.conf.backend !== 'cassandra' &&
+        options.conf.backend !== 'mysql' &&
+        options.conf.backend !== 'sqlite'
+    ) {
         throw new Error(`Unsupported backend version specified: ${options.conf.backend}`);
     }
 

diff --git a/test/utils/cleandb.sh b/test/utils/cleandb.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+rb_test_backend=${RB_TEST_BACKEND:-$2}
+
 dropKeyspaces ( ) {
   if [ "$#" -eq 1 ]
   then
@@ -13,4 +15,26 @@ dropKeyspaces ( ) {
   fi
 }
 
-dropKeyspaces "local_group_test"
+dropTables ( ) {
+  if [ "$#" -eq 1 ]
+  then
+    DATABASE=$1
+    echo "looking for database named '*$DATABASE*'..."
+    echo 'begin;' | mysql -BD${DATABASE}
+    for TABLE in `echo 'show tables;' | mysql -BD${DATABASE}`
+    do
+      echo dropping table $TABLE
+      echo "drop table if exists $TABLE;" | mysql -BD${DATABASE}
+    done
+    echo 'commit;' | mysql -BD${DATABASE}
+  fi
+}
+
+if [ "$rb_test_backend" = "cassandra" ]; then
+  dropKeyspaces "local_group_test"
+elif [ "$rb_test_backend" = "mysql" ]; then
+  dropTables "test_db"
+else
+    echo "Invalid TEST_TARGET $rb_test_backend. Must me 'sqlite', 'cassandra' or 'mysql' if specified"
+    exit 1
+fi
diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh
@@ -33,9 +33,24 @@ elif [ "$test_target" = "cassandra" ]; then
       echo "Cassandra is ready."
     fi
     export RB_TEST_BACKEND=cassandra
+    export RB_TEST_BACKEND_USERNAME=cassandra
+    export RB_TEST_BACKEND_PASSWORD=cassandra
+    sh ./test/utils/cleandb.sh local_group_test
+elif [ "$test_target" = "mysql" ]; then
+    echo "Running with MySQL backend"
+    if [ `nc -z localhost 3306 < /dev/null; echo $?` != 0 ]; then
+      echo "Waiting for MySQL to start..."
+      while [ `nc -z localhost 3306; echo $?` != 0 ]; do
+        sleep 1
+      done
+      echo "MySQL is ready."
+    fi
+    export RB_TEST_BACKEND=mysql
+    export RB_TEST_BACKEND_USERNAME=mysql
+    export RB_TEST_BACKEND_PASSWORD=mysql
     sh ./test/utils/cleandb.sh local_group_test
 else
-    echo "Invalid TEST_TARGET $test_target. Must me 'sqlite' or 'cassandra' if specified"
+    echo "Invalid TEST_TARGET $test_target. Must me 'sqlite', 'cassandra' or 'mysql' if specified"
     exit 1
 fi