feat(migrations) allow DML migrations

Migrations can now execute arbitrary DML queries (they are not only for
DDL statements such as `CREATE`, `DROP`, `ALTER`...). This is very handy for
any migration such as plugin configuration schema changes, plugins
renaming, etc.

- plugins migrations modules are now under `kong.plugins.*.schema.migrations`

kong-0.5.4-1.rockspec

@@ -96,22 +96,22 @@ build = {
 
     ["kong.plugins.base_plugin"] = "kong/plugins/base_plugin.lua",
 
-    ["kong.plugins.basic-auth.migrations.cassandra"] = "kong/plugins/basic-auth/migrations/cassandra.lua",
+    ["kong.plugins.basic-auth.schema.migrations"] = "kong/plugins/basic-auth/schema/migrations.lua",
     ["kong.plugins.basic-auth.crypto"] = "kong/plugins/basic-auth/crypto.lua",
     ["kong.plugins.basic-auth.handler"] = "kong/plugins/basic-auth/handler.lua",
     ["kong.plugins.basic-auth.access"] = "kong/plugins/basic-auth/access.lua",
     ["kong.plugins.basic-auth.schema"] = "kong/plugins/basic-auth/schema.lua",
     ["kong.plugins.basic-auth.api"] = "kong/plugins/basic-auth/api.lua",
     ["kong.plugins.basic-auth.daos"] = "kong/plugins/basic-auth/daos.lua",
 
-    ["kong.plugins.key-auth.migrations.cassandra"] = "kong/plugins/key-auth/migrations/cassandra.lua",
+    ["kong.plugins.key-auth.schema.migrations"] = "kong/plugins/key-auth/schema/migrations.lua",
     ["kong.plugins.key-auth.handler"] = "kong/plugins/key-auth/handler.lua",
     ["kong.plugins.key-auth.access"] = "kong/plugins/key-auth/access.lua",
     ["kong.plugins.key-auth.schema"] = "kong/plugins/key-auth/schema.lua",
     ["kong.plugins.key-auth.api"] = "kong/plugins/key-auth/api.lua",
     ["kong.plugins.key-auth.daos"] = "kong/plugins/key-auth/daos.lua",
 
-    ["kong.plugins.oauth2.migrations.cassandra"] = "kong/plugins/oauth2/migrations/cassandra.lua",
+    ["kong.plugins.oauth2.schema.migrations"] = "kong/plugins/oauth2/schema/migrations.lua",
     ["kong.plugins.oauth2.handler"] = "kong/plugins/oauth2/handler.lua",
     ["kong.plugins.oauth2.access"] = "kong/plugins/oauth2/access.lua",
     ["kong.plugins.oauth2.schema"] = "kong/plugins/oauth2/schema.lua",
@@ -142,13 +142,13 @@ build = {
     ["kong.plugins.mashape-analytics.schema"] = "kong/plugins/mashape-analytics/schema.lua",
     ["kong.plugins.mashape-analytics.buffer"] = "kong/plugins/mashape-analytics/buffer.lua",
 
-    ["kong.plugins.rate-limiting.migrations.cassandra"] = "kong/plugins/rate-limiting/migrations/cassandra.lua",
+    ["kong.plugins.rate-limiting.schema.migrations"] = "kong/plugins/rate-limiting/schema/migrations.lua",
     ["kong.plugins.rate-limiting.handler"] = "kong/plugins/rate-limiting/handler.lua",
     ["kong.plugins.rate-limiting.access"] = "kong/plugins/rate-limiting/access.lua",
     ["kong.plugins.rate-limiting.schema"] = "kong/plugins/rate-limiting/schema.lua",
     ["kong.plugins.rate-limiting.daos"] = "kong/plugins/rate-limiting/daos.lua",
 
-    ["kong.plugins.response-ratelimiting.migrations.cassandra"] = "kong/plugins/response-ratelimiting/migrations/cassandra.lua",
+    ["kong.plugins.response-ratelimiting.schema.migrations"] = "kong/plugins/response-ratelimiting/schema/migrations.lua",
     ["kong.plugins.response-ratelimiting.handler"] = "kong/plugins/response-ratelimiting/handler.lua",
     ["kong.plugins.response-ratelimiting.access"] = "kong/plugins/response-ratelimiting/access.lua",
     ["kong.plugins.response-ratelimiting.header_filter"] = "kong/plugins/response-ratelimiting/header_filter.lua",
@@ -184,14 +184,14 @@ build = {
     ["kong.plugins.ip-restriction.access"] = "kong/plugins/ip-restriction/access.lua",
     ["kong.plugins.ip-restriction.schema"] = "kong/plugins/ip-restriction/schema.lua",
 
-    ["kong.plugins.acl.migrations.cassandra"] = "kong/plugins/acl/migrations/cassandra.lua",
+    ["kong.plugins.acl.schema.migrations"] = "kong/plugins/acl/schema/migrations.lua",
     ["kong.plugins.acl.handler"] = "kong/plugins/acl/handler.lua",
     ["kong.plugins.acl.access"] = "kong/plugins/acl/access.lua",
     ["kong.plugins.acl.schema"] = "kong/plugins/acl/schema.lua",
     ["kong.plugins.acl.api"] = "kong/plugins/acl/api.lua",
     ["kong.plugins.acl.daos"] = "kong/plugins/acl/daos.lua",
 
-    ["kong.plugins.acl.migrations.cassandra"] = "kong/plugins/acl/migrations/cassandra.lua",
+    ["kong.plugins.acl.schema.migrations"] = "kong/plugins/acl/schema/migrations.lua",
     ["kong.plugins.acl.handler"] = "kong/plugins/acl/handler.lua",
     ["kong.plugins.acl.access"] = "kong/plugins/acl/access.lua",
     ["kong.plugins.acl.schema"] = "kong/plugins/acl/schema.lua",
@@ -207,15 +207,15 @@ build = {
     ["kong.api.routes.plugins"] = "kong/api/routes/plugins.lua",
     ["kong.api.routes.plugins"] = "kong/api/routes/plugins.lua",
 
-    ["kong.plugins.jwt.migrations.cassandra"] = "kong/plugins/jwt/migrations/cassandra.lua",
+    ["kong.plugins.jwt.schema.migrations"] = "kong/plugins/jwt/schema/migrations.lua",
     ["kong.plugins.jwt.handler"] = "kong/plugins/jwt/handler.lua",
     ["kong.plugins.jwt.access"] = "kong/plugins/jwt/access.lua",
     ["kong.plugins.jwt.schema"] = "kong/plugins/jwt/schema.lua",
     ["kong.plugins.jwt.api"] = "kong/plugins/jwt/api.lua",
     ["kong.plugins.jwt.daos"] = "kong/plugins/jwt/daos.lua",
     ["kong.plugins.jwt.jwt_parser"] = "kong/plugins/jwt/jwt_parser.lua",
 
-    ["kong.plugins.hmac-auth.migrations.cassandra"] = "kong/plugins/hmac-auth/migrations/cassandra.lua",
+    ["kong.plugins.hmac-auth.schema.migrations"] = "kong/plugins/hmac-auth/schema/migrations.lua",
     ["kong.plugins.hmac-auth.handler"] = "kong/plugins/hmac-auth/handler.lua",
     ["kong.plugins.hmac-auth.access"] = "kong/plugins/hmac-auth/access.lua",
     ["kong.plugins.hmac-auth.schema"] = "kong/plugins/hmac-auth/schema.lua",

kong/cli/migrations.lua

@@ -29,7 +29,7 @@ end
 
 local config_path = cutils.get_kong_config_path(args.config)
 local configuration, dao_factory = IO.load_configuration_and_dao(config_path)
-local migrations = Migrations(dao_factory)
+local migrations = Migrations(dao_factory, configuration)
 
 local kind = args.type
 if kind ~= "all" and kind ~= "core" then
@@ -67,38 +67,33 @@ if args.command == "list" then
 
 elseif args.command == "up" then
 
-  local function migrate(identifier)
+  local function before(identifier)
     cutils.logger:info(string.format(
       "Migrating %s on keyspace \"%s\" (%s)",
       cutils.colors.yellow(identifier),
       cutils.colors.yellow(dao_factory._properties.keyspace),
       dao_factory.type
     ))
+  end
 
-    local err = migrations:migrate(identifier, function(identifier, migration)
-      if migration then
-        cutils.logger:info(string.format(
-          "%s migrated up to: %s",
-          identifier,
-          cutils.colors.yellow(migration.name)
-        ))
-      end
-    end)
-    if err then
-      cutils.logger:error_exit(err)
-    end
+  local function on_each_success(identifier, migration)
+    cutils.logger:info(string.format(
+      "%s migrated up to: %s",
+      identifier,
+      cutils.colors.yellow(migration.name)
+    ))
   end
 
   if kind == "all" then
-    migrate("core")
-    for _, plugin_name in ipairs(configuration.plugins_available) do
-      local has_migrations = utils.load_module_if_exists("kong.plugins."..plugin_name..".migrations."..dao_factory.type)
-      if has_migrations then
-        migrate(plugin_name)
-      end
+    local err = migrations:run_all_migrations(before, on_each_success)
+    if err then
+      cutils.logger:error_exit(err)
     end
   else
-    migrate(kind)
+    local err = migrations:run_migrations(kind, before, on_each_success)
+    if err then
+      cutils.logger:error_exit(err)
+    end
   end
 
   cutils.logger:success("Schema up to date")
@@ -109,20 +104,26 @@ elseif args.command == "down" then
     cutils.logger:error_exit("You must specify 'core' or a plugin name for this command.")
   end
 
-  cutils.logger:info(string.format(
-    "Rollbacking %s in keyspace \"%s\" (%s)",
-    cutils.colors.yellow(kind),
-    cutils.colors.yellow(dao_factory._properties.keyspace),
-    dao_factory.type
-  ))
+  local function before(identifier)
+    cutils.logger:info(string.format(
+      "Rollbacking %s in keyspace \"%s\" (%s)",
+      cutils.colors.yellow(identifier),
+      cutils.colors.yellow(dao_factory._properties.keyspace),
+      dao_factory.type
+    ))
+  end
+
+  local function on_success(identifier, migration)
+    if migration then
+      cutils.logger:success("\""..identifier.."\" rollbacked: "..cutils.colors.yellow(migration.name))
+    else
+      cutils.logger:success("No migration to rollback")
+    end
+  end
 
-  local rollbacked, err = migrations:rollback(kind)
+  local err = migrations:run_rollback(kind, before, on_success)
   if err then
     cutils.logger:error_exit(err)
-  elseif rollbacked then
-    cutils.logger:success("\""..kind.."\" rollbacked: "..cutils.colors.yellow(rollbacked.name))
-  else
-    cutils.logger:success("No migration to rollback")
   end
 
 elseif args.command == "reset" then

kong/cli/utils/signal.lua

@@ -162,7 +162,7 @@ end
 -- @param args_config Path to the desired configuration (usually from the --config CLI argument)
 local function prepare_database(args_config)
   local kong_config, _, dao_factory = get_kong_config(args_config)
-  local migrations = require("kong.tools.migrations")(dao_factory)
+  local migrations = require("kong.tools.migrations")(dao_factory, kong_config)
 
   local keyspace_exists, err = dao_factory.migrations:keyspace_exists()
   if err then
@@ -171,11 +171,24 @@ local function prepare_database(args_config)
     cutils.logger:info("Database not initialized. Running migrations...")
   end
 
-  local err = migrations:migrate_all(kong_config, function(identifier, migration)
-    if migration then
-      cutils.logger:success(string.format("%s migrated up to: %s", identifier, cutils.colors.yellow(migration.name)))
-    end
-  end)
+  local function before(identifier)
+    cutils.logger:info(string.format(
+      "Migrating %s on keyspace \"%s\" (%s)",
+      cutils.colors.yellow(identifier),
+      cutils.colors.yellow(dao_factory._properties.keyspace),
+      dao_factory.type
+    ))
+  end
+
+  local function on_each_success(identifier, migration)
+    cutils.logger:info(string.format(
+      "%s migrated up to: %s",
+      identifier,
+      cutils.colors.yellow(migration.name)
+    ))
+  end
+
+  local err = migrations:run_all_migrations(before, on_each_success)
   if err then
     cutils.logger:error_exit(err)
   end

kong/dao/cassandra/factory.lua

@@ -115,7 +115,7 @@ end
 -- Useful for huge DDL operations such as migrations
 -- @param {string} queries Semicolon separated string of queries
 -- @param {boolean} no_keyspace Won't set the keyspace if true
--- @return {string} error if any
+-- @return {table} error if any
 function CassandraFactory:execute_queries(queries, no_keyspace)
   local ok, err
   local session = cassandra:new()

kong/dao/cassandra/plugins.lua

@@ -37,11 +37,11 @@ function Plugins:_unmarshall(t)
 end
 
 -- @override
-function Plugins:update(t)
+function Plugins:update(t, full)
   if not t.consumer_id then
     t.consumer_id = constants.DATABASE_NULL_ID
   end
-  return Plugins.super.update(self, t)
+  return Plugins.super.update(self, t, full)
 end
 
 function Plugins:find_distinct()

kong/dao/cassandra/schema/migrations.lua

@@ -3,8 +3,8 @@ local Migrations = {
   {
     init = true,
     name = "2015-01-12-175310_skeleton",
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries([[
         CREATE KEYSPACE IF NOT EXISTS "]]..options.keyspace..[["
           WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};
 
@@ -14,19 +14,19 @@ local Migrations = {
           id text PRIMARY KEY,
           migrations list<text>
         );
-      ]]
+      ]], true)
     end,
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP KEYSPACE "]]..options.keyspace..[[";
       ]]
     end
   },
   -- init schema migration
   {
     name = "2015-01-12-175310_init_schema",
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         CREATE TABLE IF NOT EXISTS consumers(
           id uuid,
           custom_id text,
@@ -70,8 +70,8 @@ local Migrations = {
         CREATE INDEX IF NOT EXISTS ON plugins(consumer_id);
       ]]
     end,
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP TABLE consumers;
         DROP TABLE apis;
         DROP TABLE plugins;

kong/plugins/acl/migrations/cassandra.lua → kong/plugins/acl/schema/migrations.lua

@@ -1,8 +1,8 @@
 local Migrations = {
   {
     name = "2015-08-25-841841_init_acl",
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         CREATE TABLE IF NOT EXISTS acls(
           id uuid,
           consumer_id uuid,
@@ -15,8 +15,8 @@ local Migrations = {
         CREATE INDEX IF NOT EXISTS acls_consumer_id ON acls(consumer_id);
       ]]
     end,
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP TABLE acls;
       ]]
     end

kong/plugins/basic-auth/migrations/cassandra.lua → kong/plugins/basic-auth/schema/migrations.lua

@@ -1,8 +1,8 @@
 local Migrations = {
   {
     name = "2015-08-03-132400_init_basicauth",
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries [[
          CREATE TABLE IF NOT EXISTS basicauth_credentials(
           id uuid,
           consumer_id uuid,
@@ -16,8 +16,8 @@ local Migrations = {
         CREATE INDEX IF NOT EXISTS basicauth_consumer_id ON basicauth_credentials(consumer_id);
       ]]
     end,
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP TABLE basicauth_credentials;
       ]]
     end

kong/plugins/hmac-auth/migrations/cassandra.lua → kong/plugins/hmac-auth/schema/migrations.lua

@@ -1,8 +1,8 @@
 local Migrations = {
   {
     name = "2015-09-16-132400_init_hmacauth",
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries [[
          CREATE TABLE IF NOT EXISTS hmacauth_credentials(
           id uuid,
           consumer_id uuid,
@@ -16,8 +16,8 @@ local Migrations = {
         CREATE INDEX IF NOT EXISTS hmacauth_consumer_id ON hmacauth_credentials(consumer_id);
       ]]
     end,
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP TABLE hmacauth_credentials;
       ]]
     end

kong/plugins/jwt/migrations/cassandra.lua → kong/plugins/jwt/schema/migrations.lua

@@ -1,9 +1,8 @@
 local Migration = {
   {
     name = "2015-06-09-jwt-auth",
-
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         CREATE TABLE IF NOT EXISTS jwt_secrets(
           id uuid,
           consumer_id uuid,
@@ -18,9 +17,8 @@ local Migration = {
         CREATE INDEX IF NOT EXISTS ON jwt_secrets(consumer_id);
       ]]
     end,
-
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP TABLE jwt_secrets;
       ]]
     end

kong/plugins/key-auth/migrations/cassandra.lua → kong/plugins/key-auth/schema/migrations.lua

@@ -1,8 +1,8 @@
 local Migrations = {
   {
     name = "2015-07-31-172400_init_keyauth",
-    up = function(options)
-      return [[
+    up = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         CREATE TABLE IF NOT EXISTS keyauth_credentials(
           id uuid,
           consumer_id uuid,
@@ -15,8 +15,8 @@ local Migrations = {
         CREATE INDEX IF NOT EXISTS keyauth_consumer_id ON keyauth_credentials(consumer_id);
       ]]
     end,
-    down = function(options)
-      return [[
+    down = function(options, dao_factory)
+      return dao_factory:execute_queries [[
         DROP TABLE keyauth_credentials;
       ]]
     end